mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
9
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fd85b80c78 | ||
|
|
464a8da86b | ||
|
|
9b6a525439 | ||
|
|
61a0e0d218 | ||
|
|
67b7227abf | ||
|
|
02d1d69379 | ||
|
|
025751ef03 | ||
|
|
13a61b2688 | ||
|
|
1508839bf9 |
@@ -1,5 +1,5 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
|
||||
import { RECORD_START, RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
|
||||
import { handleRecordContentMessage, startRecordCapture } from "../record-capture";
|
||||
|
||||
vi.stubGlobal("chrome", {
|
||||
@@ -96,6 +96,65 @@ describe("handleRecordContentMessage stop/cancel", () => {
|
||||
expect(dispose).not.toHaveBeenCalled();
|
||||
expect(onStop).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps a failed STOP retryable and redelivers its recorded step", async () => {
|
||||
document.body.innerHTML = `<label for="retry">Draft</label><input id="retry" />`;
|
||||
const sendMessage = vi.mocked(chrome.runtime.sendMessage);
|
||||
sendMessage.mockReset();
|
||||
sendMessage.mockRejectedValueOnce(new Error("service worker unavailable"));
|
||||
sendMessage.mockResolvedValueOnce(undefined);
|
||||
|
||||
let activeRequestId: string | null = null;
|
||||
let capture: ReturnType<typeof startRecordCapture> | null = null;
|
||||
const onStop = vi.fn();
|
||||
const dispatch = (
|
||||
message:
|
||||
| { type: typeof RECORD_START; requestId: string; startedAtMs?: number }
|
||||
| { type: typeof RECORD_STOP; requestId: string },
|
||||
sendResponse?: (response: unknown) => void,
|
||||
) =>
|
||||
handleRecordContentMessage(
|
||||
message,
|
||||
{
|
||||
activeRequestId,
|
||||
capture,
|
||||
setActiveRequestId: (id) => {
|
||||
activeRequestId = id;
|
||||
},
|
||||
setCapture: (next) => {
|
||||
capture = next;
|
||||
},
|
||||
onStart: vi.fn(),
|
||||
onStop,
|
||||
},
|
||||
sendResponse,
|
||||
);
|
||||
|
||||
dispatch({ type: RECORD_START, requestId: "rec-retry" });
|
||||
const input = document.querySelector("input")!;
|
||||
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||
input.value = "dirty final value";
|
||||
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
|
||||
const firstResponse = vi.fn();
|
||||
expect(dispatch({ type: RECORD_STOP, requestId: "rec-retry" }, firstResponse)).toBe(true);
|
||||
await vi.waitFor(() =>
|
||||
expect(firstResponse).toHaveBeenCalledWith({
|
||||
ok: false,
|
||||
error: "failed to deliver one or more recorded steps",
|
||||
}),
|
||||
);
|
||||
expect(activeRequestId).toBe("rec-retry");
|
||||
expect(onStop).not.toHaveBeenCalled();
|
||||
|
||||
const retryResponse = vi.fn();
|
||||
expect(dispatch({ type: RECORD_STOP, requestId: "rec-retry" }, retryResponse)).toBe(true);
|
||||
await vi.waitFor(() => expect(retryResponse).toHaveBeenCalledWith({ ok: true }));
|
||||
|
||||
expect(sendMessage).toHaveBeenCalledTimes(2);
|
||||
expect(activeRequestId).toBeNull();
|
||||
expect(onStop).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("record-capture semantic", () => {
|
||||
@@ -228,6 +287,7 @@ describe("record-capture semantic", () => {
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "hover",
|
||||
target: { role: "button", name: "Open user navigation menu" },
|
||||
geometry: { tag: "button", rect: { x: 900, y: 8, w: 32, h: 32 } },
|
||||
});
|
||||
expect(steps[1]).toMatchObject({
|
||||
op: "click",
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import {
|
||||
type CaptureTargetDescriptor,
|
||||
describeEventTarget,
|
||||
describeTarget,
|
||||
resolveClickableElement,
|
||||
resolveHoverElement,
|
||||
type TargetDescriptor,
|
||||
} from "@/lib/describe-target";
|
||||
import {
|
||||
evaluateHoverTrigger,
|
||||
@@ -39,17 +39,26 @@ import {
|
||||
|
||||
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
|
||||
const failedStepDeliveries = new Set<string>();
|
||||
const failedStepPayloads = new Map<string, RecordStepPayload[]>();
|
||||
const pendingStopFlushes = new Map<string, Promise<RecordStopAck>>();
|
||||
const knownRecordRequests = new Set<string>();
|
||||
|
||||
function sendRecordStep(requestId: string, step: RecordStepPayload): void {
|
||||
function deliverRecordStep(requestId: string, step: RecordStepPayload): Promise<boolean> {
|
||||
const payload = { type: RECORD_STEP, requestId, step };
|
||||
const pending = Promise.resolve(chrome.runtime.sendMessage(payload)).then(
|
||||
return Promise.resolve(chrome.runtime.sendMessage(payload)).then(
|
||||
() => true,
|
||||
() => {
|
||||
failedStepDeliveries.add(requestId);
|
||||
const failed = failedStepPayloads.get(requestId) ?? [];
|
||||
failed.push(step);
|
||||
failedStepPayloads.set(requestId, failed);
|
||||
return false;
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function sendRecordStep(requestId: string, step: RecordStepPayload): void {
|
||||
const pending = deliverRecordStep(requestId, step);
|
||||
const sends = pendingStepSends.get(requestId) ?? new Set<Promise<boolean>>();
|
||||
sends.add(pending);
|
||||
pendingStepSends.set(requestId, sends);
|
||||
@@ -65,14 +74,15 @@ export interface RecordCaptureController {
|
||||
|
||||
interface FillSession {
|
||||
element: FillableElement;
|
||||
target: TargetDescriptor;
|
||||
target: CaptureTargetDescriptor;
|
||||
baselineValue: string;
|
||||
lastValue: string;
|
||||
pendingCommit?: "enter" | "suggestion" | "blur";
|
||||
}
|
||||
|
||||
interface HoverCandidate {
|
||||
element: Element;
|
||||
target: TargetDescriptor;
|
||||
target: CaptureTargetDescriptor;
|
||||
recordedAt: number;
|
||||
score: number;
|
||||
eligible: boolean;
|
||||
@@ -157,6 +167,28 @@ function nearbyFillableFromSearchChrome(target: Element): FillableElement | null
|
||||
return null;
|
||||
}
|
||||
|
||||
function captureGeometry(el: Element): RecordStepPayload["geometry"] {
|
||||
const rect = el.getBoundingClientRect();
|
||||
const style = window.getComputedStyle(el);
|
||||
return {
|
||||
rect: { x: rect.x, y: rect.y, w: rect.width, h: rect.height },
|
||||
scrollX: window.scrollX,
|
||||
scrollY: window.scrollY,
|
||||
position: style.position,
|
||||
tag: el.tagName.toLowerCase(),
|
||||
};
|
||||
}
|
||||
|
||||
function geometryForEventTarget(
|
||||
target: EventTarget | null,
|
||||
): RecordStepPayload["geometry"] | undefined {
|
||||
if (!(target instanceof Element)) return undefined;
|
||||
const clickable = target.closest(
|
||||
'a,button,input,select,textarea,[role="button"],[role="link"],[role="menuitem"],[contenteditable="true"]',
|
||||
);
|
||||
return captureGeometry(clickable ?? target);
|
||||
}
|
||||
|
||||
function fillableValue(el: FillableElement): string {
|
||||
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
|
||||
return el.value;
|
||||
@@ -232,7 +264,10 @@ function collectHoverTriggerLabelText(root: Element): string {
|
||||
return normalizeLabelText(text);
|
||||
}
|
||||
|
||||
function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDescriptor {
|
||||
function compactHoverTargetName(
|
||||
el: Element,
|
||||
desc: CaptureTargetDescriptor,
|
||||
): CaptureTargetDescriptor {
|
||||
if (!desc.name) return desc;
|
||||
const fullText = normalizeLabelText(el.textContent ?? "");
|
||||
const compactName = collectHoverTriggerLabelText(el);
|
||||
@@ -250,7 +285,7 @@ function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDesc
|
||||
return desc;
|
||||
}
|
||||
|
||||
function isWeakHoverTarget(target: TargetDescriptor): boolean {
|
||||
function isWeakHoverTarget(target: CaptureTargetDescriptor): boolean {
|
||||
return !target.role && !target.name && target.tag === "div";
|
||||
}
|
||||
|
||||
@@ -261,9 +296,9 @@ function looksLikeAvatarElement(el: Element): boolean {
|
||||
|
||||
function normalizeHoverTarget(
|
||||
el: Element,
|
||||
desc: TargetDescriptor,
|
||||
desc: CaptureTargetDescriptor,
|
||||
decision: HoverTriggerDecision,
|
||||
): TargetDescriptor {
|
||||
): CaptureTargetDescriptor {
|
||||
if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
|
||||
return { ...desc, name: "image" };
|
||||
}
|
||||
@@ -277,7 +312,7 @@ function normalizeHoverTarget(
|
||||
return desc;
|
||||
}
|
||||
|
||||
function hoverTriggerSignals(el: Element, desc: TargetDescriptor) {
|
||||
function hoverTriggerSignals(el: Element, desc: CaptureTargetDescriptor) {
|
||||
if (!(el instanceof HTMLElement)) return null;
|
||||
const style = hoverTriggerStyle(el);
|
||||
return {
|
||||
@@ -411,14 +446,17 @@ export function startRecordCapture(
|
||||
emitStep({
|
||||
op: "fill",
|
||||
target: session.target,
|
||||
geometry: captureGeometry(session.element),
|
||||
value,
|
||||
commit: session.pendingCommit ?? "blur",
|
||||
...(isPassword ? { redacted: true } : {}),
|
||||
});
|
||||
};
|
||||
|
||||
const commitFillSession = () => {
|
||||
const commitFillSession = (commit: "enter" | "suggestion" | "blur" = "blur") => {
|
||||
if (!fillSession || composing) return;
|
||||
const session = fillSession;
|
||||
session.pendingCommit = commit;
|
||||
fillSession = null;
|
||||
emitFill(session);
|
||||
committedValues.set(session.element, session.lastValue);
|
||||
@@ -624,6 +662,7 @@ export function startRecordCapture(
|
||||
emitStep({
|
||||
op: "hover",
|
||||
target: hover.target,
|
||||
geometry: captureGeometry(hover.element),
|
||||
});
|
||||
emittedHoverElements.add(hover.element);
|
||||
};
|
||||
@@ -678,6 +717,7 @@ export function startRecordCapture(
|
||||
emitStep({
|
||||
op: "click",
|
||||
target,
|
||||
geometry: geometryForEventTarget(eventTarget(event)),
|
||||
expects_navigation: true,
|
||||
});
|
||||
};
|
||||
@@ -779,7 +819,7 @@ export function startRecordCapture(
|
||||
scheduleInputCompletionCommit(
|
||||
sessionElement,
|
||||
syncFillSessionValue,
|
||||
commitFillSession,
|
||||
() => commitFillSession("suggestion"),
|
||||
(el) => fillSession?.element === el,
|
||||
);
|
||||
return;
|
||||
@@ -839,6 +879,7 @@ export function startRecordCapture(
|
||||
emitStep({
|
||||
op: "select",
|
||||
target: desc,
|
||||
geometry: captureGeometry(target),
|
||||
values,
|
||||
labels,
|
||||
expects_navigation: true,
|
||||
@@ -879,7 +920,7 @@ export function startRecordCapture(
|
||||
};
|
||||
}
|
||||
if (fillable) {
|
||||
commitFillSession();
|
||||
commitFillSession(event.key === "Enter" ? "enter" : "blur");
|
||||
}
|
||||
const desc = describeEventTarget(target);
|
||||
if (!desc && !event.key) return;
|
||||
@@ -887,7 +928,7 @@ export function startRecordCapture(
|
||||
emitStep({
|
||||
op: "press",
|
||||
key: event.key,
|
||||
...(desc ? { target: desc } : {}),
|
||||
...(desc ? { target: desc, geometry: geometryForEventTarget(target) } : {}),
|
||||
...(modifiers.length ? { modifiers } : {}),
|
||||
expects_navigation: event.key === "Enter",
|
||||
});
|
||||
@@ -906,9 +947,12 @@ export function startRecordCapture(
|
||||
const urlObserver = new MutationObserver(() => emitNavigateIfChanged());
|
||||
urlObserver.observe(document, { subtree: true, childList: true });
|
||||
const onUrlEvent = () => emitNavigateIfChanged();
|
||||
// Wrap: passing commitFillSession directly would forward the DOM event as
|
||||
// the `commit` argument and stamp it onto the recorded fill step.
|
||||
const onPageHide = () => commitFillSession();
|
||||
window.addEventListener("hashchange", onUrlEvent);
|
||||
window.addEventListener("popstate", onUrlEvent);
|
||||
window.addEventListener("pagehide", commitFillSession);
|
||||
window.addEventListener("pagehide", onPageHide);
|
||||
|
||||
const originalPushState = history.pushState;
|
||||
const originalReplaceState = history.replaceState;
|
||||
@@ -936,7 +980,7 @@ export function startRecordCapture(
|
||||
urlObserver.disconnect();
|
||||
window.removeEventListener("hashchange", onUrlEvent);
|
||||
window.removeEventListener("popstate", onUrlEvent);
|
||||
window.removeEventListener("pagehide", commitFillSession);
|
||||
window.removeEventListener("pagehide", onPageHide);
|
||||
history.pushState = originalPushState;
|
||||
history.replaceState = originalReplaceState;
|
||||
},
|
||||
@@ -965,6 +1009,7 @@ export function handleRecordContentMessage(
|
||||
if (!knownRecordRequests.has(message.requestId)) {
|
||||
knownRecordRequests.add(message.requestId);
|
||||
failedStepDeliveries.delete(message.requestId);
|
||||
failedStepPayloads.delete(message.requestId);
|
||||
}
|
||||
state.capture?.dispose();
|
||||
state.setCapture(
|
||||
@@ -992,27 +1037,51 @@ export function handleRecordContentMessage(
|
||||
state.setActiveRequestId(null);
|
||||
};
|
||||
if (isRecordStopMessage(message) && sendResponse) {
|
||||
const existingFlush = pendingStopFlushes.get(message.requestId);
|
||||
if (existingFlush) {
|
||||
void existingFlush.then(sendResponse);
|
||||
return true;
|
||||
}
|
||||
|
||||
const failedBeforeStop = [...(failedStepPayloads.get(message.requestId) ?? [])];
|
||||
if (failedBeforeStop.length > 0) {
|
||||
failedStepPayloads.delete(message.requestId);
|
||||
failedStepDeliveries.delete(message.requestId);
|
||||
}
|
||||
const pending = [...(pendingStepSends.get(message.requestId) ?? [])];
|
||||
void Promise.all(pending).then((delivered) => {
|
||||
finishStop();
|
||||
const succeeded = delivered.every(Boolean) && !failedStepDeliveries.has(message.requestId);
|
||||
const flush = Promise.all(pending).then(async (delivered): Promise<RecordStopAck> => {
|
||||
const retried = await Promise.all(
|
||||
failedBeforeStop.map((step) => deliverRecordStep(message.requestId, step)),
|
||||
);
|
||||
const succeeded =
|
||||
delivered.every(Boolean) &&
|
||||
retried.every(Boolean) &&
|
||||
!failedStepDeliveries.has(message.requestId);
|
||||
if (succeeded) {
|
||||
failedStepDeliveries.delete(message.requestId);
|
||||
failedStepPayloads.delete(message.requestId);
|
||||
knownRecordRequests.delete(message.requestId);
|
||||
finishStop();
|
||||
}
|
||||
return succeeded
|
||||
? { ok: true }
|
||||
: {
|
||||
ok: false,
|
||||
error: "failed to deliver one or more recorded steps",
|
||||
};
|
||||
});
|
||||
pendingStopFlushes.set(message.requestId, flush);
|
||||
void flush.then(sendResponse).finally(() => {
|
||||
if (pendingStopFlushes.get(message.requestId) === flush) {
|
||||
pendingStopFlushes.delete(message.requestId);
|
||||
}
|
||||
sendResponse(
|
||||
succeeded
|
||||
? { ok: true }
|
||||
: {
|
||||
ok: false,
|
||||
error: "failed to deliver one or more recorded steps",
|
||||
},
|
||||
);
|
||||
});
|
||||
return true;
|
||||
}
|
||||
if (isRecordCancelMessage(message)) {
|
||||
failedStepDeliveries.delete(message.requestId);
|
||||
failedStepPayloads.delete(message.requestId);
|
||||
pendingStopFlushes.delete(message.requestId);
|
||||
knownRecordRequests.delete(message.requestId);
|
||||
}
|
||||
finishStop();
|
||||
|
||||
@@ -172,6 +172,7 @@ export default defineBackground(() => {
|
||||
dispatcher.start();
|
||||
const recordDeps = {
|
||||
tabsApi: chrome.tabs,
|
||||
cdp,
|
||||
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
|
||||
chrome.tabs.sendMessage(tabId, msg),
|
||||
bypassOverlay: async (tabId: number, enabled: boolean) => {
|
||||
@@ -188,7 +189,7 @@ export default defineBackground(() => {
|
||||
// Message listeners stay up (cheap; fire only for record message types).
|
||||
// Tab / webNavigation observation attaches lazily while a recording is
|
||||
// active — see ensureBrowserObservationListeners in tools/record.ts.
|
||||
attachRecordStepListener();
|
||||
attachRecordStepListener(recordDeps);
|
||||
attachRecordFinishListener(recordDeps);
|
||||
attachRecordQueryListener(recordDeps);
|
||||
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
|
||||
|
||||
@@ -100,6 +100,24 @@ describe("App", () => {
|
||||
expect(screen.queryByText("录制你的操作,供 Agent 参考")).toBeNull();
|
||||
});
|
||||
|
||||
it("renders localized copy for a version failure instead of protocol details", () => {
|
||||
mockUseConnectionState.mockReturnValue({
|
||||
snapshot: {
|
||||
...baseSnapshot,
|
||||
lastError:
|
||||
"[handshake] daemon rejected handshake: version_too_old — peer protocol 1.0 is below local min_compatible_protocol 1.0",
|
||||
},
|
||||
statusState: "disconnected",
|
||||
setLabel,
|
||||
setConnectionEnabled,
|
||||
});
|
||||
|
||||
render(<App />);
|
||||
|
||||
const error = screen.getByText(/bsk 与 BrowserSkill 扩展版本不兼容/);
|
||||
expect(error.textContent).not.toContain("min_compatible_protocol");
|
||||
});
|
||||
|
||||
it("shows single-line compact metadata and copies the instance id", async () => {
|
||||
mockUseConnectionState.mockReturnValue({
|
||||
snapshot: {
|
||||
@@ -200,6 +218,13 @@ describe("App", () => {
|
||||
expect(copied).toContain("--browser 03c3e47f");
|
||||
expect(copied).toContain('--purpose "发布 wiki 文档"');
|
||||
expect(copied).not.toMatch(/bsk record start[^\n]*--url/);
|
||||
expect(copied).toContain("./trace");
|
||||
expect(copied).toContain("trace.json");
|
||||
expect(copied).toContain("pages/");
|
||||
expect(copied).toContain("总结我做了什么");
|
||||
expect(copied).not.toContain("不要重跑");
|
||||
expect(copied).not.toContain("pages + steps");
|
||||
expect(copied).not.toContain("@eN");
|
||||
// Button label stays static; a transient toast confirms the copy.
|
||||
expect(copyButton.textContent).toContain("复制录制指令");
|
||||
await waitFor(() => expect(screen.getByRole("status").textContent).toContain("已复制"));
|
||||
|
||||
@@ -83,6 +83,9 @@ export function App() {
|
||||
const daemonProtocol = snapshot.handshake?.protocol_version ?? "—";
|
||||
const extensionVersion = snapshot.extensionVersion || "—";
|
||||
const instanceId = snapshot.instanceId || "—";
|
||||
const errorMessage = snapshot.lastError?.includes("version_too_old")
|
||||
? t("popup.versionIncompatibleError")
|
||||
: snapshot.lastError;
|
||||
|
||||
const copyInstanceId = async () => {
|
||||
if (!snapshot.instanceId || !navigator.clipboard?.writeText) return;
|
||||
@@ -252,12 +255,12 @@ export function App() {
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{snapshot.lastError && (
|
||||
{errorMessage && (
|
||||
<div
|
||||
className="rounded-lg border border-destructive/25 bg-destructive/10 px-3 py-2 text-xs leading-snug text-destructive"
|
||||
data-slot="popup-error"
|
||||
>
|
||||
{snapshot.lastError}
|
||||
{errorMessage}
|
||||
</div>
|
||||
)}
|
||||
|
||||
|
||||
@@ -27,19 +27,19 @@ function handshake(
|
||||
|
||||
describe("computeConnectedState (protocol-based compat)", () => {
|
||||
it("returns connected when protocol strings match", () => {
|
||||
expect(computeConnectedState(handshake("1.0", "1.0"), MIN_COMPATIBLE_PROTOCOL)).toEqual({
|
||||
expect(computeConnectedState(handshake("1.1", "1.1"), MIN_COMPATIBLE_PROTOCOL)).toEqual({
|
||||
kind: "connected",
|
||||
});
|
||||
});
|
||||
|
||||
it("returns version_skew when daemon protocol minor is newer", () => {
|
||||
expect(computeConnectedState(handshake("1.1", "1.0"))).toEqual({
|
||||
expect(computeConnectedState(handshake("1.2", "1.1"))).toEqual({
|
||||
kind: "version_skew",
|
||||
});
|
||||
});
|
||||
|
||||
it("returns version_skew when daemon protocol string differs but floor is satisfied", () => {
|
||||
expect(computeConnectedState(handshake("1", "1.0"))).toEqual({
|
||||
expect(computeConnectedState(handshake("1.1.0", "1.1"))).toEqual({
|
||||
kind: "version_skew",
|
||||
});
|
||||
});
|
||||
@@ -48,16 +48,19 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
||||
const result = computeConnectedState(handshake("2.0", "1.0"));
|
||||
expect(result.kind).toBe("rejected");
|
||||
if (result.kind === "rejected") {
|
||||
expect(result.reason).toContain("protocol-major mismatch");
|
||||
expect(result.reason).toContain("bsk CLI");
|
||||
expect(result.reason).toContain("BrowserSkill extension");
|
||||
expect(result.reason).not.toContain("protocol");
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects when extension is below daemon min_compatible_protocol", () => {
|
||||
const result = computeConnectedState(handshake("1.0", "1.5"));
|
||||
const result = computeConnectedState(handshake("1.1", "1.5"));
|
||||
expect(result.kind).toBe("rejected");
|
||||
if (result.kind === "rejected") {
|
||||
expect(result.reason).toContain("min_compatible_protocol");
|
||||
expect(result.reason).toContain("1.5");
|
||||
expect(result.reason).toContain("BrowserSkill extension is out of date");
|
||||
expect(result.reason).toContain("Update and reload the extension");
|
||||
expect(result.reason).not.toContain("min_compatible_protocol");
|
||||
}
|
||||
});
|
||||
|
||||
@@ -65,7 +68,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
||||
const result = computeConnectedState({
|
||||
server: "browser-skill-daemon",
|
||||
version: "0.1.0",
|
||||
protocol_version: "1.0",
|
||||
protocol_version: "1.1",
|
||||
min_compatible_peer: "0.1.0",
|
||||
});
|
||||
expect(result).toEqual({ kind: "connected" });
|
||||
@@ -75,16 +78,24 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
||||
const result = computeConnectedState(handshake("1.0", "1.0"), "1.1");
|
||||
expect(result.kind).toBe("rejected");
|
||||
if (result.kind === "rejected") {
|
||||
expect(result.reason).toContain("below extension min_compatible_protocol");
|
||||
expect(result.reason).toContain("bsk CLI is out of date");
|
||||
expect(result.reason).toContain("bsk daemon restart");
|
||||
expect(result.reason).not.toContain("min_compatible_protocol");
|
||||
}
|
||||
});
|
||||
|
||||
it("returns version_skew when daemon protocol is 1.0 and floor is satisfied", () => {
|
||||
expect(computeConnectedState(handshake("1.0", "1.0"))).toEqual({
|
||||
kind: "version_skew",
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects malformed daemon min_compatible_protocol with a daemon-floor reason", () => {
|
||||
const result = computeConnectedState(handshake("1.0", "not-a-protocol"));
|
||||
const result = computeConnectedState(handshake("1.1", "not-a-protocol"));
|
||||
expect(result.kind).toBe("rejected");
|
||||
if (result.kind === "rejected") {
|
||||
expect(result.reason).toContain("daemon min_compatible_protocol");
|
||||
expect(result.reason).toContain("not-a-protocol");
|
||||
expect(result.reason).toContain("could not verify version compatibility");
|
||||
expect(result.reason).not.toContain("min_compatible_protocol");
|
||||
}
|
||||
});
|
||||
|
||||
@@ -242,11 +253,11 @@ describe("ConnectionController connectionEnabled", () => {
|
||||
const second = transport.send.mock.calls[1]?.[0] as { id: string };
|
||||
expect(second.id).not.toBe(first.id);
|
||||
|
||||
transport.emitMessage({ id: first.id, result: handshake("1.0", "1.0") });
|
||||
transport.emitMessage({ id: first.id, result: handshake("1.1", "1.1") });
|
||||
await Promise.resolve();
|
||||
expect(controller.snapshot().state).not.toBe("connected");
|
||||
|
||||
transport.emitMessage({ id: second.id, result: handshake("1.0", "1.0") });
|
||||
transport.emitMessage({ id: second.id, result: handshake("1.1", "1.1") });
|
||||
await vi.waitFor(() => expect(controller.snapshot().state).toBe("connected"));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
import type { RenderedRef } from "@browser-skill/vom";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { CapturedNode } from "@/tools/vom/capture";
|
||||
import { matchTarget } from "../match-target";
|
||||
|
||||
function node(backendNodeId: number, overrides: Partial<CapturedNode> = {}): CapturedNode {
|
||||
return {
|
||||
backendNodeId,
|
||||
parentBackendNodeId: null,
|
||||
tag: "button",
|
||||
attrs: {},
|
||||
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("matchTarget", () => {
|
||||
const refs: RenderedRef[] = [
|
||||
{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 3 },
|
||||
];
|
||||
|
||||
it("matches a unique node by viewport coordinates", () => {
|
||||
const target = matchTarget({
|
||||
geometry: {
|
||||
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
scrollX: 0,
|
||||
scrollY: 0,
|
||||
position: "static",
|
||||
tag: "button",
|
||||
},
|
||||
captured: [node(42)],
|
||||
refs,
|
||||
fallback: { tag: "button", name: "发布" },
|
||||
});
|
||||
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
|
||||
});
|
||||
|
||||
it("matches a non-fixed node after the top frame scrolls", () => {
|
||||
const target = matchTarget({
|
||||
geometry: {
|
||||
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
scrollX: 40,
|
||||
scrollY: 300,
|
||||
position: "static",
|
||||
tag: "button",
|
||||
},
|
||||
captured: [
|
||||
node(42, {
|
||||
documentRect: { x: 50, y: 320, w: 100, h: 30 },
|
||||
localRect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
}),
|
||||
],
|
||||
refs,
|
||||
fallback: { tag: "button", name: "发布" },
|
||||
});
|
||||
|
||||
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
|
||||
});
|
||||
|
||||
it("matches a static node by document coordinates when scroll changed after observation", () => {
|
||||
const target = matchTarget({
|
||||
geometry: {
|
||||
rect: { x: 10, y: 50, w: 100, h: 30 },
|
||||
scrollX: 0,
|
||||
scrollY: 250,
|
||||
position: "static",
|
||||
tag: "button",
|
||||
},
|
||||
captured: [
|
||||
node(42, {
|
||||
documentRect: { x: 10, y: 300, w: 100, h: 30 },
|
||||
localRect: { x: 10, y: 200, w: 100, h: 30 },
|
||||
rect: { x: 10, y: 200, w: 100, h: 30 },
|
||||
}),
|
||||
],
|
||||
refs,
|
||||
fallback: { tag: "button", name: "发布" },
|
||||
});
|
||||
|
||||
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
|
||||
});
|
||||
|
||||
it("returns unmatched when multiple candidates match", () => {
|
||||
const target = matchTarget({
|
||||
geometry: {
|
||||
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
scrollX: 0,
|
||||
scrollY: 0,
|
||||
position: "static",
|
||||
tag: "button",
|
||||
},
|
||||
captured: [node(42), node(43)],
|
||||
refs,
|
||||
});
|
||||
expect(target.unmatched).toBe(true);
|
||||
});
|
||||
|
||||
it("uses viewport coordinates for fixed elements", () => {
|
||||
const target = matchTarget({
|
||||
geometry: {
|
||||
rect: { x: 5, y: 5, w: 80, h: 24 },
|
||||
scrollX: 100,
|
||||
scrollY: 200,
|
||||
position: "fixed",
|
||||
tag: "button",
|
||||
},
|
||||
captured: [node(42, { position: "fixed", rect: { x: 5, y: 5, w: 80, h: 24 } })],
|
||||
refs,
|
||||
});
|
||||
expect(target.ref).toBe("e1");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,114 @@
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
import type { CapturedNode } from "@/tools/vom/capture";
|
||||
import type { DraftTraceStep } from "@/transport/types";
|
||||
import { resetStateIdCounterForTests } from "../record-constants";
|
||||
import {
|
||||
applyTargetMatching,
|
||||
buildTraceV3,
|
||||
createObservationState,
|
||||
rememberStepOnPage,
|
||||
} from "../record-observation";
|
||||
import { registerObservation } from "../trace-reducer";
|
||||
|
||||
const URL = "https://example.com/login";
|
||||
|
||||
function capturedInput(): CapturedNode {
|
||||
return {
|
||||
backendNodeId: 42,
|
||||
parentBackendNodeId: null,
|
||||
tag: "input",
|
||||
attrs: {},
|
||||
rect: { x: 20, y: 40, w: 200, h: 30 },
|
||||
localRect: { x: 20, y: 40, w: 200, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
};
|
||||
}
|
||||
|
||||
function finalizedFillBody(value: string, redactValues: boolean): string {
|
||||
const obs = createObservationState({ redactValues });
|
||||
const rawVomText = '@vom 1\ntextbox "Password" value="••••••" [ref=e1]';
|
||||
const stateId = registerObservation(obs.stateRegistry, { url: URL, rawVomText });
|
||||
obs.lastSettled = {
|
||||
stateId,
|
||||
captured: [capturedInput()],
|
||||
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
|
||||
url: URL,
|
||||
vomText: rawVomText,
|
||||
};
|
||||
|
||||
const draft: DraftTraceStep = {
|
||||
op: "fill",
|
||||
target: { unmatched: true },
|
||||
value,
|
||||
geometry: {
|
||||
rect: { x: 20, y: 40, w: 200, h: 30 },
|
||||
scrollX: 0,
|
||||
scrollY: 0,
|
||||
position: "static",
|
||||
tag: "input",
|
||||
},
|
||||
};
|
||||
applyTargetMatching(obs, draft, 1);
|
||||
draft.postStateId = stateId;
|
||||
rememberStepOnPage(obs, draft, 1);
|
||||
|
||||
return (
|
||||
buildTraceV3({
|
||||
obs,
|
||||
steps: [draft],
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
stoppedBy: "user_finish",
|
||||
bskVersion: "test",
|
||||
}).states[0]?.body ?? ""
|
||||
);
|
||||
}
|
||||
|
||||
describe("record observation annotations", () => {
|
||||
beforeEach(() => {
|
||||
resetStateIdCounterForTests();
|
||||
});
|
||||
|
||||
it("omits a fill literal from finalized state bodies when values are redacted", () => {
|
||||
const secret = "hunter2-private";
|
||||
const body = finalizedFillBody(secret, true);
|
||||
|
||||
expect(body).toContain("step 1: fill");
|
||||
expect(body).not.toContain(secret);
|
||||
});
|
||||
|
||||
it("keeps fill details in ordinary recording annotations", () => {
|
||||
const value = "ordinary text";
|
||||
const body = finalizedFillBody(value, false);
|
||||
|
||||
expect(body).toContain(`step 1: fill: ${JSON.stringify(value)}`);
|
||||
});
|
||||
});
|
||||
|
||||
describe("applyTargetMatching without a settled observation", () => {
|
||||
it("keeps the capture role/name instead of a bare unmatched target", () => {
|
||||
const obs = createObservationState();
|
||||
const draft: DraftTraceStep = {
|
||||
op: "hover",
|
||||
target: { unmatched: true },
|
||||
captureTarget: { tag: "button", role: "button", name: "新建" },
|
||||
geometry: {
|
||||
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||
scrollX: 0,
|
||||
scrollY: 0,
|
||||
position: "static",
|
||||
tag: "button",
|
||||
},
|
||||
};
|
||||
|
||||
applyTargetMatching(obs, draft, 1);
|
||||
|
||||
expect(draft.preStateId).toBeUndefined();
|
||||
expect(draft.target).toEqual({
|
||||
role: "button",
|
||||
name: "新建",
|
||||
unmatched: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -12,12 +12,42 @@ describe("recording-step-buffer", () => {
|
||||
expect(buffer.steps).toEqual([
|
||||
{
|
||||
op: "click",
|
||||
target: { tag: "button", role: "button", name: "发布" },
|
||||
target: { unmatched: true },
|
||||
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||
},
|
||||
]);
|
||||
expect(buffer.pendingNavigation).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps the hovered element description as a capture fallback", () => {
|
||||
const buffer = { steps: [], pendingNavigation: false };
|
||||
appendRecordedPayload(buffer, {
|
||||
op: "hover",
|
||||
target: { tag: "button", role: "button", name: "新建" },
|
||||
geometry: {
|
||||
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||
scrollX: 0,
|
||||
scrollY: 0,
|
||||
position: "static",
|
||||
tag: "button",
|
||||
},
|
||||
});
|
||||
expect(buffer.steps).toEqual([
|
||||
{
|
||||
op: "hover",
|
||||
target: { unmatched: true },
|
||||
captureTarget: { tag: "button", role: "button", name: "新建" },
|
||||
geometry: {
|
||||
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||
scrollX: 0,
|
||||
scrollY: 0,
|
||||
position: "static",
|
||||
tag: "button",
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("annotates navigated_to on action-caused navigation instead of wait_for_navigation", () => {
|
||||
const buffer = {
|
||||
steps: [
|
||||
@@ -71,10 +101,41 @@ describe("recording-step-buffer", () => {
|
||||
currentUrl: "https://example.com/a",
|
||||
pendingNavigation: false,
|
||||
};
|
||||
observeRecordedNavigation(buffer, "https://example.com/b", false);
|
||||
const result = observeRecordedNavigation(buffer, "https://example.com/b", false);
|
||||
expect(result).toEqual({ kind: "appended", index: 0 });
|
||||
expect(buffer.steps[0]).toMatchObject({
|
||||
op: "navigate",
|
||||
url: "https://example.com/b",
|
||||
});
|
||||
});
|
||||
|
||||
it("asks the recorder to coalesce redirect hops instead of emitting each one", () => {
|
||||
const buffer = {
|
||||
steps: [],
|
||||
currentUrl: "https://passport.example/login",
|
||||
pendingNavigation: false,
|
||||
};
|
||||
const hop1 = observeRecordedNavigation(
|
||||
buffer,
|
||||
"https://passport.example/callback",
|
||||
false,
|
||||
"link",
|
||||
["server_redirect"],
|
||||
);
|
||||
expect(hop1).toEqual({
|
||||
kind: "coalesce_redirect",
|
||||
url: "https://passport.example/callback",
|
||||
});
|
||||
expect(buffer.steps).toEqual([]);
|
||||
|
||||
const hop2 = observeRecordedNavigation(buffer, "https://app.example/dashboard", false, "link", [
|
||||
"client_redirect",
|
||||
]);
|
||||
expect(hop2).toEqual({
|
||||
kind: "coalesce_redirect",
|
||||
url: "https://app.example/dashboard",
|
||||
});
|
||||
expect(buffer.steps).toEqual([]);
|
||||
expect(buffer.currentUrl).toBe("https://app.example/dashboard");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildTraceV2, shouldRecordPress } from "@/lib/trace-reducer-v2";
|
||||
import type { DraftTraceStep } from "@/transport/types";
|
||||
|
||||
describe("trace-reducer-v2", () => {
|
||||
it("drops scroll and bare character press steps", () => {
|
||||
const steps: DraftTraceStep[] = [
|
||||
{
|
||||
op: "scroll",
|
||||
page_url: "https://example.com/",
|
||||
},
|
||||
{
|
||||
op: "press",
|
||||
key: "a",
|
||||
page_url: "https://example.com/",
|
||||
},
|
||||
];
|
||||
const trace = buildTraceV2({
|
||||
steps,
|
||||
startedAt: "2026-01-01T00:00:00.000Z",
|
||||
startUrl: "https://example.com/",
|
||||
});
|
||||
expect(trace.steps).toHaveLength(0);
|
||||
expect(trace.pages).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("drops hover steps unsupported by historical v2 clients", () => {
|
||||
const trace = buildTraceV2({
|
||||
steps: [
|
||||
{
|
||||
op: "hover",
|
||||
target: { unmatched: true },
|
||||
captureTarget: { tag: "button", role: "button", name: "结束" },
|
||||
page_url: "https://example.com/",
|
||||
},
|
||||
],
|
||||
startedAt: "2026-01-01T00:00:00.000Z",
|
||||
startUrl: "https://example.com/",
|
||||
});
|
||||
|
||||
expect(trace.steps).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("maps captureTarget to v2 target with required tag", () => {
|
||||
const steps: DraftTraceStep[] = [
|
||||
{
|
||||
op: "click",
|
||||
target: { unmatched: true },
|
||||
captureTarget: { tag: "button", role: "button", name: "Submit" },
|
||||
page_url: "https://example.com/",
|
||||
},
|
||||
];
|
||||
const trace = buildTraceV2({
|
||||
steps,
|
||||
startedAt: "2026-01-01T00:00:00.000Z",
|
||||
startUrl: "https://example.com/",
|
||||
});
|
||||
expect(trace.steps).toHaveLength(1);
|
||||
expect(trace.steps[0]?.op).toBe("click");
|
||||
if (trace.steps[0]?.op === "click") {
|
||||
expect(trace.steps[0].target.tag).toBe("button");
|
||||
expect(trace.steps[0].target.name).toBe("Submit");
|
||||
}
|
||||
expect("version" in trace).toBe(false);
|
||||
});
|
||||
|
||||
it("records Enter presses", () => {
|
||||
expect(shouldRecordPress("Enter")).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,11 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { DraftTraceStep } from "@/transport/types";
|
||||
import { reduceTraceSteps, resolveTraceStartUrl, shouldRecordPress } from "../trace-reducer";
|
||||
import { resetStateIdCounterForTests } from "@/lib/record-constants";
|
||||
import {
|
||||
reduceTraceSteps,
|
||||
registerObservation,
|
||||
resolveTraceStartUrl,
|
||||
shouldRecordPress,
|
||||
} from "../trace-reducer";
|
||||
|
||||
describe("shouldRecordPress", () => {
|
||||
it("keeps Enter and Escape", () => {
|
||||
@@ -17,86 +22,85 @@ describe("shouldRecordPress", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("reduceTraceSteps", () => {
|
||||
it("builds steps with pages dictionary and page id references", () => {
|
||||
const drafts: DraftTraceStep[] = [
|
||||
{
|
||||
op: "navigate",
|
||||
url: "https://example.com/search?q=hello&utm_source=x",
|
||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
||||
},
|
||||
{
|
||||
op: "fill",
|
||||
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
||||
value: "browser skill",
|
||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
||||
},
|
||||
{
|
||||
op: "press",
|
||||
key: "Enter",
|
||||
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
||||
navigated_to: "https://example.com/results/42",
|
||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
||||
},
|
||||
{
|
||||
op: "click",
|
||||
target: { tag: "button", role: "button", name: "发布" },
|
||||
navigated_to: "https://example.com/p/99",
|
||||
page_url: "https://example.com/results/42",
|
||||
},
|
||||
{
|
||||
op: "press",
|
||||
key: "a",
|
||||
page_url: "https://example.com/p/99",
|
||||
},
|
||||
];
|
||||
|
||||
const { pages, steps } = reduceTraceSteps(
|
||||
drafts,
|
||||
"https://example.com/search?q=hello&utm_source=x",
|
||||
);
|
||||
expect(JSON.stringify(steps)).not.toContain("parameters");
|
||||
expect(JSON.stringify(steps)).not.toContain("intent");
|
||||
expect(JSON.stringify(steps)).not.toContain("summary");
|
||||
expect(steps.map((s) => s.op)).toEqual(["navigate", "fill", "press", "click"]);
|
||||
expect(pages.map((p) => p.url)).toEqual([
|
||||
"https://example.com/search?q=hello&utm_source=x",
|
||||
"https://example.com/results/42",
|
||||
"https://example.com/p/99",
|
||||
]);
|
||||
|
||||
expect(steps[0]).toMatchObject({
|
||||
id: 1,
|
||||
op: "navigate",
|
||||
page: "p1",
|
||||
to: "https://example.com/search?q=hello&utm_source=x",
|
||||
describe("reduceTraceSteps v3", () => {
|
||||
it("builds steps with state/result references and deduped states", () => {
|
||||
resetStateIdCounterForTests();
|
||||
const registry = new Map();
|
||||
const body = "@vom 1\nL1 page";
|
||||
const s1 = registerObservation(registry, {
|
||||
url: "https://example.com/search",
|
||||
rawVomText: body,
|
||||
});
|
||||
const s2 = registerObservation(registry, {
|
||||
url: "https://example.com/results/42",
|
||||
rawVomText: "@vom 1\nL1 results",
|
||||
});
|
||||
|
||||
const { states, steps } = reduceTraceSteps(
|
||||
[
|
||||
{
|
||||
op: "navigate",
|
||||
url: "https://example.com/search",
|
||||
preStateId: s1,
|
||||
postStateId: s1,
|
||||
},
|
||||
{
|
||||
op: "fill",
|
||||
target: { ref: "e1", role: "textbox", name: "搜索" },
|
||||
value: "browser skill",
|
||||
commit: "enter",
|
||||
preStateId: s1,
|
||||
postStateId: s1,
|
||||
},
|
||||
{
|
||||
op: "press",
|
||||
key: "Enter",
|
||||
target: { ref: "e1", role: "textbox", name: "搜索" },
|
||||
preStateId: s1,
|
||||
postStateId: s2,
|
||||
navigated_to: "https://example.com/results/42",
|
||||
},
|
||||
{
|
||||
op: "click",
|
||||
target: { ref: "e2", role: "button", name: "发布" },
|
||||
preStateId: s2,
|
||||
postStateId: s2,
|
||||
},
|
||||
],
|
||||
registry,
|
||||
);
|
||||
|
||||
expect(states).toHaveLength(2);
|
||||
expect(steps.map((s) => s.op)).toEqual(["navigate", "fill", "press", "click"]);
|
||||
expect(steps[1]).toMatchObject({
|
||||
op: "fill",
|
||||
page: "p1",
|
||||
value: "browser skill",
|
||||
state: s1,
|
||||
result: { state: s1 },
|
||||
commit: "enter",
|
||||
});
|
||||
|
||||
expect(steps[2]).toMatchObject({
|
||||
op: "press",
|
||||
key: "Enter",
|
||||
page: "p1",
|
||||
effect: { navigated_to: "p2" },
|
||||
});
|
||||
|
||||
expect(steps[3]).toMatchObject({
|
||||
op: "click",
|
||||
page: "p2",
|
||||
effect: { navigated_to: "p3" },
|
||||
state: s1,
|
||||
result: { state: s2 },
|
||||
});
|
||||
expect(JSON.stringify(steps)).not.toContain("page");
|
||||
expect(JSON.stringify(steps)).not.toContain("effect");
|
||||
});
|
||||
|
||||
it("collapses consecutive navigations", () => {
|
||||
const { steps } = reduceTraceSteps([
|
||||
{ op: "navigate", url: "https://a.example/redirect1" },
|
||||
{ op: "navigate", url: "https://a.example/final" },
|
||||
]);
|
||||
resetStateIdCounterForTests();
|
||||
const registry = new Map();
|
||||
const s1 = registerObservation(registry, {
|
||||
url: "https://a.example/redirect1",
|
||||
rawVomText: "a",
|
||||
});
|
||||
const { steps } = reduceTraceSteps(
|
||||
[
|
||||
{ op: "navigate", url: "https://a.example/redirect1", preStateId: s1, postStateId: s1 },
|
||||
{ op: "navigate", url: "https://a.example/final", preStateId: s1, postStateId: s1 },
|
||||
],
|
||||
registry,
|
||||
);
|
||||
expect(steps).toHaveLength(1);
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "navigate",
|
||||
@@ -104,52 +108,135 @@ describe("reduceTraceSteps", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("maps select navigated_to onto effect.navigated_to (page id)", () => {
|
||||
const { pages, steps } = reduceTraceSteps(
|
||||
it("keeps the origin state and cause of the hop that started a redirect chain", () => {
|
||||
resetStateIdCounterForTests();
|
||||
const registry = new Map();
|
||||
const start = registerObservation(registry, {
|
||||
url: "https://example.com/",
|
||||
rawVomText: "start",
|
||||
});
|
||||
const hop = registerObservation(registry, {
|
||||
url: "https://iwiki.woa.com/",
|
||||
rawVomText: "hop",
|
||||
});
|
||||
const landing = registerObservation(registry, {
|
||||
url: "https://iwiki.woa.com/dashboard",
|
||||
rawVomText: "landing",
|
||||
});
|
||||
|
||||
const { steps, stepIdByDraftId } = reduceTraceSteps(
|
||||
[
|
||||
{
|
||||
op: "select",
|
||||
target: { tag: "select", role: "combobox", name: "分类" },
|
||||
values: ["tech"],
|
||||
labels: ["技术"],
|
||||
navigated_to: "https://example.com/list?cat=tech",
|
||||
page_url: "https://example.com/list",
|
||||
op: "navigate",
|
||||
url: "https://iwiki.woa.com/",
|
||||
preStateId: start,
|
||||
postStateId: hop,
|
||||
transitionType: "typed",
|
||||
},
|
||||
{
|
||||
op: "navigate",
|
||||
url: "https://iwiki.woa.com/dashboard",
|
||||
preStateId: hop,
|
||||
postStateId: landing,
|
||||
transitionType: "link",
|
||||
},
|
||||
],
|
||||
"https://example.com/list",
|
||||
registry,
|
||||
);
|
||||
expect(pages.map((p) => p.url)).toEqual([
|
||||
"https://example.com/list",
|
||||
"https://example.com/list?cat=tech",
|
||||
]);
|
||||
|
||||
expect(steps).toHaveLength(1);
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "select",
|
||||
page: "p1",
|
||||
selection: [{ value: "tech", label: "技术" }],
|
||||
effect: { navigated_to: "p2" },
|
||||
op: "navigate",
|
||||
id: 1,
|
||||
state: start,
|
||||
result: { state: landing },
|
||||
to: "https://iwiki.woa.com/dashboard",
|
||||
cause: "user_typed",
|
||||
});
|
||||
// Both drafts fold into step 1, so anything tagged with either draft id
|
||||
// must resolve to it.
|
||||
expect(stepIdByDraftId.get(1)).toBe(1);
|
||||
expect(stepIdByDraftId.get(2)).toBe(1);
|
||||
});
|
||||
|
||||
it("maps draft ids to published step ids across dropped drafts", () => {
|
||||
resetStateIdCounterForTests();
|
||||
const registry = new Map();
|
||||
const s1 = registerObservation(registry, { url: "https://a.example/", rawVomText: "a" });
|
||||
|
||||
const { steps, stepIdByDraftId } = reduceTraceSteps(
|
||||
[
|
||||
// Dropped: bare typing press.
|
||||
{ op: "press", key: "a", preStateId: s1, postStateId: s1 },
|
||||
{ op: "click", target: { ref: "e2" }, preStateId: s1, postStateId: s1 },
|
||||
],
|
||||
registry,
|
||||
);
|
||||
|
||||
expect(steps.map((step) => step.op)).toEqual(["click"]);
|
||||
expect(stepIdByDraftId.get(1)).toBeUndefined();
|
||||
expect(stepIdByDraftId.get(2)).toBe(1);
|
||||
});
|
||||
|
||||
it("keeps committed empty fills", () => {
|
||||
resetStateIdCounterForTests();
|
||||
const registry = new Map();
|
||||
const s1 = registerObservation(registry, { url: "https://a.example/", rawVomText: "a" });
|
||||
|
||||
const { steps } = reduceTraceSteps(
|
||||
[
|
||||
{
|
||||
op: "fill",
|
||||
target: { ref: "e1", role: "textbox", name: "Search" },
|
||||
value: "",
|
||||
preStateId: s1,
|
||||
postStateId: s1,
|
||||
},
|
||||
{
|
||||
op: "click",
|
||||
target: { ref: "e2", role: "button", name: "Apply" },
|
||||
preStateId: s1,
|
||||
postStateId: s1,
|
||||
},
|
||||
],
|
||||
registry,
|
||||
);
|
||||
|
||||
expect(steps.map((step) => step.op)).toEqual(["fill", "click"]);
|
||||
expect(steps[0]).toMatchObject({ op: "fill", value: "" });
|
||||
});
|
||||
|
||||
it("keeps hover steps before menu clicks", () => {
|
||||
resetStateIdCounterForTests();
|
||||
const registry = new Map();
|
||||
const state = registerObservation(registry, {
|
||||
url: "https://example.com/app",
|
||||
rawVomText: '@e1 button "Account"\n@e2 link "Profile"',
|
||||
});
|
||||
const { steps } = reduceTraceSteps(
|
||||
[
|
||||
{
|
||||
op: "hover",
|
||||
target: { tag: "span", role: "button", name: "Account" },
|
||||
target: { ref: "e1", role: "button", name: "Account" },
|
||||
page_url: "https://example.com/app",
|
||||
preStateId: state,
|
||||
postStateId: state,
|
||||
},
|
||||
{
|
||||
op: "click",
|
||||
target: { tag: "a", role: "link", name: "Profile" },
|
||||
target: { ref: "e2", role: "link", name: "Profile" },
|
||||
page_url: "https://example.com/app",
|
||||
preStateId: state,
|
||||
postStateId: state,
|
||||
},
|
||||
],
|
||||
"https://example.com/app",
|
||||
registry,
|
||||
);
|
||||
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "hover",
|
||||
page: "p1",
|
||||
state,
|
||||
result: { state },
|
||||
target: { name: "Account" },
|
||||
});
|
||||
});
|
||||
@@ -157,7 +244,7 @@ describe("reduceTraceSteps", () => {
|
||||
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
||||
expect(
|
||||
resolveTraceStartUrl(
|
||||
[{ op: "navigate", url: "https://example.com/other" }],
|
||||
[{ op: "navigate", url: "https://example.com/other", preStateId: "s1", postStateId: "s1" }],
|
||||
"https://example.com/start",
|
||||
),
|
||||
).toBe("https://example.com/start");
|
||||
|
||||
@@ -313,6 +313,15 @@ export type HandshakeVerdict =
|
||||
| { kind: "version_skew" }
|
||||
| { kind: "rejected"; reason: string };
|
||||
|
||||
const UPDATE_BOTH_VERSION_MESSAGE =
|
||||
"The installed bsk CLI and BrowserSkill extension are incompatible. Update both, reload the extension, then run `bsk daemon restart`.";
|
||||
const UPDATE_BSK_VERSION_MESSAGE =
|
||||
"The installed bsk CLI is out of date for this BrowserSkill extension. Update bsk, then run `bsk daemon restart`.";
|
||||
const UPDATE_EXTENSION_VERSION_MESSAGE =
|
||||
"This BrowserSkill extension is out of date for the installed bsk CLI. Update and reload the extension, then try again.";
|
||||
const INVALID_VERSION_MESSAGE =
|
||||
"BrowserSkill could not verify version compatibility. Update bsk and the BrowserSkill extension, reload the extension, then run `bsk daemon restart`.";
|
||||
|
||||
/**
|
||||
* Symmetric extension-side **protocol** compat check.
|
||||
*
|
||||
@@ -339,19 +348,19 @@ export function computeConnectedState(
|
||||
if (daemonMajor === null || ourMajor === null || daemonMajor !== ourMajor) {
|
||||
return {
|
||||
kind: "rejected",
|
||||
reason: `protocol-major mismatch (daemon protocol v${handshake.protocol_version}, extension protocol v${PROTOCOL_VERSION})`,
|
||||
reason: UPDATE_BOTH_VERSION_MESSAGE,
|
||||
};
|
||||
}
|
||||
if (compareProtocol(handshake.protocol_version, localMinCompatibleProtocol) === null) {
|
||||
return {
|
||||
kind: "rejected",
|
||||
reason: `daemon protocol v${handshake.protocol_version} is unparseable`,
|
||||
reason: INVALID_VERSION_MESSAGE,
|
||||
};
|
||||
}
|
||||
if (compareProtocol(handshake.protocol_version, localMinCompatibleProtocol)! < 0) {
|
||||
return {
|
||||
kind: "rejected",
|
||||
reason: `daemon protocol v${handshake.protocol_version} below extension min_compatible_protocol ${localMinCompatibleProtocol}`,
|
||||
reason: UPDATE_BSK_VERSION_MESSAGE,
|
||||
};
|
||||
}
|
||||
const peerFloor = handshake.min_compatible_protocol;
|
||||
@@ -359,13 +368,13 @@ export function computeConnectedState(
|
||||
if (compareProtocol(PROTOCOL_VERSION, peerFloor) === null) {
|
||||
return {
|
||||
kind: "rejected",
|
||||
reason: `daemon min_compatible_protocol ${peerFloor} is unparseable`,
|
||||
reason: INVALID_VERSION_MESSAGE,
|
||||
};
|
||||
}
|
||||
if (compareProtocol(PROTOCOL_VERSION, peerFloor)! < 0) {
|
||||
return {
|
||||
kind: "rejected",
|
||||
reason: `extension protocol v${PROTOCOL_VERSION} below daemon min_compatible_protocol ${peerFloor}`,
|
||||
reason: UPDATE_EXTENSION_VERSION_MESSAGE,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,8 @@
|
||||
* `{ "tag": "div" }` / “点击div” is useless and must not be recorded.
|
||||
*/
|
||||
|
||||
export interface TargetDescriptor {
|
||||
/** Content-script capture descriptor before VOM geometric matching. */
|
||||
export interface CaptureTargetDescriptor {
|
||||
role?: string;
|
||||
name?: string;
|
||||
tag: string;
|
||||
@@ -247,7 +248,7 @@ export function resolveHoverElement(target: Element): Element | null {
|
||||
* (visible name), or at least a form `name_attr` for checkbox/radio.
|
||||
* Recording “点击div” with no name fails this bar.
|
||||
*/
|
||||
export function isMeaningfulClickTarget(target: TargetDescriptor): boolean {
|
||||
export function isMeaningfulClickTarget(target: CaptureTargetDescriptor): boolean {
|
||||
const name = target.name?.trim();
|
||||
if (name && isActionableLabel(name)) return true;
|
||||
if (
|
||||
@@ -323,7 +324,7 @@ function nearbyLabelText(el: Element): string | undefined {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function describeTarget(el: Element): TargetDescriptor {
|
||||
export function describeTarget(el: Element): CaptureTargetDescriptor {
|
||||
const tag = el.tagName.toLowerCase();
|
||||
const role = inferRole(el);
|
||||
const name = accessibleName(el);
|
||||
@@ -350,7 +351,7 @@ export function describeTarget(el: Element): TargetDescriptor {
|
||||
};
|
||||
}
|
||||
|
||||
export function describeEventTarget(target: EventTarget | null): TargetDescriptor | null {
|
||||
export function describeEventTarget(target: EventTarget | null): CaptureTargetDescriptor | null {
|
||||
if (!(target instanceof Element)) return null;
|
||||
const clickable = resolveClickableElement(target);
|
||||
if (!clickable) return null;
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
import type { Step } from "@/transport/types";
|
||||
import { OBSERVATION_FILE_VERSION } from "./record-constants";
|
||||
|
||||
export interface ObservationAnnotation {
|
||||
stepId: number;
|
||||
op: Step["op"];
|
||||
line: number;
|
||||
stateId: string;
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
function formatAnnotation({ stepId, op, detail }: ObservationAnnotation): string {
|
||||
const suffix = detail ? `: ${detail}` : "";
|
||||
return ` ⟵ step ${stepId}: ${op}${suffix}`;
|
||||
}
|
||||
|
||||
/** Serialize a page observation file (front matter + VOM body + step annotations). */
|
||||
export function formatObservationFile(input: {
|
||||
stateId: string;
|
||||
url: string;
|
||||
title?: string;
|
||||
stepsHere: number[];
|
||||
body: string;
|
||||
annotations?: ObservationAnnotation[];
|
||||
}): string {
|
||||
const lines: string[] = [
|
||||
`# bsk-observation ${OBSERVATION_FILE_VERSION}`,
|
||||
`state: ${input.stateId}`,
|
||||
`url: ${input.url}`,
|
||||
];
|
||||
if (input.title) lines.push(`title: ${input.title}`);
|
||||
if (input.stepsHere.length > 0) {
|
||||
lines.push(`steps_here: [${input.stepsHere.join(", ")}]`);
|
||||
}
|
||||
lines.push("---");
|
||||
|
||||
const bodyLines = input.body.split("\n");
|
||||
const annotationMap = new Map<number, ObservationAnnotation[]>();
|
||||
for (const ann of input.annotations ?? []) {
|
||||
const bucket = annotationMap.get(ann.line) ?? [];
|
||||
bucket.push(ann);
|
||||
annotationMap.set(ann.line, bucket);
|
||||
}
|
||||
|
||||
for (let i = 0; i < bodyLines.length; i += 1) {
|
||||
let line = bodyLines[i] ?? "";
|
||||
const anns = annotationMap.get(i);
|
||||
if (anns) {
|
||||
for (const ann of anns) {
|
||||
line += formatAnnotation(ann);
|
||||
}
|
||||
}
|
||||
lines.push(line);
|
||||
}
|
||||
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
/** Hash input: VOM body **before** annotations are inserted. */
|
||||
export function observationBodyForHash(vomText: string): string {
|
||||
return vomText;
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
import type { RenderedRef } from "@browser-skill/vom";
|
||||
import type { CapturedNode } from "@/tools/vom/capture";
|
||||
import type { CaptureGeometry, TargetDescriptor } from "@/transport/types";
|
||||
import type { CaptureTargetDescriptor } from "./describe-target";
|
||||
import { GEOM_MATCH_TOLERANCE_PX } from "./record-constants";
|
||||
|
||||
export interface MatchTargetInput {
|
||||
geometry: CaptureGeometry;
|
||||
captured: CapturedNode[];
|
||||
refs: RenderedRef[];
|
||||
fallback?: CaptureTargetDescriptor;
|
||||
}
|
||||
|
||||
function withinTolerance(a: number, b: number): boolean {
|
||||
return Math.abs(a - b) <= GEOM_MATCH_TOLERANCE_PX;
|
||||
}
|
||||
|
||||
function rectsMatch(
|
||||
a: { x: number; y: number; w: number; h: number },
|
||||
b: { x: number; y: number; w: number; h: number },
|
||||
): boolean {
|
||||
return (
|
||||
withinTolerance(a.x, b.x) &&
|
||||
withinTolerance(a.y, b.y) &&
|
||||
withinTolerance(a.w, b.w) &&
|
||||
withinTolerance(a.h, b.h)
|
||||
);
|
||||
}
|
||||
|
||||
function nodeViewportRect(
|
||||
node: CapturedNode,
|
||||
): { x: number; y: number; w: number; h: number } | null {
|
||||
return node.localRect ?? node.rect;
|
||||
}
|
||||
|
||||
function matchingRects(
|
||||
geometry: CaptureGeometry,
|
||||
node: CapturedNode,
|
||||
): {
|
||||
target: { x: number; y: number; w: number; h: number };
|
||||
node: { x: number; y: number; w: number; h: number } | null;
|
||||
} {
|
||||
const topFrame = (geometry.ownerFrameBackendNodeId ?? null) === null;
|
||||
const viewportPosition = geometry.position === "fixed" || geometry.position === "sticky";
|
||||
if (!topFrame || viewportPosition || !node.documentRect) {
|
||||
return { target: geometry.rect, node: nodeViewportRect(node) };
|
||||
}
|
||||
return {
|
||||
target: {
|
||||
x: geometry.rect.x + geometry.scrollX,
|
||||
y: geometry.rect.y + geometry.scrollY,
|
||||
w: geometry.rect.w,
|
||||
h: geometry.rect.h,
|
||||
},
|
||||
node: node.documentRect,
|
||||
};
|
||||
}
|
||||
|
||||
function tagMatches(geometryTag: string, nodeTag: string): boolean {
|
||||
return geometryTag.toLowerCase() === nodeTag.toLowerCase();
|
||||
}
|
||||
|
||||
/** Best description available when the element cannot be located in the VOM. */
|
||||
export function fallbackDescriptor(fallback?: CaptureTargetDescriptor): TargetDescriptor {
|
||||
if (!fallback) {
|
||||
return { unmatched: true };
|
||||
}
|
||||
return {
|
||||
...(fallback.role ? { role: fallback.role } : {}),
|
||||
...(fallback.name ? { name: fallback.name } : {}),
|
||||
unmatched: true,
|
||||
};
|
||||
}
|
||||
|
||||
/** Locate the interacted element in the last settled observation by geometry. */
|
||||
export function matchTarget(input: MatchTargetInput): TargetDescriptor {
|
||||
const ownerFrame = input.geometry.ownerFrameBackendNodeId ?? null;
|
||||
|
||||
const candidates = input.captured.filter((node) => {
|
||||
if (ownerFrame !== (node.ownerFrameBackendNodeId ?? null)) return false;
|
||||
if (!tagMatches(input.geometry.tag, node.tag)) return false;
|
||||
const rects = matchingRects(input.geometry, node);
|
||||
if (!rects.node) return false;
|
||||
return rectsMatch(rects.target, rects.node);
|
||||
});
|
||||
|
||||
if (candidates.length !== 1) {
|
||||
return fallbackDescriptor(input.fallback);
|
||||
}
|
||||
|
||||
const backendNodeId = candidates[0]!.backendNodeId;
|
||||
const refEntry = input.refs.find((r) => r.backendNodeId === backendNodeId);
|
||||
if (!refEntry) {
|
||||
return fallbackDescriptor(input.fallback);
|
||||
}
|
||||
|
||||
return {
|
||||
ref: refEntry.ref,
|
||||
...(refEntry.role ? { role: refEntry.role } : {}),
|
||||
...(refEntry.name ? { name: refEntry.name } : {}),
|
||||
...(refEntry.ctx ? { ctx: refEntry.ctx } : {}),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
// Deciding *when* a page is worth observing.
|
||||
//
|
||||
// A fixed delay cannot serve both a modal that toggles in one frame and a
|
||||
// route change that renders for a second. Instead of guessing, ask the page:
|
||||
// a document that has stopped mutating and is no longer loading is done
|
||||
// reacting to whatever the user just did.
|
||||
|
||||
import type { CdpRunner } from "@/tools/shared";
|
||||
import { SETTLE_MAX_MS, SETTLE_MIN_MS, SETTLE_POLL_MS, SETTLE_QUIET_MS } from "./record-constants";
|
||||
|
||||
/**
|
||||
* Installed once per document. Records only the timestamp of the last DOM
|
||||
* change so each poll stays O(1) no matter how large the page is.
|
||||
*/
|
||||
const QUIET_PROBE = `(() => {
|
||||
const scope = window;
|
||||
let probe = scope.__bskRecordQuiet;
|
||||
if (!probe) {
|
||||
probe = { changedAt: Date.now() };
|
||||
const observer = new MutationObserver(() => {
|
||||
probe.changedAt = Date.now();
|
||||
});
|
||||
observer.observe(document.documentElement, {
|
||||
subtree: true,
|
||||
childList: true,
|
||||
attributes: true,
|
||||
characterData: true,
|
||||
});
|
||||
scope.__bskRecordQuiet = probe;
|
||||
}
|
||||
return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState };
|
||||
})()`;
|
||||
|
||||
export type SettleOutcome =
|
||||
/** The page stopped changing on its own. */
|
||||
| "quiet"
|
||||
/** Still changing when the budget ran out; observe it as it is. */
|
||||
| "timeout"
|
||||
/** A newer action took over; this observation is no longer wanted. */
|
||||
| "cancelled";
|
||||
|
||||
interface QuietProbe {
|
||||
idleMs: number;
|
||||
readyState: string;
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function readQuietProbe(cdp: CdpRunner, tabId: number): Promise<QuietProbe | null> {
|
||||
try {
|
||||
const reply = await cdp.send<{ result?: { value?: unknown } }>(tabId, "Runtime.evaluate", {
|
||||
expression: QUIET_PROBE,
|
||||
returnByValue: true,
|
||||
});
|
||||
const value = reply.result?.value;
|
||||
if (!value || typeof value !== "object") return null;
|
||||
const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown };
|
||||
if (typeof idleMs !== "number" || typeof readyState !== "string") return null;
|
||||
return { idleMs, readyState };
|
||||
} catch {
|
||||
// An unreadable page is a page mid-swap; that is a reason to keep waiting,
|
||||
// not a reason to give up.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Wait until the page has finished reacting, or until the budget runs out. */
|
||||
export async function waitForPageSettled(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
options: { cancelled?: () => boolean } = {},
|
||||
): Promise<SettleOutcome> {
|
||||
const startedAt = Date.now();
|
||||
const floor = startedAt + SETTLE_MIN_MS;
|
||||
const deadline = startedAt + SETTLE_MAX_MS;
|
||||
|
||||
for (;;) {
|
||||
if (options.cancelled?.()) return "cancelled";
|
||||
await sleep(SETTLE_POLL_MS);
|
||||
if (options.cancelled?.()) return "cancelled";
|
||||
|
||||
const probe = await readQuietProbe(cdp, tabId);
|
||||
const now = Date.now();
|
||||
if (now < floor) continue;
|
||||
if (now >= deadline) return "timeout";
|
||||
if (!probe) continue;
|
||||
if (probe.readyState === "loading") continue;
|
||||
if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet";
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,7 @@
|
||||
* service worker and a tab's content script.
|
||||
*/
|
||||
|
||||
import type { TargetDescriptor } from "./describe-target";
|
||||
import type { CaptureTargetDescriptor } from "./describe-target";
|
||||
|
||||
export const RECORD_START = "bsk-record-start";
|
||||
export const RECORD_STEP = "bsk-record-step";
|
||||
@@ -41,8 +41,8 @@ export interface RecordStartMessage {
|
||||
}
|
||||
|
||||
export interface RecordStepPayload {
|
||||
op: "click" | "hover" | "fill" | "press" | "select" | "navigate";
|
||||
target?: TargetDescriptor;
|
||||
op: "click" | "hover" | "fill" | "press" | "select" | "navigate" | "scroll";
|
||||
target?: CaptureTargetDescriptor;
|
||||
value?: string;
|
||||
key?: string;
|
||||
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">;
|
||||
@@ -50,12 +50,25 @@ export interface RecordStepPayload {
|
||||
labels?: string[];
|
||||
url?: string;
|
||||
redacted?: boolean;
|
||||
commit?: "enter" | "suggestion" | "blur";
|
||||
/** Page URL when the step was captured. */
|
||||
page_url?: string;
|
||||
/** Document geometry at capture time for background geometric matching. */
|
||||
geometry?: {
|
||||
rect: { x: number; y: number; w: number; h: number };
|
||||
scrollX: number;
|
||||
scrollY: number;
|
||||
position: string;
|
||||
tag: string;
|
||||
ownerFrameBackendNodeId?: number | null;
|
||||
};
|
||||
/** Capture-only hint; never persisted unless converted to navigated_to. */
|
||||
expects_navigation?: boolean;
|
||||
/** Whether an observed URL change was synchronously caused by the action. */
|
||||
navigation_caused_by_action?: boolean;
|
||||
/** Raw webNavigation transition metadata for cause mapping. */
|
||||
transitionType?: string;
|
||||
transitionQualifiers?: string[];
|
||||
}
|
||||
|
||||
export interface RecordStepMessage {
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
import type { FillCommit, NavigationCause } from "@/transport/types";
|
||||
|
||||
/**
|
||||
* Floor before a post-action observation may be taken. A page usually has not
|
||||
* started reacting in the first frames after an action, and capturing then
|
||||
* would record the page as it was, not as the action left it.
|
||||
*/
|
||||
export const SETTLE_MIN_MS = 150;
|
||||
|
||||
/** How long the DOM must stop changing before a page counts as settled. */
|
||||
export const SETTLE_QUIET_MS = 250;
|
||||
|
||||
/** Upper bound on waiting for a page to settle; animations never stop. */
|
||||
export const SETTLE_MAX_MS = 2_000;
|
||||
|
||||
/** How often to ask the page whether it has gone quiet. */
|
||||
export const SETTLE_POLL_MS = 60;
|
||||
|
||||
/** Minimum interval between consecutive observations on the same tab. */
|
||||
export const OBSERVATION_MIN_INTERVAL_MS = 200;
|
||||
|
||||
/** Delay before retrying a capture that lost its execution context. */
|
||||
export const CAPTURE_RETRY_DELAY_MS = 250;
|
||||
|
||||
/** Document-coordinate matching tolerance in CSS pixels. */
|
||||
export const GEOM_MATCH_TOLERANCE_PX = 2;
|
||||
|
||||
/** Default max tokens per page observation when CLI omits `--max-page-tokens`. */
|
||||
export const DEFAULT_MAX_PAGE_TOKENS = 3000;
|
||||
|
||||
/** VOM observation file format version (front matter header). */
|
||||
export const OBSERVATION_FILE_VERSION = 1;
|
||||
|
||||
/** FNV-1a 64-bit offset basis. */
|
||||
const FNV_OFFSET = 0xcbf29ce484222325n;
|
||||
/** FNV-1a 64-bit prime. */
|
||||
const FNV_PRIME = 0x100000001b3n;
|
||||
|
||||
/** Content-hash for state deduplication (sync, non-cryptographic). */
|
||||
export function fnv1a64(text: string): string {
|
||||
let hash = FNV_OFFSET;
|
||||
for (let i = 0; i < text.length; i += 1) {
|
||||
hash ^= BigInt(text.charCodeAt(i));
|
||||
hash = (hash * FNV_PRIME) & 0xffffffffffffffffn;
|
||||
}
|
||||
return hash.toString(16).padStart(16, "0");
|
||||
}
|
||||
|
||||
let nextStateSerial = 0;
|
||||
|
||||
/** Monotonic state id generator (`s1`, `s2`, …). */
|
||||
export function nextStateId(): string {
|
||||
nextStateSerial += 1;
|
||||
return `s${nextStateSerial}`;
|
||||
}
|
||||
|
||||
/** Test seam: reset the state id counter. */
|
||||
export function resetStateIdCounterForTests(): void {
|
||||
nextStateSerial = 0;
|
||||
}
|
||||
|
||||
const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
|
||||
|
||||
const TRANSITION_TO_CAUSE: Record<string, NavigationCause> = {
|
||||
typed: "user_typed",
|
||||
generated: "user_typed",
|
||||
keyword: "user_typed",
|
||||
keyword_generated: "user_typed",
|
||||
link: "link",
|
||||
form_submit: "form_submit",
|
||||
reload: "reload",
|
||||
auto_bookmark: "browser",
|
||||
start_page: "browser",
|
||||
};
|
||||
|
||||
export interface NavigationTransitionMeta {
|
||||
transitionType?: string;
|
||||
transitionQualifiers?: string[];
|
||||
navigationActionPending?: boolean;
|
||||
}
|
||||
|
||||
/** Map webNavigation metadata to protocol `NavigationCause`. Returns null for redirects. */
|
||||
export function mapNavigationCause(meta: NavigationTransitionMeta): NavigationCause | null {
|
||||
const qualifiers = meta.transitionQualifiers ?? [];
|
||||
if (qualifiers.includes("forward_back")) return "history";
|
||||
if (qualifiers.some((q) => REDIRECT_QUALIFIERS.has(q))) return null;
|
||||
if (qualifiers.includes("from_address_bar")) return "user_typed";
|
||||
|
||||
const type = meta.transitionType ?? "";
|
||||
const mapped = TRANSITION_TO_CAUSE[type];
|
||||
if (mapped) return mapped;
|
||||
|
||||
if (!type && meta.navigationActionPending === false) return "script";
|
||||
return "browser";
|
||||
}
|
||||
|
||||
export const DEFAULT_FILL_COMMIT: FillCommit = "blur";
|
||||
@@ -0,0 +1,663 @@
|
||||
import type { RenderedRef } from "@browser-skill/vom";
|
||||
import { formatObservationFile, type ObservationAnnotation } from "@/lib/format-observation-file";
|
||||
import { fallbackDescriptor, matchTarget } from "@/lib/match-target";
|
||||
import { waitForPageSettled } from "@/lib/page-settled";
|
||||
import {
|
||||
CAPTURE_RETRY_DELAY_MS,
|
||||
DEFAULT_MAX_PAGE_TOKENS,
|
||||
OBSERVATION_MIN_INTERVAL_MS,
|
||||
} from "@/lib/record-constants";
|
||||
import { registerObservation, type StateRegistryEntry } from "@/lib/trace-reducer";
|
||||
import { captureVomObservation } from "@/tools/capture-vom-observation";
|
||||
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||
import type { CapturedNode } from "@/tools/vom/capture";
|
||||
import type { DraftTraceStep, Step, StopReason, Trace, TraceState } from "@/transport/types";
|
||||
import { reduceTraceSteps, resolveTraceStartUrl } from "./trace-reducer";
|
||||
|
||||
export interface LastSettledObservation {
|
||||
stateId: string;
|
||||
captured: CapturedNode[];
|
||||
refs: RenderedRef[];
|
||||
url: string;
|
||||
title?: string;
|
||||
vomText: string;
|
||||
}
|
||||
|
||||
/** A post-action observation that has been queued but not yet written down. */
|
||||
interface PendingSettle {
|
||||
draftIndex: number;
|
||||
/** Set when a newer action has already decided where this step landed. */
|
||||
cancelled: boolean;
|
||||
}
|
||||
|
||||
/** Latest URL in an in-flight redirect chain awaiting coalesce + settle. */
|
||||
export interface PendingRedirectLanding {
|
||||
url: string;
|
||||
/** Bumped on every hop so an in-flight `waitForPageSettled` can cancel. */
|
||||
generation: number;
|
||||
}
|
||||
|
||||
export interface RecordingObservationState {
|
||||
stateRegistry: Map<string, StateRegistryEntry>;
|
||||
lastSettled: LastSettledObservation | null;
|
||||
maxPageTokens: number;
|
||||
redactValues: boolean;
|
||||
lastCaptureAtMs: number;
|
||||
/**
|
||||
* Tail of the settle chain. Observations run one at a time so that a slow
|
||||
* capture can never land after a faster one and report the pages out of the
|
||||
* order the user visited them.
|
||||
*/
|
||||
settleQueue: Promise<void>;
|
||||
/** Queued and in-flight settles, by draft index, so newer actions can supersede them. */
|
||||
settles: Map<number, PendingSettle>;
|
||||
stepAnnotations: Map<number, ObservationAnnotation[]>;
|
||||
/**
|
||||
* OAuth / server redirects are coalesced here: intermediate hops only update
|
||||
* `url`+`generation`, and one `navigate` is emitted after the page settles.
|
||||
*/
|
||||
pendingRedirect: PendingRedirectLanding | null;
|
||||
redirectFlushQueue: Promise<void>;
|
||||
}
|
||||
|
||||
export function createObservationState(options?: {
|
||||
maxPageTokens?: number;
|
||||
redactValues?: boolean;
|
||||
}): RecordingObservationState {
|
||||
return {
|
||||
stateRegistry: new Map(),
|
||||
lastSettled: null,
|
||||
maxPageTokens: options?.maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS,
|
||||
redactValues: options?.redactValues ?? false,
|
||||
lastCaptureAtMs: 0,
|
||||
settleQueue: Promise.resolve(),
|
||||
settles: new Map(),
|
||||
stepAnnotations: new Map(),
|
||||
pendingRedirect: null,
|
||||
redirectFlushQueue: Promise.resolve(),
|
||||
};
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function readTabMeta(
|
||||
tabsApi: ChromeTabsApi,
|
||||
tabId: number,
|
||||
): Promise<{ url: string; title?: string }> {
|
||||
try {
|
||||
const tab = await tabsApi.get(tabId);
|
||||
return { url: tab.url ?? "about:blank", title: tab.title };
|
||||
} catch {
|
||||
return { url: "about:blank" };
|
||||
}
|
||||
}
|
||||
|
||||
export async function captureAndRegisterObservation(
|
||||
obs: RecordingObservationState,
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
urlOverride?: string,
|
||||
): Promise<LastSettledObservation> {
|
||||
const now = Date.now();
|
||||
const waitMs = Math.max(0, OBSERVATION_MIN_INTERVAL_MS - (now - obs.lastCaptureAtMs));
|
||||
if (waitMs > 0) await sleep(waitMs);
|
||||
|
||||
const { url, title } = urlOverride
|
||||
? { url: urlOverride, title: undefined }
|
||||
: await readTabMeta(tabsApi, tabId);
|
||||
const meta = urlOverride ? await readTabMeta(tabsApi, tabId) : { url, title };
|
||||
const resolvedTitle = title ?? meta.title;
|
||||
|
||||
const rendered = await captureVomObservation(cdp, tabId, url, {
|
||||
maxTokens: obs.maxPageTokens,
|
||||
redactValues: obs.redactValues,
|
||||
conditionalSurfaceProbe: false,
|
||||
});
|
||||
|
||||
const stateId = registerObservation(obs.stateRegistry, {
|
||||
url,
|
||||
title: resolvedTitle,
|
||||
rawVomText: rendered.text,
|
||||
truncated: rendered.truncated,
|
||||
});
|
||||
|
||||
obs.lastCaptureAtMs = Date.now();
|
||||
const settled: LastSettledObservation = {
|
||||
stateId,
|
||||
captured: rendered.captured,
|
||||
refs: rendered.refs,
|
||||
url,
|
||||
title: resolvedTitle,
|
||||
vomText: rendered.text,
|
||||
};
|
||||
obs.lastSettled = settled;
|
||||
return settled;
|
||||
}
|
||||
|
||||
/**
|
||||
* Bind a draft to the page it was performed on: origin state, geometric target
|
||||
* match, and the inline annotation. All three read the *current* observation,
|
||||
* which is only the page the user acted on while the draft is still fresh —
|
||||
* once the action settles, `lastSettled` is the destination page instead.
|
||||
*/
|
||||
export function applyTargetMatching(
|
||||
obs: RecordingObservationState,
|
||||
draft: DraftTraceStep,
|
||||
stepId: number,
|
||||
): void {
|
||||
if (!obs.lastSettled) {
|
||||
// Initial observation can fail or race the first action. Keep whatever
|
||||
// the content script captured so the exported step is still teachable.
|
||||
if ("target" in draft && "captureTarget" in draft) {
|
||||
draft.target = fallbackDescriptor(draft.captureTarget);
|
||||
}
|
||||
rememberStepAnnotation(obs, stepId, draft);
|
||||
return;
|
||||
}
|
||||
draft.preStateId = obs.lastSettled.stateId;
|
||||
|
||||
if ("target" in draft) {
|
||||
draft.target =
|
||||
"geometry" in draft && draft.geometry
|
||||
? matchTarget({
|
||||
geometry: draft.geometry,
|
||||
captured: obs.lastSettled.captured,
|
||||
refs: obs.lastSettled.refs,
|
||||
fallback: draft.captureTarget,
|
||||
})
|
||||
: // No geometry to match on, but the capture still knew what the user
|
||||
// touched — keep that instead of an anonymous unmatched target.
|
||||
fallbackDescriptor(draft.captureTarget);
|
||||
}
|
||||
rememberStepAnnotation(obs, stepId, draft);
|
||||
}
|
||||
|
||||
function fillDetailForDraft(
|
||||
obs: RecordingObservationState,
|
||||
draft: DraftTraceStep,
|
||||
): string | undefined {
|
||||
if (obs.redactValues) return undefined;
|
||||
if (draft.op === "fill") return JSON.stringify(draft.value);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function refLineForDraft(
|
||||
obs: RecordingObservationState,
|
||||
draft: DraftTraceStep,
|
||||
): number | undefined {
|
||||
if (!("target" in draft) || !obs.lastSettled) return undefined;
|
||||
const targetRef = draft.target?.ref;
|
||||
if (!targetRef) return undefined;
|
||||
return obs.lastSettled.refs.find((r) => r.ref === targetRef)?.line;
|
||||
}
|
||||
|
||||
function rememberStepAnnotation(
|
||||
obs: RecordingObservationState,
|
||||
stepId: number,
|
||||
draft: DraftTraceStep,
|
||||
): void {
|
||||
const line = refLineForDraft(obs, draft);
|
||||
if (line === undefined || draft.op === "navigate" || draft.op === "scroll") return;
|
||||
if (!draft.preStateId) return;
|
||||
const ann: ObservationAnnotation = {
|
||||
stepId,
|
||||
op: draft.op,
|
||||
line,
|
||||
stateId: draft.preStateId,
|
||||
detail: fillDetailForDraft(obs, draft),
|
||||
};
|
||||
const bucket = obs.stepAnnotations.get(line) ?? [];
|
||||
bucket.push(ann);
|
||||
obs.stepAnnotations.set(line, bucket);
|
||||
}
|
||||
|
||||
/** Record the step under the page it was performed on, not the one it led to. */
|
||||
export function rememberStepOnPage(
|
||||
obs: RecordingObservationState,
|
||||
draft: DraftTraceStep,
|
||||
stepId: number,
|
||||
): void {
|
||||
const stateId = draft.preStateId;
|
||||
if (!stateId) return;
|
||||
const entry = obs.stateRegistry.get(stateId);
|
||||
if (entry && !entry.stepsHere.includes(stepId)) {
|
||||
entry.stepsHere.push(stepId);
|
||||
}
|
||||
}
|
||||
|
||||
/** Drop an in-flight redirect coalesce (e.g. a real navigate superseded it). */
|
||||
export function clearPendingRedirectLanding(obs: RecordingObservationState): void {
|
||||
obs.pendingRedirect = null;
|
||||
}
|
||||
|
||||
function enqueueRedirectFlush(obs: RecordingObservationState, task: () => Promise<void>): void {
|
||||
obs.redirectFlushQueue = obs.redirectFlushQueue.then(task, task).catch(() => {});
|
||||
}
|
||||
|
||||
/**
|
||||
* Remember a redirect hop and schedule a settle-then-emit of a single navigate
|
||||
* to the final URL. Later hops only bump `generation` / replace `url`.
|
||||
*/
|
||||
export function scheduleRedirectLandingFlush(
|
||||
obs: RecordingObservationState,
|
||||
steps: DraftTraceStep[],
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
url: string,
|
||||
options: { cancelled?: () => boolean } = {},
|
||||
): void {
|
||||
const generation = (obs.pendingRedirect?.generation ?? 0) + 1;
|
||||
obs.pendingRedirect = { url, generation };
|
||||
enqueueRedirectFlush(obs, () =>
|
||||
coalesceRedirectLanding(obs, steps, cdp, tabId, tabsApi, options),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Drain any coalesced redirect so the next action matches against the real
|
||||
* landing page. Safe to call when nothing is pending.
|
||||
*/
|
||||
export async function flushPendingRedirectLanding(
|
||||
obs: RecordingObservationState,
|
||||
steps: DraftTraceStep[],
|
||||
cdp: CdpRunner | undefined,
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
options: { cancelled?: () => boolean } = {},
|
||||
): Promise<void> {
|
||||
if (obs.pendingRedirect && cdp) {
|
||||
enqueueRedirectFlush(obs, () =>
|
||||
coalesceRedirectLanding(obs, steps, cdp, tabId, tabsApi, options),
|
||||
);
|
||||
}
|
||||
await obs.redirectFlushQueue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait until the redirect chain stops changing the document, then emit one
|
||||
* `navigate` (cause `browser`) to the tab's final URL and settle its observation.
|
||||
*/
|
||||
async function coalesceRedirectLanding(
|
||||
obs: RecordingObservationState,
|
||||
steps: DraftTraceStep[],
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
options: { cancelled?: () => boolean },
|
||||
): Promise<void> {
|
||||
while (obs.pendingRedirect) {
|
||||
if (options.cancelled?.()) {
|
||||
obs.pendingRedirect = null;
|
||||
return;
|
||||
}
|
||||
|
||||
const snap = obs.pendingRedirect;
|
||||
const outcome = await waitForPageSettled(cdp, tabId, {
|
||||
cancelled: () =>
|
||||
!!options.cancelled?.() ||
|
||||
obs.pendingRedirect === null ||
|
||||
obs.pendingRedirect.generation !== snap.generation,
|
||||
});
|
||||
|
||||
if (options.cancelled?.()) {
|
||||
obs.pendingRedirect = null;
|
||||
return;
|
||||
}
|
||||
// A newer hop cancelled this wait — loop and settle against the latest URL.
|
||||
if (outcome === "cancelled") continue;
|
||||
if (!obs.pendingRedirect || obs.pendingRedirect.generation !== snap.generation) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const finalUrl = (await readTabMeta(tabsApi, tabId)).url || snap.url;
|
||||
obs.pendingRedirect = null;
|
||||
|
||||
if (!finalUrl || finalUrl === "about:blank") return;
|
||||
if (obs.lastSettled?.url === finalUrl) return;
|
||||
|
||||
const last = steps[steps.length - 1];
|
||||
if (last?.op === "navigate" && last.url === finalUrl) {
|
||||
if (!last.postStateId) {
|
||||
const draftIndex = steps.length - 1;
|
||||
scheduleDraftSettle(obs, draftIndex, draftIndex + 1, cdp, tabId, tabsApi, steps);
|
||||
await obs.settleQueue;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (outcome === "timeout") {
|
||||
console.debug(
|
||||
`[bsk record] redirect landing still changing after settle budget; ` +
|
||||
`recording navigate to ${finalUrl}`,
|
||||
);
|
||||
}
|
||||
|
||||
const draft: DraftTraceStep = {
|
||||
op: "navigate",
|
||||
url: finalUrl,
|
||||
page_url: finalUrl,
|
||||
cause: "browser",
|
||||
preStateId: obs.lastSettled?.stateId,
|
||||
};
|
||||
steps.push(draft);
|
||||
const draftIndex = steps.length - 1;
|
||||
const stepId = draftIndex + 1;
|
||||
rememberStepOnPage(obs, draft, stepId);
|
||||
scheduleDraftSettle(obs, draftIndex, stepId, cdp, tabId, tabsApi, steps);
|
||||
// Callers that flush before the next action need `lastSettled` to already
|
||||
// be the landing page so target matching does not use the pre-redirect view.
|
||||
await obs.settleQueue;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A capture that lands mid-navigation fails on a destroyed execution context.
|
||||
* That is transient, so give it one more chance before the step has to fall
|
||||
* back to a stale state.
|
||||
*/
|
||||
async function captureWithRetry(
|
||||
obs: RecordingObservationState,
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
urlOverride?: string,
|
||||
): Promise<LastSettledObservation> {
|
||||
try {
|
||||
return await captureAndRegisterObservation(obs, cdp, tabId, tabsApi, urlOverride);
|
||||
} catch (err) {
|
||||
console.debug("[bsk record] observation failed, retrying once", err);
|
||||
await sleep(CAPTURE_RETRY_DELAY_MS);
|
||||
return captureAndRegisterObservation(obs, cdp, tabId, tabsApi, urlOverride);
|
||||
}
|
||||
}
|
||||
|
||||
export async function settleDraftObservation(
|
||||
obs: RecordingObservationState,
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
steps: DraftTraceStep[],
|
||||
draftIndex: number,
|
||||
pending?: PendingSettle,
|
||||
): Promise<void> {
|
||||
const cancelled = () => pending?.cancelled === true;
|
||||
const outcome = await waitForPageSettled(cdp, tabId, { cancelled });
|
||||
if (outcome === "cancelled") return;
|
||||
if (outcome === "timeout") {
|
||||
console.debug(
|
||||
`[bsk record] step ${draftIndex + 1} was still changing the page after the settle ` +
|
||||
"budget; observing it as it stands",
|
||||
);
|
||||
}
|
||||
|
||||
const started = steps[draftIndex];
|
||||
if (!started) return;
|
||||
|
||||
// The URL attached to the draft may only be an intermediate redirect hop.
|
||||
// Once the page is settled, register the capture against live tab metadata.
|
||||
const settled = await captureWithRetry(obs, cdp, tabId, tabsApi);
|
||||
// The observation still counts as the latest view of the page even when this
|
||||
// step no longer wants it — the next action will start from it.
|
||||
if (cancelled()) return;
|
||||
// Re-read the slot: a navigation observed while the capture ran may have
|
||||
// rewritten this draft, and the observation belongs to whatever occupies the
|
||||
// slot now — writing to the object we started with would strand it.
|
||||
const draft = steps[draftIndex] ?? started;
|
||||
draft.postStateId = settled.stateId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recover landing states from the shape of the recording itself: wherever the
|
||||
* next action was performed is, by definition, where the previous one landed.
|
||||
* Only the immediately following draft counts — a later one would claim a page
|
||||
* that several unobserved actions away, which is worse than admitting we saw
|
||||
* no change.
|
||||
*/
|
||||
export function inferMissingPostStates(steps: DraftTraceStep[]): void {
|
||||
for (let i = 0; i < steps.length - 1; i += 1) {
|
||||
const draft = steps[i];
|
||||
const next = steps[i + 1];
|
||||
if (!draft || !next || draft.postStateId || !next.preStateId) continue;
|
||||
draft.postStateId = next.preStateId;
|
||||
console.debug(
|
||||
`[bsk record] step ${i + 1} (${draft.op}) had no post-action observation; ` +
|
||||
`using where step ${i + 2} started (${next.preStateId})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Last chance for the closing actions of a recording. Stopping right after the
|
||||
* final action is normal, and so is that action navigating away, so take one
|
||||
* final look at the page. Only trailing drafts may claim it: an earlier step
|
||||
* did not land on whatever the user happens to be looking at when they stop.
|
||||
*/
|
||||
export async function settleUnsettledDrafts(
|
||||
obs: RecordingObservationState,
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
steps: DraftTraceStep[],
|
||||
): Promise<void> {
|
||||
const trailing: DraftTraceStep[] = [];
|
||||
for (let i = steps.length - 1; i >= 0; i -= 1) {
|
||||
const draft = steps[i];
|
||||
if (!draft || draft.postStateId) break;
|
||||
trailing.push(draft);
|
||||
}
|
||||
if (trailing.length === 0) return;
|
||||
|
||||
let settled: LastSettledObservation;
|
||||
try {
|
||||
settled = await captureWithRetry(obs, cdp, tabId, tabsApi);
|
||||
} catch (err) {
|
||||
console.warn("[bsk record] final observation at stop failed", err);
|
||||
return;
|
||||
}
|
||||
for (const draft of trailing) draft.postStateId = settled.stateId;
|
||||
console.debug(
|
||||
`[bsk record] settled ${trailing.length} trailing step(s) against the page at stop ` +
|
||||
`(${settled.stateId} ${settled.url})`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Performing a new action ends the previous one: the page the user reached for
|
||||
* is, by definition, where the earlier step left them. Taking the landing from
|
||||
* the newer step also keeps the trace monotonic — a capture that finished late
|
||||
* would otherwise credit an earlier step with a page that only exists because
|
||||
* of the newer action.
|
||||
*/
|
||||
function supersedeEarlierSettles(
|
||||
obs: RecordingObservationState,
|
||||
draftIndex: number,
|
||||
steps: DraftTraceStep[],
|
||||
): void {
|
||||
const landing = steps[draftIndex]?.preStateId;
|
||||
for (const [index, pending] of obs.settles) {
|
||||
if (index >= draftIndex) continue;
|
||||
pending.cancelled = true;
|
||||
obs.settles.delete(index);
|
||||
|
||||
const draft = steps[index];
|
||||
if (!draft || draft.postStateId) continue;
|
||||
if (!landing) continue;
|
||||
draft.postStateId = landing;
|
||||
console.debug(
|
||||
`[bsk record] step ${index + 1} (${draft.op}) was still settling when step ` +
|
||||
`${draftIndex + 1} started; landing it on ${landing}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function scheduleDraftSettle(
|
||||
obs: RecordingObservationState,
|
||||
draftIndex: number,
|
||||
stepId: number,
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
steps: DraftTraceStep[],
|
||||
): void {
|
||||
supersedeEarlierSettles(obs, draftIndex, steps);
|
||||
|
||||
// Rescheduling the same step (a navigation showed up after the action)
|
||||
// replaces the pending observation rather than racing it.
|
||||
const superseded = obs.settles.get(draftIndex);
|
||||
if (superseded) superseded.cancelled = true;
|
||||
|
||||
const pending: PendingSettle = { draftIndex, cancelled: false };
|
||||
obs.settles.set(draftIndex, pending);
|
||||
|
||||
enqueueSettle(obs, async () => {
|
||||
if (pending.cancelled) return;
|
||||
try {
|
||||
await settleDraftObservation(obs, cdp, tabId, tabsApi, steps, draftIndex, pending);
|
||||
} catch (err) {
|
||||
// Recoverable: stop-time repair still gives the step a landing page.
|
||||
const op = steps[draftIndex]?.op ?? "?";
|
||||
console.warn(`[bsk record] post-action observation failed for step ${stepId} (${op})`, err);
|
||||
} finally {
|
||||
if (obs.settles.get(draftIndex) === pending) obs.settles.delete(draftIndex);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function enqueueSettle(obs: RecordingObservationState, task: () => Promise<void>): void {
|
||||
obs.settleQueue = obs.settleQueue.then(task, task).catch(() => {});
|
||||
}
|
||||
|
||||
export function cancelPendingSettles(obs: RecordingObservationState): void {
|
||||
for (const pending of obs.settles.values()) pending.cancelled = true;
|
||||
obs.settles.clear();
|
||||
}
|
||||
|
||||
/** Bound the drain loop so a self-rescheduling settle cannot block stop. */
|
||||
const MAX_SETTLE_FLUSH_ROUNDS = 10;
|
||||
|
||||
/**
|
||||
* Drain settle work before exporting, including work queued while draining —
|
||||
* a navigation observed during the last capture schedules another one, and
|
||||
* the trace would drop it if stop did not wait.
|
||||
*/
|
||||
export async function flushPendingSettles(obs: RecordingObservationState): Promise<void> {
|
||||
for (let round = 0; round < MAX_SETTLE_FLUSH_ROUNDS; round += 1) {
|
||||
const drained = obs.settleQueue;
|
||||
await drained;
|
||||
if (obs.settleQueue === drained) return;
|
||||
}
|
||||
console.warn("[bsk record] settle queue kept growing at stop; exporting what has been observed");
|
||||
}
|
||||
|
||||
function finalizeStateBodies(
|
||||
entries: StateRegistryEntry[],
|
||||
annotationsByState: Map<string, ObservationAnnotation[]>,
|
||||
stepIdByDraftId: Map<number, number>,
|
||||
idByOldId: Map<string, string>,
|
||||
): TraceState[] {
|
||||
return entries.map((entry) => {
|
||||
const id = idByOldId.get(entry.id) ?? entry.id;
|
||||
const body = formatObservationFile({
|
||||
stateId: id,
|
||||
url: entry.url,
|
||||
title: entry.title,
|
||||
stepsHere: remapStepIds(entry.stepsHere, stepIdByDraftId),
|
||||
body: entry.rawVomText,
|
||||
annotations: annotationsByState.get(entry.id) ?? [],
|
||||
});
|
||||
return {
|
||||
id,
|
||||
url: entry.url,
|
||||
...(entry.title ? { title: entry.title } : {}),
|
||||
body,
|
||||
...(entry.truncated ? { truncated: true } : {}),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** Draft ids only become step ids after collapsing and filtering. */
|
||||
function remapStepIds(draftIds: number[], stepIdByDraftId: Map<number, number>): number[] {
|
||||
const mapped = new Set<number>();
|
||||
for (const draftId of draftIds) {
|
||||
const stepId = stepIdByDraftId.get(draftId);
|
||||
if (stepId !== undefined) mapped.add(stepId);
|
||||
}
|
||||
return [...mapped].sort((a, b) => a - b);
|
||||
}
|
||||
|
||||
function collectAnnotationsByState(
|
||||
obs: RecordingObservationState,
|
||||
stepIdByDraftId: Map<number, number>,
|
||||
): Map<string, ObservationAnnotation[]> {
|
||||
const byState = new Map<string, ObservationAnnotation[]>();
|
||||
for (const anns of obs.stepAnnotations.values()) {
|
||||
for (const ann of anns) {
|
||||
const stepId = stepIdByDraftId.get(ann.stepId);
|
||||
if (stepId === undefined) continue;
|
||||
const bucket = byState.get(ann.stateId) ?? [];
|
||||
bucket.push({ ...ann, stepId });
|
||||
byState.set(ann.stateId, bucket);
|
||||
}
|
||||
}
|
||||
return byState;
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep only the pages the published steps point at. Redirect hops and
|
||||
* mid-load captures land in the registry too, and shipping them would invite
|
||||
* a reader to treat a page the flow merely passed through as a real stop.
|
||||
* With no steps at all the first observation is the whole artifact, so it
|
||||
* stays.
|
||||
*/
|
||||
function selectPublishedStates(
|
||||
registry: Map<string, StateRegistryEntry>,
|
||||
steps: Step[],
|
||||
): StateRegistryEntry[] {
|
||||
const entries = [...registry.values()];
|
||||
if (steps.length === 0) return entries.slice(0, 1);
|
||||
const referenced = new Set<string>();
|
||||
for (const step of steps) {
|
||||
referenced.add(step.state);
|
||||
referenced.add(step.result.state);
|
||||
}
|
||||
return entries.filter((entry) => referenced.has(entry.id));
|
||||
}
|
||||
|
||||
export function buildTraceV3(input: {
|
||||
obs: RecordingObservationState;
|
||||
steps: DraftTraceStep[];
|
||||
startedAt: string;
|
||||
purpose?: string;
|
||||
startUrl?: string;
|
||||
stoppedBy: StopReason;
|
||||
bskVersion: string;
|
||||
}): Trace {
|
||||
const { steps, stepIdByDraftId } = reduceTraceSteps(input.steps, input.obs.stateRegistry);
|
||||
const published = selectPublishedStates(input.obs.stateRegistry, steps);
|
||||
// Renumber so the shipped dictionary reads s1..sN without holes where the
|
||||
// dropped captures used to be.
|
||||
const idByOldId = new Map(published.map((entry, index) => [entry.id, `s${index + 1}`]));
|
||||
for (const step of steps) {
|
||||
step.state = idByOldId.get(step.state) ?? step.state;
|
||||
step.result.state = idByOldId.get(step.result.state) ?? step.result.state;
|
||||
}
|
||||
const annotationsByState = collectAnnotationsByState(input.obs, stepIdByDraftId);
|
||||
const states = finalizeStateBodies(published, annotationsByState, stepIdByDraftId, idByOldId);
|
||||
const startUrl = resolveTraceStartUrl(input.steps, input.startUrl, states);
|
||||
return {
|
||||
version: 3,
|
||||
...(input.purpose ? { purpose: input.purpose } : {}),
|
||||
recorded_at: new Date().toISOString(),
|
||||
started_at: input.startedAt,
|
||||
stopped_by: input.stoppedBy,
|
||||
entry: { start_url: startUrl },
|
||||
recorder: { bsk: input.bskVersion, vom: 1 },
|
||||
states,
|
||||
steps,
|
||||
};
|
||||
}
|
||||
@@ -1,23 +1,29 @@
|
||||
import type { DraftTraceStep } from "@/transport/types";
|
||||
import type { DraftTraceStep, NavigationCause } from "@/transport/types";
|
||||
import type { RecordStepPayload } from "./record-bridge";
|
||||
import { mapNavigationCause } from "./record-constants";
|
||||
|
||||
export interface RecordingStepBuffer {
|
||||
steps: DraftTraceStep[];
|
||||
currentUrl?: string;
|
||||
pendingNavigation: boolean;
|
||||
pendingNavigationDeadline?: number;
|
||||
lastTransitionType?: string;
|
||||
lastTransitionQualifiers?: string[];
|
||||
}
|
||||
|
||||
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
|
||||
|
||||
function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
||||
const pageUrl = payload.page_url;
|
||||
const geometry = payload.geometry;
|
||||
switch (payload.op) {
|
||||
case "click":
|
||||
return payload.target
|
||||
? {
|
||||
op: "click",
|
||||
target: payload.target,
|
||||
target: { unmatched: true },
|
||||
captureTarget: payload.target,
|
||||
...(geometry ? { geometry } : {}),
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
}
|
||||
: null;
|
||||
@@ -25,7 +31,9 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
||||
return payload.target
|
||||
? {
|
||||
op: "hover",
|
||||
target: payload.target,
|
||||
target: { unmatched: true },
|
||||
captureTarget: payload.target,
|
||||
...(geometry ? { geometry } : {}),
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
}
|
||||
: null;
|
||||
@@ -33,8 +41,11 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
||||
return payload.target
|
||||
? {
|
||||
op: "fill",
|
||||
target: payload.target,
|
||||
target: { unmatched: true },
|
||||
captureTarget: payload.target,
|
||||
...(geometry ? { geometry } : {}),
|
||||
value: payload.value ?? "",
|
||||
...(payload.commit ? { commit: payload.commit } : {}),
|
||||
...(payload.redacted ? { redacted: true } : {}),
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
}
|
||||
@@ -44,7 +55,10 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
||||
? {
|
||||
op: "press",
|
||||
key: payload.key,
|
||||
...(payload.target ? { target: payload.target } : {}),
|
||||
...(payload.target
|
||||
? { target: { unmatched: true }, captureTarget: payload.target }
|
||||
: {}),
|
||||
...(geometry ? { geometry } : {}),
|
||||
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
}
|
||||
@@ -53,37 +67,69 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
||||
return payload.target && payload.values
|
||||
? {
|
||||
op: "select",
|
||||
target: payload.target,
|
||||
target: { unmatched: true },
|
||||
captureTarget: payload.target,
|
||||
...(geometry ? { geometry } : {}),
|
||||
values: payload.values,
|
||||
...(payload.labels?.length ? { labels: payload.labels } : {}),
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
}
|
||||
: null;
|
||||
case "scroll":
|
||||
return { op: "scroll", ...(pageUrl ? { page_url: pageUrl } : {}) };
|
||||
case "navigate":
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): boolean {
|
||||
/** Annotate the latest action with `navigated_to`. Returns its index, or -1. */
|
||||
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): number {
|
||||
for (let i = buffer.steps.length - 1; i >= 0; i -= 1) {
|
||||
const step = buffer.steps[i];
|
||||
if (!step) continue;
|
||||
if (step.op === "click" || step.op === "press" || step.op === "select") {
|
||||
buffer.steps[i] = { ...step, navigated_to: url };
|
||||
return true;
|
||||
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
||||
// Mutate in place: an in-flight settle holds this object, and replacing
|
||||
// it would strand that settle's observation on the discarded copy.
|
||||
step.navigated_to = url;
|
||||
return i;
|
||||
}
|
||||
break;
|
||||
}
|
||||
return false;
|
||||
return -1;
|
||||
}
|
||||
|
||||
function causeFromBuffer(buffer: RecordingStepBuffer): NavigationCause | null {
|
||||
return mapNavigationCause({
|
||||
transitionType: buffer.lastTransitionType,
|
||||
transitionQualifiers: buffer.lastTransitionQualifiers,
|
||||
});
|
||||
}
|
||||
|
||||
/** Outcome of folding a main-frame URL change into the draft buffer. */
|
||||
export type NavigationObserveResult =
|
||||
| { kind: "noop" }
|
||||
| { kind: "annotated"; index: number }
|
||||
| { kind: "appended"; index: number }
|
||||
/**
|
||||
* Redirect hop: do not emit a step yet. The recorder coalesces the chain and
|
||||
* emits one `navigate` after `waitForPageSettled` on the final URL.
|
||||
*/
|
||||
| { kind: "coalesce_redirect"; url: string };
|
||||
|
||||
export function observeRecordedNavigation(
|
||||
buffer: RecordingStepBuffer,
|
||||
url: string,
|
||||
causedByAction?: boolean,
|
||||
): void {
|
||||
if (!url || url === buffer.currentUrl) return;
|
||||
transitionType?: string,
|
||||
transitionQualifiers?: string[],
|
||||
): NavigationObserveResult {
|
||||
if (!url || url === buffer.currentUrl) return { kind: "noop" };
|
||||
buffer.currentUrl = url;
|
||||
if (transitionType !== undefined) buffer.lastTransitionType = transitionType;
|
||||
if (transitionQualifiers !== undefined) {
|
||||
buffer.lastTransitionQualifiers = transitionQualifiers;
|
||||
}
|
||||
|
||||
const pendingIsCurrent =
|
||||
buffer.pendingNavigation &&
|
||||
(buffer.pendingNavigationDeadline === undefined ||
|
||||
@@ -92,40 +138,69 @@ export function observeRecordedNavigation(
|
||||
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
|
||||
buffer.pendingNavigation = false;
|
||||
buffer.pendingNavigationDeadline = undefined;
|
||||
if (!annotateLastStepNavigation(buffer, url)) {
|
||||
buffer.steps.push({
|
||||
op: "navigate",
|
||||
url,
|
||||
page_url: url,
|
||||
});
|
||||
const annotatedIndex = annotateLastStepNavigation(buffer, url);
|
||||
if (annotatedIndex >= 0) {
|
||||
return { kind: "annotated", index: annotatedIndex };
|
||||
}
|
||||
return;
|
||||
const cause = causeFromBuffer(buffer);
|
||||
if (cause === null) {
|
||||
return { kind: "coalesce_redirect", url };
|
||||
}
|
||||
buffer.steps.push({
|
||||
op: "navigate",
|
||||
url,
|
||||
page_url: url,
|
||||
cause,
|
||||
transitionType: buffer.lastTransitionType,
|
||||
transitionQualifiers: buffer.lastTransitionQualifiers,
|
||||
});
|
||||
return { kind: "appended", index: buffer.steps.length - 1 };
|
||||
}
|
||||
|
||||
buffer.pendingNavigation = false;
|
||||
buffer.pendingNavigationDeadline = undefined;
|
||||
const cause = causeFromBuffer(buffer);
|
||||
if (cause === null) {
|
||||
return { kind: "coalesce_redirect", url };
|
||||
}
|
||||
buffer.steps.push({
|
||||
op: "navigate",
|
||||
url,
|
||||
page_url: url,
|
||||
cause,
|
||||
transitionType: buffer.lastTransitionType,
|
||||
transitionQualifiers: buffer.lastTransitionQualifiers,
|
||||
});
|
||||
return { kind: "appended", index: buffer.steps.length - 1 };
|
||||
}
|
||||
|
||||
/**
|
||||
* Buffer one captured payload. Returns the index of the draft it appended, or
|
||||
* `null` when the payload produced none (an unnamed click target, a navigation
|
||||
* that only annotated the previous draft, …). Callers must not fall back to
|
||||
* "the last draft" on `null`: that would re-match an already recorded step
|
||||
* against a newer observation and overwrite its origin state.
|
||||
*/
|
||||
export function appendRecordedPayload(
|
||||
buffer: RecordingStepBuffer,
|
||||
payload: RecordStepPayload,
|
||||
): void {
|
||||
): number | null {
|
||||
if (payload.op === "navigate") {
|
||||
if (payload.url) {
|
||||
observeRecordedNavigation(buffer, payload.url, payload.navigation_caused_by_action);
|
||||
}
|
||||
return;
|
||||
if (!payload.url) return null;
|
||||
const result = observeRecordedNavigation(
|
||||
buffer,
|
||||
payload.url,
|
||||
payload.navigation_caused_by_action,
|
||||
payload.transitionType,
|
||||
payload.transitionQualifiers,
|
||||
);
|
||||
return result.kind === "appended" ? result.index : null;
|
||||
}
|
||||
const step = toDraftStep({
|
||||
...payload,
|
||||
page_url: payload.page_url ?? buffer.currentUrl,
|
||||
});
|
||||
if (!step) return;
|
||||
if (!step) return null;
|
||||
buffer.steps.push(step);
|
||||
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
||||
buffer.pendingNavigation = payload.expects_navigation === true;
|
||||
@@ -133,4 +208,5 @@ export function appendRecordedPayload(
|
||||
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
|
||||
: undefined;
|
||||
}
|
||||
return buffer.steps.length - 1;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,297 @@
|
||||
import type { CaptureTargetDescriptor } from "@/lib/describe-target";
|
||||
import type { DraftTraceStep, KeyModifier } from "@/transport/types";
|
||||
|
||||
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
|
||||
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
|
||||
|
||||
export interface TargetDescriptorV2 {
|
||||
role?: string;
|
||||
name?: string;
|
||||
tag: string;
|
||||
name_attr?: string;
|
||||
placeholder?: string;
|
||||
nearby_label?: string;
|
||||
}
|
||||
|
||||
export interface PageRef {
|
||||
id: string;
|
||||
url: string;
|
||||
title?: string;
|
||||
}
|
||||
|
||||
export interface SelectedOptionV2 {
|
||||
value: string;
|
||||
label?: string;
|
||||
}
|
||||
|
||||
export interface StepEffectV2 {
|
||||
navigated_to: string;
|
||||
}
|
||||
|
||||
export type StepV2 =
|
||||
| { op: "navigate"; id: number; page: string; to: string; effect?: StepEffectV2 }
|
||||
| { op: "click"; id: number; page: string; target: TargetDescriptorV2; effect?: StepEffectV2 }
|
||||
| {
|
||||
op: "fill";
|
||||
id: number;
|
||||
page: string;
|
||||
target: TargetDescriptorV2;
|
||||
value: string;
|
||||
redacted?: boolean;
|
||||
effect?: StepEffectV2;
|
||||
}
|
||||
| {
|
||||
op: "select";
|
||||
id: number;
|
||||
page: string;
|
||||
target: TargetDescriptorV2;
|
||||
selection: SelectedOptionV2[];
|
||||
effect?: StepEffectV2;
|
||||
}
|
||||
| {
|
||||
op: "press";
|
||||
id: number;
|
||||
page: string;
|
||||
key: string;
|
||||
modifiers?: KeyModifier[];
|
||||
target?: TargetDescriptorV2;
|
||||
effect?: StepEffectV2;
|
||||
};
|
||||
|
||||
export interface TraceV2 {
|
||||
recorded_at: string;
|
||||
started_at?: string;
|
||||
purpose?: string;
|
||||
entry: { start_url: string };
|
||||
pages: PageRef[];
|
||||
steps: StepV2[];
|
||||
}
|
||||
|
||||
export function shouldRecordPress(
|
||||
key: string,
|
||||
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
|
||||
): boolean {
|
||||
if (MODIFIER_ONLY_KEYS.has(key)) return false;
|
||||
const mods = modifiers ?? [];
|
||||
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
|
||||
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
|
||||
if (key === "Enter" || key === "Escape") return true;
|
||||
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
|
||||
return false;
|
||||
}
|
||||
|
||||
function shouldIncludeDraft(step: DraftTraceStep): boolean {
|
||||
if (step.op === "scroll" || step.op === "hover") return false;
|
||||
if (step.op === "fill" && !(step.value ?? "").trim() && !step.redacted) return false;
|
||||
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
|
||||
const out: DraftTraceStep[] = [];
|
||||
for (const step of steps) {
|
||||
const prev = out[out.length - 1];
|
||||
if (step.op === "navigate" && prev?.op === "navigate") {
|
||||
out[out.length - 1] = step;
|
||||
continue;
|
||||
}
|
||||
out.push(step);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
||||
const urls: string[] = [];
|
||||
if (startUrl) urls.push(startUrl);
|
||||
for (const step of steps) {
|
||||
if (step.op === "navigate") {
|
||||
urls.push(step.url);
|
||||
continue;
|
||||
}
|
||||
if ("page_url" in step && step.page_url) urls.push(step.page_url);
|
||||
if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to);
|
||||
}
|
||||
const seen = new Set<string>();
|
||||
const unique: string[] = [];
|
||||
for (const url of urls) {
|
||||
if (!url || seen.has(url)) continue;
|
||||
seen.add(url);
|
||||
unique.push(url);
|
||||
}
|
||||
return unique;
|
||||
}
|
||||
|
||||
function buildPageRegistry(
|
||||
steps: DraftTraceStep[],
|
||||
startUrl?: string,
|
||||
): { pages: PageRef[]; urlToId: Map<string, string> } {
|
||||
const urls = collectUrls(steps, startUrl);
|
||||
const urlToId = new Map<string, string>();
|
||||
const pages = urls.map((url, index) => {
|
||||
const id = `p${index + 1}`;
|
||||
urlToId.set(url, id);
|
||||
return { id, url };
|
||||
});
|
||||
return { pages, urlToId };
|
||||
}
|
||||
|
||||
function pageIdFor(
|
||||
url: string | undefined,
|
||||
urlToId: Map<string, string>,
|
||||
fallbackUrl?: string,
|
||||
): string {
|
||||
if (url && urlToId.has(url)) return urlToId.get(url)!;
|
||||
if (fallbackUrl && urlToId.has(fallbackUrl)) return urlToId.get(fallbackUrl)!;
|
||||
return urlToId.values().next().value ?? "p1";
|
||||
}
|
||||
|
||||
function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined {
|
||||
if (step.op === "navigate") return step.page_url ?? step.url;
|
||||
if ("page_url" in step && step.page_url) return step.page_url;
|
||||
return fallbackUrl;
|
||||
}
|
||||
|
||||
function effectForNavigation(
|
||||
navigatedTo: string | undefined,
|
||||
urlToId: Map<string, string>,
|
||||
): StepEffectV2 | undefined {
|
||||
if (!navigatedTo) return undefined;
|
||||
const pageId = urlToId.get(navigatedTo);
|
||||
if (!pageId) return undefined;
|
||||
return { navigated_to: pageId };
|
||||
}
|
||||
|
||||
function captureTargetToV2Target(capture?: CaptureTargetDescriptor): TargetDescriptorV2 {
|
||||
return {
|
||||
tag: capture?.tag ?? "unknown",
|
||||
...(capture?.role ? { role: capture.role } : {}),
|
||||
...(capture?.name ? { name: capture.name } : {}),
|
||||
...(capture?.name_attr ? { name_attr: capture.name_attr } : {}),
|
||||
...(capture?.placeholder ? { placeholder: capture.placeholder } : {}),
|
||||
...(capture?.nearby_label ? { nearby_label: capture.nearby_label } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function targetForDraft(step: DraftTraceStep): TargetDescriptorV2 | undefined {
|
||||
if (!("target" in step) && !("captureTarget" in step)) return undefined;
|
||||
const capture = "captureTarget" in step ? step.captureTarget : undefined;
|
||||
if (capture) return captureTargetToV2Target(capture);
|
||||
if ("target" in step && step.target && "tag" in (step.target as object)) {
|
||||
const legacy = step.target as TargetDescriptorV2 & { tag?: string };
|
||||
if (legacy.tag) return legacy;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function toSelection(values: string[], labels?: string[]): SelectedOptionV2[] {
|
||||
return values.map((value, index) => ({
|
||||
value,
|
||||
...(labels?.[index] ? { label: labels[index] } : {}),
|
||||
}));
|
||||
}
|
||||
|
||||
function toV2Step(
|
||||
step: DraftTraceStep,
|
||||
id: number,
|
||||
urlToId: Map<string, string>,
|
||||
fallbackUrl?: string,
|
||||
): StepV2 | null {
|
||||
if (!shouldIncludeDraft(step)) return null;
|
||||
|
||||
const pageUrl = pageUrlForDraft(step, fallbackUrl);
|
||||
const page = pageIdFor(pageUrl, urlToId, fallbackUrl);
|
||||
const effect =
|
||||
"navigated_to" in step ? effectForNavigation(step.navigated_to, urlToId) : undefined;
|
||||
|
||||
switch (step.op) {
|
||||
case "navigate":
|
||||
return {
|
||||
op: "navigate",
|
||||
id,
|
||||
page: pageIdFor(step.url, urlToId, fallbackUrl),
|
||||
to: step.url,
|
||||
};
|
||||
case "click": {
|
||||
const target = targetForDraft(step);
|
||||
if (!target) return null;
|
||||
return { op: "click", id, page, target, ...(effect ? { effect } : {}) };
|
||||
}
|
||||
case "hover":
|
||||
return null;
|
||||
case "fill": {
|
||||
const target = targetForDraft(step);
|
||||
if (!target) return null;
|
||||
return {
|
||||
op: "fill",
|
||||
id,
|
||||
page,
|
||||
target,
|
||||
value: step.value,
|
||||
...(step.redacted ? { redacted: true } : {}),
|
||||
};
|
||||
}
|
||||
case "press":
|
||||
return {
|
||||
op: "press",
|
||||
id,
|
||||
page,
|
||||
key: step.key,
|
||||
...(step.target ? { target: targetForDraft(step) } : {}),
|
||||
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
|
||||
...(effect ? { effect } : {}),
|
||||
};
|
||||
case "select": {
|
||||
const target = targetForDraft(step);
|
||||
if (!target) return null;
|
||||
return {
|
||||
op: "select",
|
||||
id,
|
||||
page,
|
||||
target,
|
||||
selection: toSelection(step.values, step.labels),
|
||||
...(effect ? { effect } : {}),
|
||||
};
|
||||
}
|
||||
case "scroll":
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export interface BuildTraceV2Input {
|
||||
steps: DraftTraceStep[];
|
||||
startedAt: string;
|
||||
startUrl?: string;
|
||||
purpose?: string;
|
||||
}
|
||||
|
||||
export function buildTraceV2(input: BuildTraceV2Input): TraceV2 {
|
||||
const collapsed = collapseNavigations(input.steps);
|
||||
const startUrl =
|
||||
input.startUrl ??
|
||||
collapsed.find(
|
||||
(step): step is Extract<DraftTraceStep, { op: "navigate" }> => step.op === "navigate",
|
||||
)?.url ??
|
||||
collapsed.find((step) => "page_url" in step && step.page_url)?.page_url ??
|
||||
"about:blank";
|
||||
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
|
||||
const out: StepV2[] = [];
|
||||
let id = 1;
|
||||
let lastUrl = startUrl;
|
||||
for (const draft of collapsed) {
|
||||
if (draft.op === "navigate") lastUrl = draft.url;
|
||||
else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to;
|
||||
else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url;
|
||||
const step = toV2Step(draft, id, urlToId, lastUrl);
|
||||
if (!step) continue;
|
||||
out.push(step);
|
||||
id += 1;
|
||||
}
|
||||
return {
|
||||
recorded_at: new Date().toISOString(),
|
||||
started_at: input.startedAt,
|
||||
...(input.purpose ? { purpose: input.purpose } : {}),
|
||||
entry: { start_url: startUrl },
|
||||
pages,
|
||||
steps: out,
|
||||
};
|
||||
}
|
||||
@@ -1,4 +1,17 @@
|
||||
import type { DraftTraceStep, PageRef, SelectedOption, Step } from "@/transport/types";
|
||||
import type {
|
||||
DraftTraceStep,
|
||||
FillCommit,
|
||||
NavigationCause,
|
||||
Step,
|
||||
TraceState,
|
||||
} from "@/transport/types";
|
||||
import {
|
||||
DEFAULT_FILL_COMMIT,
|
||||
fnv1a64,
|
||||
mapNavigationCause,
|
||||
nextStateId,
|
||||
resetStateIdCounterForTests,
|
||||
} from "./record-constants";
|
||||
|
||||
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
|
||||
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
|
||||
@@ -12,206 +25,200 @@ export function shouldRecordPress(
|
||||
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
|
||||
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
|
||||
if (key === "Enter" || key === "Escape") return true;
|
||||
// Drop bare character typing — FillSession already records the value.
|
||||
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
|
||||
return false;
|
||||
}
|
||||
|
||||
function shouldIncludeDraft(step: DraftTraceStep): boolean {
|
||||
if (step.op === "fill" && !(step.value ?? "").trim() && !step.redacted) return false;
|
||||
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Collapse consecutive navigations to the last hop. */
|
||||
function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
|
||||
const out: DraftTraceStep[] = [];
|
||||
for (const step of steps) {
|
||||
/** A reduced draft plus the capture-order ids of every draft folded into it. */
|
||||
interface CollapsedDraft {
|
||||
draft: DraftTraceStep;
|
||||
draftIds: number[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Collapse a redirect chain into a single hop.
|
||||
*
|
||||
* The surviving step keeps the *first* hop's origin state and cause — that is
|
||||
* where the user was and why they left — and the *last* hop's destination, so
|
||||
* the intermediate URL the browser only passed through never becomes the
|
||||
* step's `state` or `to`.
|
||||
*/
|
||||
function collapseNavigations(steps: DraftTraceStep[]): CollapsedDraft[] {
|
||||
const out: CollapsedDraft[] = [];
|
||||
steps.forEach((step, index) => {
|
||||
const draftId = index + 1;
|
||||
const prev = out[out.length - 1];
|
||||
if (step.op === "navigate" && prev?.op === "navigate") {
|
||||
out[out.length - 1] = step;
|
||||
continue;
|
||||
if (step.op === "navigate" && prev?.draft.op === "navigate") {
|
||||
prev.draft = {
|
||||
...prev.draft,
|
||||
url: step.url,
|
||||
postStateId: step.postStateId ?? prev.draft.postStateId,
|
||||
};
|
||||
prev.draftIds.push(draftId);
|
||||
return;
|
||||
}
|
||||
out.push(step);
|
||||
}
|
||||
out.push({ draft: step, draftIds: [draftId] });
|
||||
});
|
||||
return out;
|
||||
}
|
||||
|
||||
function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
||||
const urls: string[] = [];
|
||||
if (startUrl) urls.push(startUrl);
|
||||
for (const step of steps) {
|
||||
if (step.op === "navigate") {
|
||||
urls.push(step.url);
|
||||
continue;
|
||||
}
|
||||
if ("page_url" in step && step.page_url) urls.push(step.page_url);
|
||||
if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to);
|
||||
}
|
||||
const seen = new Set<string>();
|
||||
const unique: string[] = [];
|
||||
for (const url of urls) {
|
||||
if (!url || seen.has(url)) continue;
|
||||
seen.add(url);
|
||||
unique.push(url);
|
||||
}
|
||||
return unique;
|
||||
export interface StateRegistryEntry extends TraceState {
|
||||
contentHash: string;
|
||||
rawVomText: string;
|
||||
stepsHere: number[];
|
||||
}
|
||||
|
||||
function buildPageRegistry(
|
||||
steps: DraftTraceStep[],
|
||||
startUrl?: string,
|
||||
): { pages: PageRef[]; urlToId: Map<string, string> } {
|
||||
const urls = collectUrls(steps, startUrl);
|
||||
const urlToId = new Map<string, string>();
|
||||
const pages = urls.map((url, index) => {
|
||||
const id = `p${index + 1}`;
|
||||
urlToId.set(url, id);
|
||||
return { id, url };
|
||||
});
|
||||
return { pages, urlToId };
|
||||
export function stateDedupKey(body: string, url: string): string {
|
||||
return `${fnv1a64(body)}:${url}`;
|
||||
}
|
||||
|
||||
function pageIdFor(
|
||||
url: string | undefined,
|
||||
urlToId: Map<string, string>,
|
||||
fallbackUrl?: string,
|
||||
/** Register or reuse a page observation in the states dictionary. */
|
||||
export function registerObservation(
|
||||
registry: Map<string, StateRegistryEntry>,
|
||||
input: {
|
||||
url: string;
|
||||
title?: string;
|
||||
rawVomText: string;
|
||||
truncated?: boolean;
|
||||
},
|
||||
): string {
|
||||
if (url && urlToId.has(url)) return urlToId.get(url)!;
|
||||
if (fallbackUrl && urlToId.has(fallbackUrl)) return urlToId.get(fallbackUrl)!;
|
||||
return urlToId.values().next().value ?? "p1";
|
||||
const key = stateDedupKey(input.rawVomText, input.url);
|
||||
for (const entry of registry.values()) {
|
||||
if (entry.contentHash === key) return entry.id;
|
||||
}
|
||||
const id = nextStateId();
|
||||
registry.set(id, {
|
||||
id,
|
||||
url: input.url,
|
||||
...(input.title ? { title: input.title } : {}),
|
||||
truncated: input.truncated ?? false,
|
||||
contentHash: key,
|
||||
rawVomText: input.rawVomText,
|
||||
stepsHere: [],
|
||||
});
|
||||
return id;
|
||||
}
|
||||
|
||||
function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined {
|
||||
if (step.op === "navigate") return step.page_url ?? step.url;
|
||||
if ("page_url" in step && step.page_url) return step.page_url;
|
||||
return fallbackUrl;
|
||||
}
|
||||
|
||||
function effectForNavigation(
|
||||
navigatedTo: string | undefined,
|
||||
urlToId: Map<string, string>,
|
||||
): Step["effect"] {
|
||||
if (!navigatedTo) return undefined;
|
||||
const pageId = urlToId.get(navigatedTo);
|
||||
if (!pageId) return undefined;
|
||||
return { navigated_to: pageId };
|
||||
}
|
||||
|
||||
function withEffect(step: Step, effect: Step["effect"]): Step {
|
||||
if (!effect) return step;
|
||||
return { ...step, effect };
|
||||
}
|
||||
|
||||
function toSelection(values: string[], labels?: string[]): SelectedOption[] {
|
||||
function toSelection(
|
||||
values: string[],
|
||||
labels?: string[],
|
||||
): Array<{ value: string; label?: string }> {
|
||||
return values.map((value, index) => ({
|
||||
value,
|
||||
...(labels?.[index] ? { label: labels[index] } : {}),
|
||||
}));
|
||||
}
|
||||
|
||||
function toV2Step(
|
||||
step: DraftTraceStep,
|
||||
id: number,
|
||||
urlToId: Map<string, string>,
|
||||
fallbackUrl?: string,
|
||||
): Step | null {
|
||||
if (!shouldIncludeDraft(step)) return null;
|
||||
function navigateCauseForDraft(
|
||||
draft: Extract<DraftTraceStep, { op: "navigate" }>,
|
||||
): NavigationCause {
|
||||
if (draft.cause) return draft.cause;
|
||||
const mapped = mapNavigationCause({
|
||||
transitionType: draft.transitionType,
|
||||
transitionQualifiers: draft.transitionQualifiers,
|
||||
});
|
||||
return mapped ?? "browser";
|
||||
}
|
||||
|
||||
const pageUrl = pageUrlForDraft(step, fallbackUrl);
|
||||
const page = pageIdFor(pageUrl, urlToId, fallbackUrl);
|
||||
function toV3Step(draft: DraftTraceStep, id: number): Step | null {
|
||||
if (!shouldIncludeDraft(draft)) return null;
|
||||
// An action the user really performed must not disappear because one of its
|
||||
// observations is missing — a capture racing a document swap is routine, and
|
||||
// the last step of a recording is the most exposed to it. Fall back to the
|
||||
// endpoint we do know; only a step with no observation at all has nothing to
|
||||
// point at.
|
||||
const preState = draft.preStateId ?? draft.postStateId;
|
||||
const postState = draft.postStateId ?? draft.preStateId;
|
||||
if (!preState || !postState) return null;
|
||||
|
||||
switch (step.op) {
|
||||
const common = { id, state: preState, result: { state: postState } };
|
||||
|
||||
switch (draft.op) {
|
||||
case "navigate":
|
||||
return {
|
||||
op: "navigate",
|
||||
id,
|
||||
page: pageIdFor(step.url, urlToId, fallbackUrl),
|
||||
to: step.url,
|
||||
...common,
|
||||
to: draft.url,
|
||||
cause: navigateCauseForDraft(draft),
|
||||
};
|
||||
case "click":
|
||||
return withEffect(
|
||||
{
|
||||
op: "click",
|
||||
id,
|
||||
page,
|
||||
target: step.target,
|
||||
},
|
||||
effectForNavigation(step.navigated_to, urlToId),
|
||||
);
|
||||
return { op: "click", ...common, target: draft.target };
|
||||
case "hover":
|
||||
return {
|
||||
op: "hover",
|
||||
id,
|
||||
page,
|
||||
target: step.target,
|
||||
};
|
||||
return { op: "hover", ...common, target: draft.target };
|
||||
case "fill":
|
||||
return {
|
||||
op: "fill",
|
||||
id,
|
||||
page,
|
||||
target: step.target,
|
||||
value: step.value,
|
||||
...(step.redacted ? { redacted: true } : {}),
|
||||
...common,
|
||||
target: draft.target,
|
||||
value: draft.value,
|
||||
commit: draft.commit ?? DEFAULT_FILL_COMMIT,
|
||||
...(draft.redacted ? { redacted: true } : {}),
|
||||
};
|
||||
case "press":
|
||||
return withEffect(
|
||||
{
|
||||
op: "press",
|
||||
id,
|
||||
page,
|
||||
key: step.key,
|
||||
...(step.target ? { target: step.target } : {}),
|
||||
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
|
||||
},
|
||||
effectForNavigation(step.navigated_to, urlToId),
|
||||
);
|
||||
return {
|
||||
op: "press",
|
||||
...common,
|
||||
key: draft.key,
|
||||
...(draft.target ? { target: draft.target } : {}),
|
||||
...(draft.modifiers?.length ? { modifiers: draft.modifiers } : {}),
|
||||
};
|
||||
case "select":
|
||||
return withEffect(
|
||||
{
|
||||
op: "select",
|
||||
id,
|
||||
page,
|
||||
target: step.target,
|
||||
selection: toSelection(step.values, step.labels),
|
||||
},
|
||||
effectForNavigation(step.navigated_to, urlToId),
|
||||
);
|
||||
return {
|
||||
op: "select",
|
||||
...common,
|
||||
target: draft.target,
|
||||
selection: toSelection(draft.values, draft.labels),
|
||||
};
|
||||
case "scroll":
|
||||
return { op: "scroll", ...common };
|
||||
}
|
||||
}
|
||||
|
||||
export interface ReducedTrace {
|
||||
pages: PageRef[];
|
||||
states: TraceState[];
|
||||
steps: Step[];
|
||||
/**
|
||||
* Capture-order draft id (`draftIndex + 1`) → published step id. Collapsed
|
||||
* and dropped drafts make the two diverge, so anything the recorder tagged
|
||||
* with a draft id (`steps_here`, inline annotations) must be remapped.
|
||||
*/
|
||||
stepIdByDraftId: Map<number, number>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compile capture drafts into record-only trace v2 steps.
|
||||
* Variable inputs are NOT classified here — executing agents infer that at run time.
|
||||
* Compile capture drafts into record-only trace v3 steps.
|
||||
* Drafts must already carry `preStateId` / `postStateId` from the recorder.
|
||||
*/
|
||||
export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): ReducedTrace {
|
||||
export function reduceTraceSteps(
|
||||
steps: DraftTraceStep[],
|
||||
registry: Map<string, StateRegistryEntry>,
|
||||
): ReducedTrace {
|
||||
const collapsed = collapseNavigations(steps);
|
||||
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
|
||||
const out: Step[] = [];
|
||||
const stepIdByDraftId = new Map<number, number>();
|
||||
let id = 1;
|
||||
let lastUrl = startUrl;
|
||||
for (const draft of collapsed) {
|
||||
if (draft.op === "navigate") lastUrl = draft.url;
|
||||
else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to;
|
||||
else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url;
|
||||
const step = toV2Step(draft, id, urlToId, lastUrl);
|
||||
for (const { draft, draftIds } of collapsed) {
|
||||
const step = toV3Step(draft, id);
|
||||
if (!step) continue;
|
||||
out.push(step);
|
||||
for (const draftId of draftIds) stepIdByDraftId.set(draftId, id);
|
||||
id += 1;
|
||||
}
|
||||
return { pages, steps: out };
|
||||
const states = [...registry.values()].map(
|
||||
({ contentHash: _hash, rawVomText: _body, stepsHere: _steps, ...state }) => state,
|
||||
);
|
||||
return { states, steps: out, stepIdByDraftId };
|
||||
}
|
||||
|
||||
export function resolveTraceStartUrl(
|
||||
drafts: DraftTraceStep[],
|
||||
startUrl?: string,
|
||||
pages?: PageRef[],
|
||||
states?: TraceState[],
|
||||
): string {
|
||||
if (startUrl) return startUrl;
|
||||
const navigate = drafts.find((step): step is Extract<DraftTraceStep, { op: "navigate" }> => {
|
||||
@@ -221,5 +228,9 @@ export function resolveTraceStartUrl(
|
||||
for (const draft of drafts) {
|
||||
if ("page_url" in draft && draft.page_url) return draft.page_url;
|
||||
}
|
||||
return pages?.[0]?.url ?? "about:blank";
|
||||
return states?.[0]?.url ?? "about:blank";
|
||||
}
|
||||
|
||||
export type { FillCommit, NavigationCause };
|
||||
/** Test seam */
|
||||
export { resetStateIdCounterForTests };
|
||||
|
||||
@@ -1034,7 +1034,7 @@ describe("buildVomScene", () => {
|
||||
expect(rendered).toContain(' @e2 textbox "验证码"');
|
||||
});
|
||||
|
||||
it("does not mark non-password inputs sensitive from password-like labels", () => {
|
||||
it("marks current-password autocomplete as sensitive even for text inputs", () => {
|
||||
const axNodes: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "1",
|
||||
@@ -1069,7 +1069,7 @@ describe("buildVomScene", () => {
|
||||
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
id: 10,
|
||||
sensitive: false,
|
||||
sensitive: true,
|
||||
}),
|
||||
);
|
||||
});
|
||||
@@ -1364,7 +1364,9 @@ describe("buildVomScene", () => {
|
||||
);
|
||||
const rendered = renderVom(scene);
|
||||
expect(rendered.text).toContain('@e1 button "close"');
|
||||
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
|
||||
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
|
||||
{ ref: "e1", backendNodeId: 20 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not promote a clickable container that wraps a real interactive control", () => {
|
||||
@@ -1508,7 +1510,9 @@ describe("buildVomScene", () => {
|
||||
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic");
|
||||
const rendered = renderVom(scene);
|
||||
expect(rendered.text).toContain('@e1 button "收藏"');
|
||||
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
|
||||
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
|
||||
{ ref: "e1", backendNodeId: 20 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("builds active scope blocks from active aria-controls relationships", () => {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,101 @@
|
||||
import { type RenderedRef, renderVom, type VomOptions } from "@browser-skill/vom";
|
||||
import type { CdpAxNode } from "./observation";
|
||||
import { buildVomScene } from "./observation";
|
||||
import type { CdpRunner } from "./shared";
|
||||
import {
|
||||
type CapturedNode,
|
||||
type CapturedSurfaceProbe,
|
||||
type CapturedViewModel,
|
||||
captureViewModel,
|
||||
collectOverlayExcludedBackendIds,
|
||||
} from "./vom/capture";
|
||||
|
||||
export interface CaptureVomObservationOptions extends VomOptions {
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface CaptureVomObservationResult {
|
||||
text: string;
|
||||
refs: RenderedRef[];
|
||||
truncated: boolean;
|
||||
captured: CapturedNode[];
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
}
|
||||
|
||||
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
|
||||
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
|
||||
}
|
||||
|
||||
interface LayoutMetricsViewportReply {
|
||||
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
}
|
||||
|
||||
async function fallbackCapturedViewModel(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
): Promise<CapturedViewModel> {
|
||||
let viewport = { width: 0, height: 0 };
|
||||
try {
|
||||
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
viewport = {
|
||||
width: vpSrc.clientWidth ?? 0,
|
||||
height: vpSrc.clientHeight ?? 0,
|
||||
};
|
||||
} catch {
|
||||
// viewport stays zero-sized
|
||||
}
|
||||
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId);
|
||||
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
async function captureForVom(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
conditionalSurfaceProbe: boolean,
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>,
|
||||
): Promise<CapturedViewModel> {
|
||||
try {
|
||||
return await captureViewModel(cdp, tabId, {
|
||||
conditionalSurfaceProbe,
|
||||
hoverProbeBypassOverlay,
|
||||
});
|
||||
} catch {
|
||||
return fallbackCapturedViewModel(cdp, tabId);
|
||||
}
|
||||
}
|
||||
|
||||
/** Pure VOM capture pipeline shared by snapshot, observe, and recording. */
|
||||
export async function captureVomObservation(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
url: string | undefined,
|
||||
opts: CaptureVomObservationOptions = {},
|
||||
): Promise<CaptureVomObservationResult> {
|
||||
await cdp.ensureAttachedToUrl?.(tabId, url);
|
||||
await cdp.send<unknown>(tabId, "Accessibility.enable", {});
|
||||
const result = await cdp.send<{ nodes: CdpAxNode[] }>(tabId, "Accessibility.getFullAXTree", {});
|
||||
const axNodes = result.nodes ?? [];
|
||||
const captured = await captureForVom(
|
||||
cdp,
|
||||
tabId,
|
||||
opts.conditionalSurfaceProbe ?? false,
|
||||
opts.hoverProbeBypassOverlay,
|
||||
);
|
||||
const scene = buildVomScene(axNodes, captured, { pageUrl: url });
|
||||
const rendered = renderVom(scene, {
|
||||
maxDepth: opts.maxDepth,
|
||||
maxTokens: opts.maxTokens,
|
||||
redactValues: opts.redactValues,
|
||||
activeRegionPolicy: opts.activeRegionPolicy ?? true,
|
||||
});
|
||||
return {
|
||||
text: rendered.text,
|
||||
refs: rendered.refs,
|
||||
truncated: rendered.truncated,
|
||||
captured: captured.nodes,
|
||||
surfaceProbes: captured.surfaceProbes,
|
||||
};
|
||||
}
|
||||
@@ -8,7 +8,6 @@ import {
|
||||
applyVomInteractionRecovery,
|
||||
type CondSurface,
|
||||
isVomReferenceNode,
|
||||
renderVom,
|
||||
type VomNode,
|
||||
type VomScene,
|
||||
} from "@browser-skill/vom";
|
||||
@@ -29,6 +28,7 @@ import type {
|
||||
SnapshotParams,
|
||||
SnapshotResult,
|
||||
} from "@/transport/types";
|
||||
import { captureVomObservation } from "./capture-vom-observation";
|
||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||
import { nodeBoundingRect, scrollNodeIntoView } from "./element-geometry";
|
||||
import { rpcError } from "./errors";
|
||||
@@ -44,12 +44,7 @@ import {
|
||||
type ToolEffect,
|
||||
} from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import {
|
||||
type CapturedNode,
|
||||
type CapturedViewModel,
|
||||
captureViewModel,
|
||||
collectOverlayExcludedBackendIds,
|
||||
} from "./vom/capture";
|
||||
import { type CapturedNode, type CapturedViewModel } from "./vom/capture";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared helpers (legacy aliases — observation.ts kept exporting these
|
||||
@@ -391,7 +386,18 @@ function isModalSignal(
|
||||
|
||||
function isSensitive(capturedNode: CapturedNode | undefined): boolean {
|
||||
const attrs = capturedNode?.attrs ?? {};
|
||||
return (attrs.type ?? "").toLowerCase() === "password";
|
||||
const type = (attrs.type ?? "").toLowerCase();
|
||||
if (type === "password") return true;
|
||||
const autocomplete = (attrs.autocomplete ?? "").toLowerCase();
|
||||
if (autocomplete.startsWith("cc-")) return true;
|
||||
if (
|
||||
autocomplete === "one-time-code" ||
|
||||
autocomplete === "current-password" ||
|
||||
autocomplete === "new-password"
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
interface AxNodeSignals {
|
||||
@@ -441,7 +447,13 @@ const AX_TEXT_STOP_ROLES = new Set([
|
||||
"columnheader",
|
||||
"rowheader",
|
||||
]);
|
||||
const SENSITIVE_AX_INPUT_TYPES = new Set(["password", "credit-card"]);
|
||||
const SENSITIVE_AX_INPUT_TYPES = new Set([
|
||||
"password",
|
||||
"credit-card",
|
||||
"one-time-code",
|
||||
"current-password",
|
||||
"new-password",
|
||||
]);
|
||||
|
||||
function axPropertyString(axNode: CdpAxNode, name: string): string | undefined {
|
||||
const prop = axNode.properties?.find((item) => item.name === name);
|
||||
@@ -1364,58 +1376,6 @@ export async function handleGetHtml(
|
||||
}
|
||||
}
|
||||
|
||||
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
|
||||
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
|
||||
}
|
||||
|
||||
interface LayoutMetricsViewportReply {
|
||||
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
}
|
||||
|
||||
async function fallbackCapturedViewModel(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CapturedViewModel> {
|
||||
throwIfAborted(signal, "observation");
|
||||
let viewport = { width: 0, height: 0 };
|
||||
try {
|
||||
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
throwIfAborted(signal, "observation");
|
||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
viewport = {
|
||||
width: vpSrc.clientWidth ?? 0,
|
||||
height: vpSrc.clientHeight ?? 0,
|
||||
};
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
// viewport stays zero-sized
|
||||
}
|
||||
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId, signal);
|
||||
throwIfAborted(signal, "observation");
|
||||
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
async function captureForVom(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
conditionalSurfaceProbe: boolean,
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CapturedViewModel> {
|
||||
try {
|
||||
return await captureViewModel(cdp, tabId, {
|
||||
conditionalSurfaceProbe,
|
||||
hoverProbeBypassOverlay,
|
||||
signal,
|
||||
});
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
return fallbackCapturedViewModel(cdp, tabId, signal);
|
||||
}
|
||||
}
|
||||
|
||||
async function handleVomObservation(
|
||||
manager: SessionManager,
|
||||
params: SnapshotParams | ObserveParams,
|
||||
@@ -1445,32 +1405,14 @@ async function handleVomObservation(
|
||||
try {
|
||||
throwIfAborted(signal, toolName);
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||
throwIfAborted(signal, toolName);
|
||||
await deps.cdp.send<unknown>(target.tabId, "Accessibility.enable", {});
|
||||
throwIfAborted(signal, toolName);
|
||||
const result = await deps.cdp.send<{ nodes: CdpAxNode[] }>(
|
||||
target.tabId,
|
||||
"Accessibility.getFullAXTree",
|
||||
{},
|
||||
);
|
||||
throwIfAborted(signal, toolName);
|
||||
const axNodes = result.nodes ?? [];
|
||||
const effectiveConditionalSurfaceProbe =
|
||||
deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe;
|
||||
const captured = await captureForVom(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
effectiveConditionalSurfaceProbe,
|
||||
deps.hoverProbeBypassOverlay,
|
||||
signal,
|
||||
);
|
||||
throwIfAborted(signal, toolName);
|
||||
const scene = buildVomScene(axNodes, captured, { pageUrl: target.url });
|
||||
const rendered = renderVom(scene, {
|
||||
const rendered = await captureVomObservation(deps.cdp, target.tabId, target.url, {
|
||||
maxDepth: params.max_depth,
|
||||
maxTokens: params.max_tokens,
|
||||
activeRegionPolicy: true,
|
||||
conditionalSurfaceProbe: effectiveConditionalSurfaceProbe,
|
||||
hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay,
|
||||
});
|
||||
throwIfAborted(signal, toolName);
|
||||
ctx.refStore.replace(
|
||||
@@ -1486,7 +1428,7 @@ async function handleVomObservation(
|
||||
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
|
||||
? {
|
||||
debug: {
|
||||
surface_probes: (captured.surfaceProbes ?? []).map((probe) => ({
|
||||
surface_probes: (rendered.surfaceProbes ?? []).map((probe) => ({
|
||||
trigger_backend_node_id: probe.triggerBackendNodeId,
|
||||
...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}),
|
||||
trigger_action: probe.triggerAction,
|
||||
|
||||
@@ -17,20 +17,40 @@ import {
|
||||
type RecordStartMessage,
|
||||
type RecordStopMessage,
|
||||
} from "@/lib/record-bridge";
|
||||
import { DEFAULT_MAX_PAGE_TOKENS } from "@/lib/record-constants";
|
||||
import {
|
||||
applyTargetMatching,
|
||||
buildTraceV3,
|
||||
cancelPendingSettles,
|
||||
captureAndRegisterObservation,
|
||||
clearPendingRedirectLanding,
|
||||
createObservationState,
|
||||
flushPendingRedirectLanding,
|
||||
flushPendingSettles,
|
||||
inferMissingPostStates,
|
||||
type RecordingObservationState,
|
||||
rememberStepOnPage,
|
||||
scheduleDraftSettle,
|
||||
scheduleRedirectLandingFlush,
|
||||
settleUnsettledDrafts,
|
||||
} from "@/lib/record-observation";
|
||||
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording-step-buffer";
|
||||
import { reduceTraceSteps, resolveTraceStartUrl } from "@/lib/trace-reducer";
|
||||
import { buildTraceV2 } from "@/lib/trace-reducer-v2";
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import { EXTENSION_VERSION } from "@/transport/handshake";
|
||||
import type {
|
||||
DraftTraceStep,
|
||||
RecordAwaitParams,
|
||||
RecordAwaitResult,
|
||||
RecordedTrace,
|
||||
RecordStartParams,
|
||||
RecordStartResult,
|
||||
RecordStopParams,
|
||||
RecordStopResult,
|
||||
RpcError,
|
||||
Trace,
|
||||
StopReason,
|
||||
} from "@/transport/types";
|
||||
import { TRACE_VERSION } from "@/transport/types";
|
||||
import { handleNavigate } from "./navigation";
|
||||
import {
|
||||
type CdpRunner,
|
||||
@@ -50,14 +70,42 @@ interface ActiveRecording {
|
||||
steps: DraftTraceStep[];
|
||||
startedAt: string;
|
||||
startedAtMs: number;
|
||||
finishPromise: Promise<Trace>;
|
||||
resolveFinish: (trace: Trace) => void;
|
||||
traceVersion: 2 | 3;
|
||||
finishPromise: Promise<RecordedTrace>;
|
||||
resolveFinish: (trace: RecordedTrace) => void;
|
||||
rejectFinish: (err: Error) => void;
|
||||
settled: boolean;
|
||||
finishing: boolean;
|
||||
currentUrl?: string;
|
||||
pendingNavigation: boolean;
|
||||
pendingNavigationDeadline?: number;
|
||||
observation: RecordingObservationState;
|
||||
stoppedBy: StopReason;
|
||||
maxPageTokens: number;
|
||||
redactValues: boolean;
|
||||
/** Tabs whose content capture acknowledged START and still need STOP. */
|
||||
armedTabIds: Set<number>;
|
||||
/** Rearms already sending START when finish begins. */
|
||||
rearmCallbacks: Set<Promise<void>>;
|
||||
/** Navigation callbacks tracked from event receipt through action enqueue. */
|
||||
navigationCallbacks: Set<Promise<void>>;
|
||||
/** Synchronous intake gate closed only after finish drains to stability. */
|
||||
acceptingNavigation: boolean;
|
||||
/**
|
||||
* Serializes step appends with navigation observation so a click is always
|
||||
* in `steps` before a same-turn `webNavigation` tries to annotate it.
|
||||
*/
|
||||
actionQueue: Promise<void>;
|
||||
/**
|
||||
* CDP runner captured at `record_start`. The message listeners are attached
|
||||
* once at service-worker startup, where no runner exists yet, so observation
|
||||
* capture must go through the recording instead of the listener deps.
|
||||
*/
|
||||
cdp?: CdpRunner;
|
||||
}
|
||||
|
||||
function enqueueRecordingAction(recording: ActiveRecording, task: () => Promise<void>): void {
|
||||
recording.actionQueue = recording.actionQueue.then(task, task).catch(() => {});
|
||||
}
|
||||
|
||||
const recordings = new Map<string, ActiveRecording>();
|
||||
@@ -78,6 +126,9 @@ function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/** Recording producer version mirrored into trace.recorder.bsk. */
|
||||
export const BSK_TRACE_VERSION = EXTENSION_VERSION;
|
||||
|
||||
/** Injectable http(s) landing page when `tool.record_start` omits `url`. */
|
||||
export const RECORD_DEFAULT_START_URL = "https://example.com/";
|
||||
|
||||
@@ -154,19 +205,64 @@ async function sendRecordStartWithAck(
|
||||
throw lastError ?? new Error("failed to start recording in content script");
|
||||
}
|
||||
|
||||
function buildTrace(recording: ActiveRecording): Trace {
|
||||
const { pages, steps } = reduceTraceSteps(recording.steps, recording.startUrl);
|
||||
const startUrl = resolveTraceStartUrl(recording.steps, recording.startUrl, pages);
|
||||
function negotiatedTraceVersion(params: RecordStartParams): 2 | 3 | RpcError {
|
||||
if (params.trace_version === undefined) return 2;
|
||||
if (params.trace_version === TRACE_VERSION) return 3;
|
||||
return {
|
||||
...(recording.purpose ? { purpose: recording.purpose } : {}),
|
||||
recorded_at: new Date().toISOString(),
|
||||
started_at: recording.startedAt,
|
||||
entry: { start_url: startUrl },
|
||||
pages,
|
||||
steps,
|
||||
code: "invalid_params",
|
||||
message: `unsupported trace_version ${params.trace_version}; supported values are omitted (v2) or ${TRACE_VERSION} (v3)`,
|
||||
};
|
||||
}
|
||||
|
||||
function buildTrace(recording: ActiveRecording): RecordedTrace {
|
||||
if (recording.traceVersion === 3) {
|
||||
return buildTraceV3({
|
||||
obs: recording.observation,
|
||||
steps: recording.steps,
|
||||
startedAt: recording.startedAt,
|
||||
...(recording.purpose ? { purpose: recording.purpose } : {}),
|
||||
startUrl: recording.startUrl,
|
||||
stoppedBy: recording.stoppedBy,
|
||||
bskVersion: BSK_TRACE_VERSION,
|
||||
});
|
||||
}
|
||||
return buildTraceV2({
|
||||
steps: recording.steps,
|
||||
startedAt: recording.startedAt,
|
||||
...(recording.startUrl ? { startUrl: recording.startUrl } : {}),
|
||||
...(recording.purpose ? { purpose: recording.purpose } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
async function processRecordedStep(
|
||||
recording: ActiveRecording,
|
||||
deps: RecordDeps,
|
||||
draftIndex: number,
|
||||
): Promise<void> {
|
||||
const cdp = recording.cdp ?? deps.cdp;
|
||||
if (!cdp) return;
|
||||
const draft = recording.steps[draftIndex];
|
||||
if (!draft) return;
|
||||
const stepId = draftIndex + 1;
|
||||
|
||||
if (draft.op === "navigate") {
|
||||
draft.preStateId = recording.observation.lastSettled?.stateId;
|
||||
} else {
|
||||
applyTargetMatching(recording.observation, draft, stepId);
|
||||
}
|
||||
rememberStepOnPage(recording.observation, draft, stepId);
|
||||
|
||||
scheduleDraftSettle(
|
||||
recording.observation,
|
||||
draftIndex,
|
||||
stepId,
|
||||
cdp,
|
||||
recording.tabId,
|
||||
deps.tabsApi,
|
||||
recording.steps,
|
||||
);
|
||||
}
|
||||
|
||||
export interface RecordDeps {
|
||||
tabsApi: ChromeTabsApi;
|
||||
sendToTab(
|
||||
@@ -241,7 +337,7 @@ export function releaseBrowserObservationListenersIfIdle(): void {
|
||||
detachBrowserObservation = null;
|
||||
}
|
||||
|
||||
export function attachRecordStepListener(): () => void {
|
||||
export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||
const listener = (
|
||||
message: unknown,
|
||||
_sender: chrome.runtime.MessageSender,
|
||||
@@ -250,7 +346,22 @@ export function attachRecordStepListener(): () => void {
|
||||
if (!isRecordStepMessage(message)) return false;
|
||||
for (const recording of recordings.values()) {
|
||||
if (recording.requestId !== message.requestId) continue;
|
||||
appendRecordedPayload(recording, message.step);
|
||||
enqueueRecordingAction(recording, async () => {
|
||||
const cdp = recording.cdp ?? deps.cdp;
|
||||
// Flush before appending so a coalesced OAuth navigate is ordered
|
||||
// ahead of the click that landed on the destination page. The flush
|
||||
// itself waits for that navigate's observation before returning.
|
||||
await flushPendingRedirectLanding(
|
||||
recording.observation,
|
||||
recording.steps,
|
||||
cdp,
|
||||
recording.tabId,
|
||||
deps.tabsApi,
|
||||
{ cancelled: () => recording.settled },
|
||||
);
|
||||
const draftIndex = appendRecordedPayload(recording, message.step);
|
||||
if (draftIndex !== null) await processRecordedStep(recording, deps, draftIndex);
|
||||
});
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
@@ -277,7 +388,9 @@ export function attachRecordFinishListener(deps: RecordDeps = getDefaultDeps()):
|
||||
|
||||
function findRecordingByTabId(tabId: number): ActiveRecording | null {
|
||||
for (const recording of recordings.values()) {
|
||||
if (recording.tabId === tabId && !recording.settled) return recording;
|
||||
if (!recording.settled && (recording.tabId === tabId || recording.armedTabIds.has(tabId))) {
|
||||
return recording;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -330,29 +443,24 @@ async function stopRecordingOnAllAgentTabs(
|
||||
deps: RecordDeps,
|
||||
): Promise<void> {
|
||||
const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
|
||||
let tabIds = [recording.tabId];
|
||||
try {
|
||||
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
|
||||
tabIds = [
|
||||
...new Set([
|
||||
recording.tabId,
|
||||
...tabs.flatMap((tab) => (typeof tab.id === "number" ? [tab.id] : [])),
|
||||
]),
|
||||
];
|
||||
} catch {
|
||||
// Fall back to the current recording tab.
|
||||
}
|
||||
|
||||
for (const tabId of tabIds) {
|
||||
let failed = false;
|
||||
for (const tabId of [...recording.armedTabIds]) {
|
||||
try {
|
||||
await deps.tabsApi.get(tabId);
|
||||
} catch {
|
||||
// A closed tab no longer has content capture to flush.
|
||||
recording.armedTabIds.delete(tabId);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const response = await deps.sendToTab(tabId, stopMsg);
|
||||
if (tabId === recording.tabId && !isRecordStartAck(response)) {
|
||||
if (!isRecordStartAck(response)) {
|
||||
throw new Error("content script did not confirm recorded steps");
|
||||
}
|
||||
recording.armedTabIds.delete(tabId);
|
||||
} catch {
|
||||
if (tabId === recording.tabId) {
|
||||
throw new Error("failed to flush recorded steps");
|
||||
}
|
||||
failed = true;
|
||||
continue;
|
||||
}
|
||||
if (deps.bypassOverlay) {
|
||||
try {
|
||||
@@ -362,6 +470,9 @@ async function stopRecordingOnAllAgentTabs(
|
||||
}
|
||||
}
|
||||
}
|
||||
if (failed || recording.armedTabIds.size > 0) {
|
||||
throw new Error("failed to flush recorded steps");
|
||||
}
|
||||
}
|
||||
|
||||
async function rearmRecording(
|
||||
@@ -369,27 +480,40 @@ async function rearmRecording(
|
||||
targetTabId: number,
|
||||
deps: RecordDeps,
|
||||
): Promise<boolean> {
|
||||
let resolveTracked!: () => void;
|
||||
const tracked = new Promise<void>((resolve) => {
|
||||
resolveTracked = resolve;
|
||||
});
|
||||
recording.rearmCallbacks.add(tracked);
|
||||
|
||||
// Do NOT toggle automation-bypass here: each retry used to increment the
|
||||
// content-script counter, and a single stop decrement left the ControlOverlay
|
||||
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
|
||||
// already hides the control chrome while activeRecord is set.
|
||||
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
|
||||
const startMsg: RecordStartMessage = {
|
||||
type: RECORD_START,
|
||||
requestId: recording.requestId,
|
||||
startedAtMs: recording.startedAtMs,
|
||||
};
|
||||
try {
|
||||
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
|
||||
recording.tabId = targetTabId;
|
||||
return true;
|
||||
} catch {
|
||||
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
|
||||
await sleep(RECORD_REARM_RETRY_DELAY_MS);
|
||||
try {
|
||||
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
|
||||
if (recording.settled || recording.finishing) return false;
|
||||
const startMsg: RecordStartMessage = {
|
||||
type: RECORD_START,
|
||||
requestId: recording.requestId,
|
||||
startedAtMs: recording.startedAtMs,
|
||||
};
|
||||
try {
|
||||
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
|
||||
recording.armedTabIds.add(targetTabId);
|
||||
recording.tabId = targetTabId;
|
||||
return true;
|
||||
} catch {
|
||||
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
|
||||
await sleep(RECORD_REARM_RETRY_DELAY_MS);
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
} finally {
|
||||
recording.rearmCallbacks.delete(tracked);
|
||||
resolveTracked();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
|
||||
@@ -432,11 +556,86 @@ export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): ()
|
||||
}
|
||||
|
||||
export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||
const observeMainFrame = (tabId: number, url?: string, causedByAction?: boolean) => {
|
||||
const observeMainFrame = (
|
||||
tabId: number,
|
||||
url?: string,
|
||||
causedByAction?: boolean,
|
||||
transitionType?: string,
|
||||
transitionQualifiers?: string[],
|
||||
) => {
|
||||
if (!url) return;
|
||||
const direct = findRecordingByTabId(tabId);
|
||||
const candidates = direct
|
||||
? direct.acceptingNavigation
|
||||
? [direct]
|
||||
: []
|
||||
: [...recordings.values()].filter(
|
||||
(recording) => !recording.settled && recording.acceptingNavigation,
|
||||
);
|
||||
if (candidates.length === 0) return;
|
||||
|
||||
let resolveTracked!: () => void;
|
||||
const tracked = new Promise<void>((resolve) => {
|
||||
resolveTracked = resolve;
|
||||
});
|
||||
for (const candidate of candidates) candidate.navigationCallbacks.add(tracked);
|
||||
|
||||
void (async () => {
|
||||
const recording = await findRecordingForTab(tabId, deps);
|
||||
if (recording && url) {
|
||||
observeRecordedNavigation(recording, url, causedByAction);
|
||||
try {
|
||||
const recording = await findRecordingForTab(tabId, deps);
|
||||
if (!recording || !recording.acceptingNavigation || !candidates.includes(recording)) {
|
||||
return;
|
||||
}
|
||||
enqueueRecordingAction(recording, async () => {
|
||||
const result = observeRecordedNavigation(
|
||||
recording,
|
||||
url,
|
||||
causedByAction,
|
||||
transitionType,
|
||||
transitionQualifiers,
|
||||
);
|
||||
const cdp = recording.cdp ?? deps.cdp;
|
||||
if (!cdp) return;
|
||||
|
||||
if (result.kind === "coalesce_redirect") {
|
||||
scheduleRedirectLandingFlush(
|
||||
recording.observation,
|
||||
recording.steps,
|
||||
cdp,
|
||||
tabId,
|
||||
deps.tabsApi,
|
||||
result.url,
|
||||
{ cancelled: () => recording.settled },
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (result.kind === "noop") return;
|
||||
|
||||
// A concrete navigation supersedes any in-flight redirect coalesce.
|
||||
clearPendingRedirectLanding(recording.observation);
|
||||
|
||||
const draftIndex = result.index;
|
||||
const lastDraft = recording.steps[draftIndex];
|
||||
if (!lastDraft) return;
|
||||
|
||||
if (result.kind === "appended") {
|
||||
lastDraft.preStateId = recording.observation.lastSettled?.stateId;
|
||||
rememberStepOnPage(recording.observation, lastDraft, draftIndex + 1);
|
||||
}
|
||||
scheduleDraftSettle(
|
||||
recording.observation,
|
||||
draftIndex,
|
||||
draftIndex + 1,
|
||||
cdp,
|
||||
tabId,
|
||||
deps.tabsApi,
|
||||
recording.steps,
|
||||
);
|
||||
});
|
||||
} finally {
|
||||
for (const candidate of candidates) candidate.navigationCallbacks.delete(tracked);
|
||||
resolveTracked();
|
||||
}
|
||||
})();
|
||||
};
|
||||
@@ -458,18 +657,13 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
|
||||
details: chrome.webNavigation.WebNavigationTransitionCallbackDetails,
|
||||
) => {
|
||||
if (details.frameId !== 0) return;
|
||||
const causedByAction =
|
||||
details.transitionType === "link" || details.transitionType === "form_submit"
|
||||
? true
|
||||
: details.transitionType === "typed" ||
|
||||
details.transitionType === "auto_bookmark" ||
|
||||
details.transitionType === "generated" ||
|
||||
details.transitionType === "keyword" ||
|
||||
details.transitionType === "keyword_generated" ||
|
||||
details.transitionType === "reload"
|
||||
? false
|
||||
: undefined;
|
||||
observeMainFrame(details.tabId, details.url, causedByAction);
|
||||
observeMainFrame(
|
||||
details.tabId,
|
||||
details.url,
|
||||
undefined,
|
||||
details.transitionType,
|
||||
details.transitionQualifiers,
|
||||
);
|
||||
};
|
||||
chrome.webNavigation.onCompleted.addListener(completedListener);
|
||||
chrome.webNavigation.onCommitted?.addListener(committedListener);
|
||||
@@ -487,6 +681,46 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
|
||||
return () => chrome.tabs.onUpdated.removeListener(listener);
|
||||
}
|
||||
|
||||
const MAX_FINISH_DRAIN_ROUNDS = 10;
|
||||
|
||||
async function drainRecordingToStability(
|
||||
recording: ActiveRecording,
|
||||
deps: RecordDeps,
|
||||
): Promise<boolean> {
|
||||
const cdp = recording.cdp ?? deps.cdp;
|
||||
for (let round = 0; round < MAX_FINISH_DRAIN_ROUNDS; round += 1) {
|
||||
await Promise.all([...recording.navigationCallbacks]);
|
||||
const actionTail = recording.actionQueue;
|
||||
await actionTail;
|
||||
await flushPendingRedirectLanding(
|
||||
recording.observation,
|
||||
recording.steps,
|
||||
cdp,
|
||||
recording.tabId,
|
||||
deps.tabsApi,
|
||||
{ cancelled: () => recording.settled },
|
||||
);
|
||||
await flushPendingSettles(recording.observation);
|
||||
const redirectTail = recording.observation.redirectFlushQueue;
|
||||
const settleTail = recording.observation.settleQueue;
|
||||
await Promise.all([redirectTail, settleTail]);
|
||||
|
||||
if (
|
||||
recording.navigationCallbacks.size === 0 &&
|
||||
recording.actionQueue === actionTail &&
|
||||
recording.observation.redirectFlushQueue === redirectTail &&
|
||||
recording.observation.settleQueue === settleTail
|
||||
) {
|
||||
// No event or promise continuation can interleave with this synchronous
|
||||
// check-and-close, so work accepted before the cutoff is fully drained.
|
||||
recording.acceptingNavigation = false;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
console.warn("[bsk record] navigation/action queues did not stabilize at stop");
|
||||
return false;
|
||||
}
|
||||
|
||||
export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||
const listener = (
|
||||
message: unknown,
|
||||
@@ -527,23 +761,49 @@ async function finishRecordingByRequest(
|
||||
if (recording.requestId !== requestId || recording.settled) continue;
|
||||
const match = await findRecordingForTab(tabId, deps);
|
||||
if (match !== recording) continue;
|
||||
await finishRecording(sessionId, deps);
|
||||
await finishRecording(sessionId, deps, "user_finish");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
async function finishRecording(sessionId: string, deps: RecordDeps): Promise<Trace | null> {
|
||||
async function finishRecording(
|
||||
sessionId: string,
|
||||
deps: RecordDeps,
|
||||
stoppedBy: StopReason,
|
||||
): Promise<RecordedTrace | null> {
|
||||
const recording = recordings.get(sessionId);
|
||||
if (!recording || recording.settled || recording.finishing) return null;
|
||||
if (!recording || recording.settled) return null;
|
||||
if (recording.finishing) return recording.finishPromise;
|
||||
recording.finishing = true;
|
||||
recording.stoppedBy = stoppedBy;
|
||||
|
||||
await clearRearmTimersForRecording(recording, deps);
|
||||
await Promise.all([...recording.rearmCallbacks]);
|
||||
try {
|
||||
// Disposing content capture may commit a final dirty fill. Flush content
|
||||
// first so all resulting RECORD_STEP messages enter actionQueue before it
|
||||
// and the observation queues are drained.
|
||||
await stopRecordingOnAllAgentTabs(recording, deps);
|
||||
} catch {
|
||||
recording.finishing = false;
|
||||
return null;
|
||||
}
|
||||
if (!(await drainRecordingToStability(recording, deps))) {
|
||||
recording.finishing = false;
|
||||
return null;
|
||||
}
|
||||
cancelPendingSettles(recording.observation);
|
||||
inferMissingPostStates(recording.steps);
|
||||
const cdp = recording.cdp ?? deps.cdp;
|
||||
if (cdp) {
|
||||
await settleUnsettledDrafts(
|
||||
recording.observation,
|
||||
cdp,
|
||||
recording.tabId,
|
||||
deps.tabsApi,
|
||||
recording.steps,
|
||||
);
|
||||
}
|
||||
|
||||
recording.settled = true;
|
||||
recordings.delete(sessionId);
|
||||
@@ -558,6 +818,9 @@ export async function handleRecordStart(
|
||||
params: RecordStartParams,
|
||||
deps: RecordDeps = getDefaultDeps(),
|
||||
): Promise<RecordStartResult | RpcError> {
|
||||
const traceVersionOrErr = negotiatedTraceVersion(params);
|
||||
if (typeof traceVersionOrErr !== "number") return traceVersionOrErr;
|
||||
|
||||
const ctxOrErr = lookupSession(manager, params, "record_start");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
@@ -574,14 +837,16 @@ export async function handleRecordStart(
|
||||
// on the destination page can RECORD_QUERY → rearm → show RecordOverlay
|
||||
// instead of flashing ControlOverlay ("Agent 正在控制").
|
||||
const requestId = makeRequestId(target.tabId);
|
||||
let resolveFinish!: (trace: Trace) => void;
|
||||
let resolveFinish!: (trace: RecordedTrace) => void;
|
||||
let rejectFinish!: (err: Error) => void;
|
||||
const finishPromise = new Promise<Trace>((resolve, reject) => {
|
||||
const finishPromise = new Promise<RecordedTrace>((resolve, reject) => {
|
||||
resolveFinish = resolve;
|
||||
rejectFinish = reject;
|
||||
});
|
||||
const navigateUrl = params.url ?? RECORD_DEFAULT_START_URL;
|
||||
const startedAtMs = Date.now();
|
||||
const maxPageTokens = params.max_page_tokens;
|
||||
const redactValues = params.redact_values ?? false;
|
||||
recordings.set(params.session_id, {
|
||||
requestId,
|
||||
tabId: target.tabId,
|
||||
@@ -591,6 +856,7 @@ export async function handleRecordStart(
|
||||
steps: [],
|
||||
startedAt: new Date(startedAtMs).toISOString(),
|
||||
startedAtMs,
|
||||
traceVersion: traceVersionOrErr,
|
||||
finishPromise,
|
||||
resolveFinish,
|
||||
rejectFinish,
|
||||
@@ -599,6 +865,16 @@ export async function handleRecordStart(
|
||||
currentUrl: navigateUrl,
|
||||
pendingNavigation: false,
|
||||
pendingNavigationDeadline: undefined,
|
||||
observation: createObservationState({ maxPageTokens, redactValues }),
|
||||
stoppedBy: "user_finish",
|
||||
maxPageTokens: maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS,
|
||||
redactValues,
|
||||
armedTabIds: new Set(),
|
||||
rearmCallbacks: new Set(),
|
||||
navigationCallbacks: new Set(),
|
||||
acceptingNavigation: true,
|
||||
actionQueue: Promise.resolve(),
|
||||
...(deps.cdp ? { cdp: deps.cdp } : {}),
|
||||
});
|
||||
// Observe navigations for the whole recording lifetime; attach before
|
||||
// optional navigate so the destination load can rearm capture.
|
||||
@@ -722,6 +998,7 @@ export async function handleRecordStart(
|
||||
|
||||
try {
|
||||
await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
|
||||
active.armedTabIds.add(target.tabId);
|
||||
} catch {
|
||||
await abortPending(true);
|
||||
return {
|
||||
@@ -731,6 +1008,23 @@ export async function handleRecordStart(
|
||||
};
|
||||
}
|
||||
|
||||
if (deps.cdp) {
|
||||
const activeRecording = recordings.get(params.session_id);
|
||||
if (activeRecording) {
|
||||
try {
|
||||
await captureAndRegisterObservation(
|
||||
activeRecording.observation,
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
deps.tabsApi,
|
||||
startUrl,
|
||||
);
|
||||
} catch {
|
||||
// Proceed without initial observation; steps may be dropped by reducer.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
const cancelled = await abortIfCancelled(true);
|
||||
if (cancelled) return cancelled;
|
||||
@@ -755,7 +1049,7 @@ export async function handleRecordStop(
|
||||
};
|
||||
}
|
||||
|
||||
const trace = await finishRecording(params.session_id, deps);
|
||||
const trace = await finishRecording(params.session_id, deps, "cli_stop");
|
||||
if (!trace) {
|
||||
return {
|
||||
code: "protocol_error",
|
||||
@@ -785,9 +1079,9 @@ export async function handleRecordAwait(
|
||||
return { code: "cancelled", message: "record_await aborted" };
|
||||
}
|
||||
|
||||
const outcome = await new Promise<{ trace: Trace } | { error: RpcError }>((resolve) => {
|
||||
const outcome = await new Promise<{ trace: RecordedTrace } | { error: RpcError }>((resolve) => {
|
||||
let settled = false;
|
||||
const finish = (result: { trace: Trace } | { error: RpcError }) => {
|
||||
const finish = (result: { trace: RecordedTrace } | { error: RpcError }) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (timer) clearTimeout(timer);
|
||||
|
||||
@@ -677,7 +677,7 @@ describe("captureViewModel", () => {
|
||||
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
|
||||
});
|
||||
|
||||
it("subtracts scroll offset to make rects viewport-relative", async () => {
|
||||
it("preserves document rects while deriving viewport rects after scroll", async () => {
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
@@ -691,11 +691,10 @@ describe("captureViewModel", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
const div = nodes.find((n) => n.backendNodeId === 12);
|
||||
// local rect preserves viewport-relative scroll math.
|
||||
expect(div?.localRect?.y).toBe(-200);
|
||||
// exported rect is clipped to the top-level viewport before VOM consumes it.
|
||||
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
|
||||
const input = nodes.find((n) => n.backendNodeId === 13);
|
||||
expect(input?.documentRect).toEqual({ x: 400, y: 300, w: 200, h: 40 });
|
||||
expect(input?.localRect).toEqual({ x: 400, y: 100, w: 200, h: 40 });
|
||||
expect(input?.rect).toEqual({ x: 400, y: 100, w: 200, h: 40 });
|
||||
});
|
||||
|
||||
it("normalizes device-pixel bounds by devicePixelRatio", async () => {
|
||||
|
||||
@@ -25,6 +25,8 @@ export interface CapturedNode {
|
||||
rect: Rect | null;
|
||||
/** Frame-local viewport-relative CSS px before top-level translation/clipping. */
|
||||
localRect?: Rect | null;
|
||||
/** Frame-local document-relative CSS px as reported by DOMSnapshot. */
|
||||
documentRect?: Rect | null;
|
||||
paintOrder: number;
|
||||
position: string;
|
||||
pointerEvents: string;
|
||||
@@ -841,6 +843,7 @@ function parseDocumentNodes(
|
||||
|
||||
let rect: Rect | null = null;
|
||||
let localRect: Rect | null = null;
|
||||
let documentRect: Rect | null = null;
|
||||
let paintOrder = 0;
|
||||
let position = "static";
|
||||
let pointerEvents = "auto";
|
||||
@@ -849,12 +852,18 @@ function parseDocumentNodes(
|
||||
if (li !== undefined) {
|
||||
const b = dl?.bounds?.[li];
|
||||
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
|
||||
localRect = {
|
||||
x: b[0] / dpr - context.scrollX,
|
||||
y: b[1] / dpr - context.scrollY,
|
||||
documentRect = {
|
||||
x: b[0] / dpr,
|
||||
y: b[1] / dpr,
|
||||
w: b[2] / dpr,
|
||||
h: b[3] / dpr,
|
||||
};
|
||||
localRect = {
|
||||
x: documentRect.x - context.scrollX,
|
||||
y: documentRect.y - context.scrollY,
|
||||
w: documentRect.w,
|
||||
h: documentRect.h,
|
||||
};
|
||||
rect = rectInTop(localRect, context);
|
||||
}
|
||||
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
||||
@@ -878,6 +887,7 @@ function parseDocumentNodes(
|
||||
attrs,
|
||||
rect,
|
||||
localRect,
|
||||
documentRect,
|
||||
paintOrder,
|
||||
position,
|
||||
pointerEvents,
|
||||
|
||||
@@ -65,6 +65,11 @@ function deferredFakeTransport(): { transport: Transport; emit: (frame: Protocol
|
||||
}
|
||||
|
||||
describe("performHandshake", () => {
|
||||
it("advertises the protocol compatibility boundary", () => {
|
||||
expect(PROTOCOL_VERSION).toBe("1.1");
|
||||
expect(MIN_COMPATIBLE_PROTOCOL).toBe("1.0");
|
||||
});
|
||||
|
||||
it("sends system.handshake with identity and both compat fields", async () => {
|
||||
let sentFrame: ProtocolFrame | null = null;
|
||||
const transport = fakeTransport((req) => {
|
||||
@@ -74,7 +79,7 @@ describe("performHandshake", () => {
|
||||
result: {
|
||||
server: "browser-skill-daemon",
|
||||
version: "0.1.0",
|
||||
protocol_version: "1.0",
|
||||
protocol_version: "1.1",
|
||||
min_compatible_peer: "0.0.0",
|
||||
min_compatible_protocol: "1.0",
|
||||
},
|
||||
@@ -129,8 +134,8 @@ describe("performHandshake", () => {
|
||||
const response = {
|
||||
server: "browser-skill-daemon",
|
||||
version: "0.1.0",
|
||||
protocol_version: "1.0",
|
||||
min_compatible_protocol: "1.0",
|
||||
protocol_version: "1.1",
|
||||
min_compatible_protocol: "1.1",
|
||||
} satisfies HandshakeResult;
|
||||
const transport = fakeTransport((req) => ({
|
||||
id: (req as { id: string }).id,
|
||||
@@ -145,7 +150,7 @@ describe("performHandshake", () => {
|
||||
});
|
||||
|
||||
expect(outcome.result.min_compatible_peer).toBeUndefined();
|
||||
expect(outcome.result.min_compatible_protocol).toBe("1.0");
|
||||
expect(outcome.result.min_compatible_protocol).toBe("1.1");
|
||||
});
|
||||
|
||||
it("rejects when the daemon responds with an error", async () => {
|
||||
@@ -178,9 +183,9 @@ describe("performHandshake", () => {
|
||||
result: {
|
||||
server: "browser-skill-daemon",
|
||||
version: "0.1.0",
|
||||
protocol_version: "1.0",
|
||||
protocol_version: "1.1",
|
||||
min_compatible_peer: "0.0.0",
|
||||
min_compatible_protocol: "1.0",
|
||||
min_compatible_protocol: "1.1",
|
||||
},
|
||||
});
|
||||
emit({
|
||||
@@ -188,14 +193,14 @@ describe("performHandshake", () => {
|
||||
result: {
|
||||
server: "browser-skill-daemon",
|
||||
version: "0.1.0",
|
||||
protocol_version: "1.0",
|
||||
protocol_version: "1.1",
|
||||
min_compatible_peer: "0.0.0",
|
||||
min_compatible_protocol: "1.0",
|
||||
min_compatible_protocol: "1.1",
|
||||
},
|
||||
});
|
||||
|
||||
await expect(pending).resolves.toMatchObject({
|
||||
result: { server: "browser-skill-daemon", protocol_version: "1.0" },
|
||||
result: { server: "browser-skill-daemon", protocol_version: "1.1" },
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ import type {
|
||||
ResponseFrame,
|
||||
} from "./types";
|
||||
|
||||
export const PROTOCOL_VERSION = "1.0";
|
||||
export const PROTOCOL_VERSION = "1.1";
|
||||
/**
|
||||
* Extension semver, injected at build time from `package.json` via
|
||||
* Vite's `define` (see `wxt.config.ts` and `vitest.config.ts`).
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
// stay structural (interface, not class) so the same JSON parses on both
|
||||
// sides without extra adapters.
|
||||
|
||||
import type { CaptureTargetDescriptor } from "@/lib/describe-target";
|
||||
|
||||
export type RpcId = string;
|
||||
|
||||
export type ErrorCode =
|
||||
@@ -657,26 +659,52 @@ export interface EmulateResult {
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Semantic record payloads — mirror bsk-protocol record.rs (Trace v2)
|
||||
// Semantic record payloads — mirror bsk-protocol record.rs (Trace v3)
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
export const TRACE_VERSION = 3;
|
||||
export const TRACE_VERSION_V2 = 2;
|
||||
export const DEFAULT_TRACE_VERSION = 2;
|
||||
export const VOM_FORMAT_VERSION = 1;
|
||||
|
||||
export interface TargetDescriptor {
|
||||
ref?: string;
|
||||
role?: string;
|
||||
name?: string;
|
||||
tag: string;
|
||||
name_attr?: string;
|
||||
placeholder?: string;
|
||||
nearby_label?: string;
|
||||
ctx?: string;
|
||||
unmatched?: boolean;
|
||||
}
|
||||
|
||||
export interface TraceEntry {
|
||||
start_url: string;
|
||||
}
|
||||
|
||||
export interface PageRef {
|
||||
export interface RecorderInfo {
|
||||
bsk: string;
|
||||
vom: number;
|
||||
}
|
||||
|
||||
export type StopReason = "user_finish" | "cli_stop";
|
||||
|
||||
export interface TraceState {
|
||||
id: string;
|
||||
url: string;
|
||||
title?: string;
|
||||
/** Wire-only: full page observation text. */
|
||||
body?: string;
|
||||
/** Disk-only: filename under the bundle `pages/` directory. */
|
||||
page?: string;
|
||||
truncated?: boolean;
|
||||
}
|
||||
|
||||
export interface StepResult {
|
||||
state: string;
|
||||
}
|
||||
|
||||
export interface StepCommon {
|
||||
id: number;
|
||||
state: string;
|
||||
result: StepResult;
|
||||
}
|
||||
|
||||
export interface SelectedOption {
|
||||
@@ -684,66 +712,140 @@ export interface SelectedOption {
|
||||
label?: string;
|
||||
}
|
||||
|
||||
export interface StepEffect {
|
||||
navigated_to: string;
|
||||
export type NavigationCause =
|
||||
| "user_typed"
|
||||
| "link"
|
||||
| "form_submit"
|
||||
| "reload"
|
||||
| "history"
|
||||
| "script"
|
||||
| "browser";
|
||||
|
||||
export type FillCommit = "enter" | "suggestion" | "blur";
|
||||
|
||||
/** Geometry reported by the content script at capture time. */
|
||||
export interface CaptureGeometry {
|
||||
rect: { x: number; y: number; w: number; h: number };
|
||||
scrollX: number;
|
||||
scrollY: number;
|
||||
position: string;
|
||||
tag: string;
|
||||
ownerFrameBackendNodeId?: number | null;
|
||||
}
|
||||
|
||||
export interface StepCommon {
|
||||
id: number;
|
||||
page: string;
|
||||
effect?: StepEffect;
|
||||
}
|
||||
|
||||
/** Capture/buffer draft before v2 reduction. */
|
||||
/** Capture/buffer draft before v3 reduction. */
|
||||
export type DraftTraceStep =
|
||||
| {
|
||||
op: "click";
|
||||
target: TargetDescriptor;
|
||||
captureTarget?: CaptureTargetDescriptor;
|
||||
geometry?: CaptureGeometry;
|
||||
navigated_to?: string;
|
||||
page_url?: string;
|
||||
title?: string;
|
||||
preStateId?: string;
|
||||
postStateId?: string;
|
||||
observationBody?: string;
|
||||
observationHash?: string;
|
||||
truncated?: boolean;
|
||||
}
|
||||
| {
|
||||
op: "hover";
|
||||
target: TargetDescriptor;
|
||||
captureTarget?: CaptureTargetDescriptor;
|
||||
geometry?: CaptureGeometry;
|
||||
page_url?: string;
|
||||
title?: string;
|
||||
preStateId?: string;
|
||||
postStateId?: string;
|
||||
observationBody?: string;
|
||||
observationHash?: string;
|
||||
truncated?: boolean;
|
||||
}
|
||||
| {
|
||||
op: "fill";
|
||||
target: TargetDescriptor;
|
||||
captureTarget?: CaptureTargetDescriptor;
|
||||
geometry?: CaptureGeometry;
|
||||
value: string;
|
||||
commit?: FillCommit;
|
||||
redacted?: boolean;
|
||||
navigated_to?: string;
|
||||
page_url?: string;
|
||||
title?: string;
|
||||
preStateId?: string;
|
||||
postStateId?: string;
|
||||
observationBody?: string;
|
||||
observationHash?: string;
|
||||
truncated?: boolean;
|
||||
}
|
||||
| {
|
||||
op: "press";
|
||||
key: string;
|
||||
target?: TargetDescriptor;
|
||||
captureTarget?: CaptureTargetDescriptor;
|
||||
geometry?: CaptureGeometry;
|
||||
modifiers?: KeyModifier[];
|
||||
navigated_to?: string;
|
||||
page_url?: string;
|
||||
title?: string;
|
||||
preStateId?: string;
|
||||
postStateId?: string;
|
||||
observationBody?: string;
|
||||
observationHash?: string;
|
||||
truncated?: boolean;
|
||||
}
|
||||
| {
|
||||
op: "select";
|
||||
target: TargetDescriptor;
|
||||
captureTarget?: CaptureTargetDescriptor;
|
||||
geometry?: CaptureGeometry;
|
||||
values: string[];
|
||||
labels?: string[];
|
||||
navigated_to?: string;
|
||||
page_url?: string;
|
||||
title?: string;
|
||||
preStateId?: string;
|
||||
postStateId?: string;
|
||||
observationBody?: string;
|
||||
observationHash?: string;
|
||||
truncated?: boolean;
|
||||
}
|
||||
| {
|
||||
op: "scroll";
|
||||
page_url?: string;
|
||||
title?: string;
|
||||
preStateId?: string;
|
||||
postStateId?: string;
|
||||
observationBody?: string;
|
||||
observationHash?: string;
|
||||
truncated?: boolean;
|
||||
}
|
||||
| {
|
||||
op: "navigate";
|
||||
url: string;
|
||||
cause?: NavigationCause;
|
||||
page_url?: string;
|
||||
title?: string;
|
||||
preStateId?: string;
|
||||
postStateId?: string;
|
||||
observationBody?: string;
|
||||
observationHash?: string;
|
||||
truncated?: boolean;
|
||||
/** Raw webNavigation transition metadata for cause mapping. */
|
||||
transitionType?: string;
|
||||
transitionQualifiers?: string[];
|
||||
};
|
||||
|
||||
/** Exported record-only step (trace v2). */
|
||||
/** Exported record-only step (trace v3). */
|
||||
export type Step =
|
||||
| ({ op: "navigate" } & StepCommon & { to: string })
|
||||
| ({ op: "navigate" } & StepCommon & { to: string; cause: NavigationCause })
|
||||
| ({ op: "click" } & StepCommon & { target: TargetDescriptor })
|
||||
| ({ op: "hover" } & StepCommon & { target: TargetDescriptor })
|
||||
| ({ op: "fill" } & StepCommon & {
|
||||
target: TargetDescriptor;
|
||||
value: string;
|
||||
commit: FillCommit;
|
||||
redacted?: boolean;
|
||||
})
|
||||
| ({ op: "select" } & StepCommon & {
|
||||
@@ -754,22 +856,95 @@ export type Step =
|
||||
key: string;
|
||||
modifiers?: KeyModifier[];
|
||||
target?: TargetDescriptor;
|
||||
});
|
||||
})
|
||||
| ({ op: "scroll" } & StepCommon);
|
||||
|
||||
export interface Trace {
|
||||
export interface TraceV3 {
|
||||
version: number;
|
||||
recorded_at: string;
|
||||
started_at?: string;
|
||||
purpose?: string;
|
||||
stopped_by: StopReason;
|
||||
entry: TraceEntry;
|
||||
recorder: RecorderInfo;
|
||||
states: TraceState[];
|
||||
steps: Step[];
|
||||
}
|
||||
|
||||
/** @deprecated Use TraceV3 — kept as alias for existing call sites. */
|
||||
export type Trace = TraceV3;
|
||||
|
||||
export interface TargetDescriptorV2 {
|
||||
role?: string;
|
||||
name?: string;
|
||||
tag: string;
|
||||
name_attr?: string;
|
||||
placeholder?: string;
|
||||
nearby_label?: string;
|
||||
}
|
||||
|
||||
export interface PageRef {
|
||||
id: string;
|
||||
url: string;
|
||||
title?: string;
|
||||
}
|
||||
|
||||
export type StepV2 =
|
||||
| { op: "navigate"; id: number; page: string; to: string; effect?: { navigated_to: string } }
|
||||
| {
|
||||
op: "click";
|
||||
id: number;
|
||||
page: string;
|
||||
target: TargetDescriptorV2;
|
||||
effect?: { navigated_to: string };
|
||||
}
|
||||
| {
|
||||
op: "fill";
|
||||
id: number;
|
||||
page: string;
|
||||
target: TargetDescriptorV2;
|
||||
value: string;
|
||||
redacted?: boolean;
|
||||
effect?: { navigated_to: string };
|
||||
}
|
||||
| {
|
||||
op: "select";
|
||||
id: number;
|
||||
page: string;
|
||||
target: TargetDescriptorV2;
|
||||
selection: SelectedOption[];
|
||||
effect?: { navigated_to: string };
|
||||
}
|
||||
| {
|
||||
op: "press";
|
||||
id: number;
|
||||
page: string;
|
||||
key: string;
|
||||
modifiers?: KeyModifier[];
|
||||
target?: TargetDescriptorV2;
|
||||
effect?: { navigated_to: string };
|
||||
};
|
||||
|
||||
export interface TraceV2 {
|
||||
recorded_at: string;
|
||||
started_at?: string;
|
||||
purpose?: string;
|
||||
entry: TraceEntry;
|
||||
pages: PageRef[];
|
||||
steps: Step[];
|
||||
steps: StepV2[];
|
||||
}
|
||||
|
||||
export type RecordedTrace = TraceV2 | TraceV3;
|
||||
|
||||
export interface RecordStartParams {
|
||||
session_id: string;
|
||||
tab_id?: number;
|
||||
url?: string;
|
||||
purpose?: string;
|
||||
max_page_tokens?: number;
|
||||
redact_values?: boolean;
|
||||
/** Omitted ⇒ v2; `3` ⇒ state-linked v3 bundle. */
|
||||
trace_version?: number;
|
||||
}
|
||||
|
||||
export interface RecordStartResult {
|
||||
@@ -782,7 +957,7 @@ export interface RecordStopParams {
|
||||
}
|
||||
|
||||
export interface RecordStopResult {
|
||||
trace: Trace;
|
||||
trace: RecordedTrace;
|
||||
}
|
||||
|
||||
export interface RecordAwaitParams {
|
||||
@@ -791,5 +966,5 @@ export interface RecordAwaitParams {
|
||||
}
|
||||
|
||||
export interface RecordAwaitResult {
|
||||
trace: Trace;
|
||||
trace: RecordedTrace;
|
||||
}
|
||||
|
||||
@@ -265,19 +265,27 @@ retry — complete the task autonomously or stop gracefully.
|
||||
|
||||
### Recording — `bsk record`
|
||||
|
||||
Capture the user's own actions in the Agent Window to a `trace.json`, for later LLM-driven automation:
|
||||
Capture the user's own actions in the Agent Window to a **trace bundle**, for later LLM-driven automation. New CLI builds request **trace v3** (page observations + action chain); older extensions may still return **trace v2** (actions only), which the CLI exports as a single `trace.json` without `pages/`.
|
||||
|
||||
```bash
|
||||
bsk record start --browser <instance-id-or-label> [--url https://…] [--purpose "publish a wiki doc"] [--output trace.json]
|
||||
bsk record start --browser <instance-id-or-label> \
|
||||
[--url https://…] [--purpose "publish a wiki doc"] \
|
||||
[--max-page-tokens 3000] [--redact-values] \
|
||||
[--output trace]
|
||||
# `--url` is optional; default https://example.com/ when omitted (must be http(s)).
|
||||
# Blocks until the user clicks Finish in the recording panel, then writes ./trace.json and closes the window.
|
||||
# Blocks until the user clicks Finish in the recording panel, then writes:
|
||||
# trace/trace.json — action chain (+ state index when v3)
|
||||
# trace/pages/ — v3 only: one `.vom.txt` observe snapshot per settled page state
|
||||
|
||||
bsk record stop [--output trace.json] # terminal fallback if the browser panel is unavailable
|
||||
bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable
|
||||
```
|
||||
|
||||
- The trace is a **record-only action log** (a `pages[]` dictionary + `navigate`/`click`/`fill`/`select`/`press` steps with `target` descriptors). It records *what the user did*; deciding which inputs are variable is left to the executing agent.
|
||||
- **v3 bundle (preferred):** `--output` is a directory (default `./trace`) containing `trace.json` and `pages/`. `trace.json` lists `states[]` (page observation ids) and `steps[]` (each step binds `state` = observe snapshot *before* the action and `result.state` = snapshot *after* settle). Page bodies live in `pages/sN.vom.txt` using the same VOM format as `bsk observe`.
|
||||
- **v2 fallback:** when the connected extension is older, export may contain only legacy `trace.json` with `pages[]` action context and no `pages/` observation files. Update the extension for full v3 bundles.
|
||||
- Each `states[]` entry is one **settled page observation** (captured after recording start, navigation landing, or action settle — not on a timer).
|
||||
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
|
||||
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
||||
- There is **no** `bsk replay` — to redo a flow, read the trace and reuse the existing `session` / `snapshot` / `@eN` / `click` / `fill` tools. Follow **Stop when the goal is met**.
|
||||
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
|
||||
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
||||
|
||||
## Error handling
|
||||
|
||||
@@ -480,7 +480,7 @@ mod m2_tests {
|
||||
fn fake_status(browsers: Vec<BrowserStatusEntry>, skew: Vec<VersionSkewEntry>) -> StatusResult {
|
||||
StatusResult {
|
||||
daemon_version: env!("CARGO_PKG_VERSION").into(),
|
||||
protocol_version: "1.0".into(),
|
||||
protocol_version: "1.1".into(),
|
||||
pid: 1,
|
||||
uptime_secs: 0,
|
||||
ws_port: 0,
|
||||
@@ -545,7 +545,7 @@ mod m2_tests {
|
||||
session_count: 0,
|
||||
connected_at_ms: 1,
|
||||
version_skew: false,
|
||||
extension_protocol_version: "1.0".into(),
|
||||
extension_protocol_version: "1.1".into(),
|
||||
}],
|
||||
Vec::new(),
|
||||
);
|
||||
@@ -566,7 +566,7 @@ mod m2_tests {
|
||||
session_count: 0,
|
||||
connected_at_ms: 1,
|
||||
version_skew: true,
|
||||
extension_protocol_version: "1.1".into(),
|
||||
extension_protocol_version: "1.2".into(),
|
||||
}],
|
||||
vec![VersionSkewEntry {
|
||||
instance_id: "alpha".into(),
|
||||
@@ -574,8 +574,8 @@ mod m2_tests {
|
||||
label: "Personal".into(),
|
||||
server_version: env!("CARGO_PKG_VERSION").into(),
|
||||
client_version: "0.0.9".into(),
|
||||
server_protocol_version: "1.0".into(),
|
||||
client_protocol_version: "1.1".into(),
|
||||
server_protocol_version: "1.1".into(),
|
||||
client_protocol_version: "1.2".into(),
|
||||
}],
|
||||
);
|
||||
let check = check_browsers_protocol_compatible(Some(&status));
|
||||
|
||||
@@ -1,15 +1,17 @@
|
||||
//! `bsk record start|stop` — capture user actions in the Agent Window.
|
||||
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
use std::collections::HashSet;
|
||||
use std::fs::{self, OpenOptions};
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::Context;
|
||||
use bsk_protocol::Method;
|
||||
use bsk_protocol::tools::{
|
||||
RecordAwaitParams, RecordAwaitResult, RecordStartParams, RecordStartResult, RecordStopParams,
|
||||
RecordStopResult, Trace,
|
||||
RecordStopResult, RecordedTrace, TRACE_VERSION, TRACE_VERSION_V2, Trace, TraceState, TraceV2,
|
||||
};
|
||||
use bsk_protocol::{ErrorCode, Method};
|
||||
use clap::{Args, Subcommand};
|
||||
|
||||
use crate::cli::TOOL_IPC_TIMEOUT;
|
||||
@@ -18,10 +20,34 @@ use crate::cli::ensure_daemon::ensure_daemon;
|
||||
use crate::cli::error::{CliError, Format};
|
||||
use crate::cli::record_state;
|
||||
use crate::cli::session::{SessionStartOptions, start_session, stop_session};
|
||||
use crate::daemon::ipc::STOP_IN_PROGRESS_REASON;
|
||||
|
||||
/// Max wait for the user to click 结束 in the browser (24 hours).
|
||||
const RECORD_AWAIT_TIMEOUT_MS: u32 = 86_400_000;
|
||||
|
||||
/// Tear down the recording session, tolerating the teardown the other half of
|
||||
/// the pair already started. `record start` blocks in `record_await` and stops
|
||||
/// the session once the trace arrives; the documented `record stop` fallback
|
||||
/// stops it too. Whichever loses that race must not report a failure — the
|
||||
/// trace is already written by then.
|
||||
fn stop_recording_session(sock: PathBuf, session_id: &str) -> Result<(), CliError> {
|
||||
match stop_session(sock, session_id) {
|
||||
Ok(_) => Ok(()),
|
||||
Err(err) if session_teardown_raced(&err) => Ok(()),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
fn session_teardown_raced(err: &CliError) -> bool {
|
||||
if err.code() == Some(ErrorCode::NotFound) {
|
||||
return true;
|
||||
}
|
||||
err.data()
|
||||
.and_then(|data| data.get("reason"))
|
||||
.and_then(serde_json::Value::as_str)
|
||||
== Some(STOP_IN_PROGRESS_REASON)
|
||||
}
|
||||
|
||||
/// `bsk record …` subcommand tree.
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct RecordCmd {
|
||||
@@ -58,15 +84,25 @@ pub struct RecordStartArgs {
|
||||
#[arg(long)]
|
||||
pub purpose: Option<String>,
|
||||
|
||||
/// Output path for the trace JSON (default `./trace.json`).
|
||||
#[arg(long, default_value = "trace.json")]
|
||||
/// Max VOM tokens per page observation file (default 3000).
|
||||
#[arg(long = "max-page-tokens")]
|
||||
pub max_page_tokens: Option<u32>,
|
||||
|
||||
/// Redact all form values in page observations (`[filled]` only).
|
||||
#[arg(long = "redact-values")]
|
||||
pub redact_values: bool,
|
||||
|
||||
/// Output directory for the trace bundle (default `./trace`).
|
||||
/// Writes `<dir>/trace.json` and `<dir>/pages/*.vom.txt`.
|
||||
#[arg(long, default_value = "trace")]
|
||||
pub output: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct RecordStopArgs {
|
||||
/// Output path for the trace JSON (default `./trace.json`).
|
||||
#[arg(long, default_value = "trace.json")]
|
||||
/// Output directory for the trace bundle (default `./trace`).
|
||||
/// Writes `<dir>/trace.json` and `<dir>/pages/*.vom.txt`.
|
||||
#[arg(long, default_value = "trace")]
|
||||
pub output: PathBuf,
|
||||
}
|
||||
|
||||
@@ -98,6 +134,9 @@ fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError>
|
||||
tab_id: args.tab_id,
|
||||
url: args.url,
|
||||
purpose: args.purpose.clone(),
|
||||
max_page_tokens: args.max_page_tokens,
|
||||
redact_values: Some(args.redact_values),
|
||||
trace_version: Some(TRACE_VERSION),
|
||||
};
|
||||
let start_result = business_rpc::call::<RecordStartParams, RecordStartResult>(
|
||||
info.sock_path.clone(),
|
||||
@@ -142,13 +181,13 @@ fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError>
|
||||
// Keep write/render inside a Result so `?` cannot skip session teardown.
|
||||
let run_result: Result<(), CliError> = match await_result {
|
||||
Ok(result) => (|| {
|
||||
write_trace_file(&args.output, &result.trace)?;
|
||||
render_finish(&result.trace, &args.output, format)
|
||||
let exported = export_recorded_trace(&args.output, &result.trace)?;
|
||||
render_finish(&result.trace, &args.output, &exported, format)
|
||||
})(),
|
||||
Err(err) => Err(err),
|
||||
};
|
||||
|
||||
let session_stop_result = stop_session(info.sock_path, &session.session_id);
|
||||
let session_stop_result = stop_recording_session(info.sock_path, &session.session_id);
|
||||
record_state::clear();
|
||||
|
||||
run_result?;
|
||||
@@ -161,22 +200,23 @@ fn dispatch_stop(args: RecordStopArgs, format: Format) -> Result<(), CliError> {
|
||||
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||
let session_id = state.session_id.clone();
|
||||
|
||||
let params = RecordStopParams {
|
||||
session_id: session_id.clone(),
|
||||
};
|
||||
let result = business_rpc::call::<RecordStopParams, RecordStopResult>(
|
||||
info.sock_path.clone(),
|
||||
"record-stop",
|
||||
Method::ToolRecordStop,
|
||||
Some(params),
|
||||
TOOL_IPC_TIMEOUT,
|
||||
)?;
|
||||
|
||||
let run_result: Result<(), CliError> = (|| {
|
||||
let params = RecordStopParams {
|
||||
session_id: session_id.clone(),
|
||||
};
|
||||
let result = business_rpc::call::<RecordStopParams, RecordStopResult>(
|
||||
info.sock_path.clone(),
|
||||
"record-stop",
|
||||
Method::ToolRecordStop,
|
||||
Some(params),
|
||||
TOOL_IPC_TIMEOUT,
|
||||
)?;
|
||||
write_trace_file(&args.output, &result.trace)?;
|
||||
render_stop(&result, &args.output, format)
|
||||
let exported = export_recorded_trace(&args.output, &result.trace)?;
|
||||
render_stop(&result, &args.output, &exported, format)
|
||||
})();
|
||||
|
||||
let session_stop_result = stop_session(info.sock_path, &session_id);
|
||||
let session_stop_result = stop_recording_session(info.sock_path, &session_id);
|
||||
record_state::clear();
|
||||
|
||||
run_result?;
|
||||
@@ -190,39 +230,444 @@ fn record_await_ipc_timeout(timeout_ms: u32) -> Duration {
|
||||
.unwrap_or(Duration::from_secs(u64::from(timeout_ms / 1_000) + 15))
|
||||
}
|
||||
|
||||
fn write_trace_file(output: &PathBuf, trace: &Trace) -> Result<(), CliError> {
|
||||
struct ExportMeta {
|
||||
pages_dir: Option<PathBuf>,
|
||||
trace_version: u32,
|
||||
/// True when the CLI requested v3 but the extension returned legacy v2.
|
||||
v2_fallback: bool,
|
||||
}
|
||||
|
||||
fn export_recorded_trace(output_dir: &Path, trace: &RecordedTrace) -> Result<ExportMeta, CliError> {
|
||||
match trace {
|
||||
RecordedTrace::V3(trace) => {
|
||||
let pages_dir = write_trace_bundle(output_dir, trace)?;
|
||||
Ok(ExportMeta {
|
||||
pages_dir: Some(pages_dir),
|
||||
trace_version: TRACE_VERSION,
|
||||
v2_fallback: false,
|
||||
})
|
||||
}
|
||||
RecordedTrace::V2(trace) => {
|
||||
write_trace_v2(output_dir, trace)?;
|
||||
Ok(ExportMeta {
|
||||
pages_dir: None,
|
||||
trace_version: TRACE_VERSION_V2,
|
||||
v2_fallback: true,
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn acquire_export_lock(output_dir: &Path) -> Result<std::fs::File, CliError> {
|
||||
validate_bundle_directory(output_dir, "output bundle directory")?;
|
||||
fs::create_dir_all(output_dir)
|
||||
.with_context(|| format!("create output dir {}", output_dir.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
let lock_path = output_dir.join(".bsk-record-export.lock");
|
||||
let lock_file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.open(&lock_path)
|
||||
.with_context(|| format!("open export lock {}", lock_path.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
fs2::FileExt::lock_exclusive(&lock_file)
|
||||
.with_context(|| format!("lock export bundle {}", output_dir.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
Ok(lock_file)
|
||||
}
|
||||
|
||||
fn write_trace_v2(output_dir: &Path, trace: &TraceV2) -> Result<(), CliError> {
|
||||
let trace_path = trace_json_path(output_dir);
|
||||
let pages_dir = pages_dir_for_output(output_dir);
|
||||
let json = serde_json::to_string_pretty(trace)
|
||||
.context("serialize trace JSON")
|
||||
.map_err(CliError::Local)?;
|
||||
if let Some(parent) = output.parent() {
|
||||
if !parent.as_os_str().is_empty() {
|
||||
fs::create_dir_all(parent)
|
||||
.with_context(|| format!("create output directory {}", parent.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
|
||||
let _lock = acquire_export_lock(output_dir)?;
|
||||
validate_replaceable_file(&trace_path, "trace JSON")?;
|
||||
|
||||
let transaction_id = uuid::Uuid::new_v4();
|
||||
let staging_dir = output_dir.join(format!(".bsk-record-stage-{transaction_id}"));
|
||||
fs::create_dir_all(&staging_dir)
|
||||
.with_context(|| format!("create staging dir {}", staging_dir.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
|
||||
let staged_trace = staging_dir.join("trace.json");
|
||||
let stage_result: Result<(), CliError> = (|| {
|
||||
fs::write(&staged_trace, format!("{json}\n"))
|
||||
.with_context(|| format!("write staged trace to {}", staged_trace.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
Ok(())
|
||||
})();
|
||||
if let Err(err) = stage_result {
|
||||
let _ = fs::remove_dir_all(&staging_dir);
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let backup = output_dir.join(format!(".trace.json.{transaction_id}.bak"));
|
||||
match commit_staged_file(&staged_trace, &trace_path, &backup) {
|
||||
Ok(old_file) => {
|
||||
if let Some(old_file) = old_file {
|
||||
let _ = fs::remove_file(old_file);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
let _ = fs::remove_dir_all(&staging_dir);
|
||||
return Err(CliError::Local(anyhow::anyhow!(
|
||||
"commit v2 trace {}: {err}",
|
||||
trace_path.display()
|
||||
)));
|
||||
}
|
||||
}
|
||||
fs::write(output, format!("{json}\n"))
|
||||
.with_context(|| format!("write trace to {}", output.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
let _ = fs::remove_dir_all(&staging_dir);
|
||||
|
||||
if pages_dir.is_dir() {
|
||||
prune_stale_pages(&pages_dir, &HashSet::new())?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn render_finish(trace: &Trace, output: &PathBuf, format: Format) -> Result<(), CliError> {
|
||||
fn pages_dir_for_output(output_dir: &Path) -> PathBuf {
|
||||
output_dir.join("pages")
|
||||
}
|
||||
|
||||
fn trace_json_path(output_dir: &Path) -> PathBuf {
|
||||
output_dir.join("trace.json")
|
||||
}
|
||||
|
||||
fn is_canonical_state_id(id: &str) -> bool {
|
||||
let Some(number) = id.strip_prefix('s') else {
|
||||
return false;
|
||||
};
|
||||
let mut digits = number.bytes();
|
||||
matches!(digits.next(), Some(b'1'..=b'9')) && digits.all(|byte| byte.is_ascii_digit())
|
||||
}
|
||||
|
||||
fn validate_bundle_directory(path: &Path, description: &str) -> Result<(), CliError> {
|
||||
match fs::symlink_metadata(path) {
|
||||
Ok(metadata) if metadata.file_type().is_symlink() => Err(CliError::Local(anyhow::anyhow!(
|
||||
"{description} {} must not be a symlink",
|
||||
path.display()
|
||||
))),
|
||||
Ok(metadata) if metadata.is_dir() => Ok(()),
|
||||
Ok(_) => Err(CliError::Local(anyhow::anyhow!(
|
||||
"{description} {} is not a directory",
|
||||
path.display()
|
||||
))),
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
|
||||
Err(err) => Err(CliError::Local(
|
||||
anyhow::Error::new(err).context(format!("inspect {description} {}", path.display())),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_replaceable_file(path: &Path, description: &str) -> Result<(), CliError> {
|
||||
match fs::symlink_metadata(path) {
|
||||
Ok(metadata) if metadata.file_type().is_file() => Ok(()),
|
||||
Ok(_) => Err(CliError::Local(anyhow::anyhow!(
|
||||
"{description} {} is not a regular file",
|
||||
path.display()
|
||||
))),
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
|
||||
Err(err) => Err(CliError::Local(
|
||||
anyhow::Error::new(err).context(format!("inspect {description} {}", path.display())),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn commit_staged_file(staged: &Path, target: &Path, backup: &Path) -> io::Result<Option<PathBuf>> {
|
||||
let old_file = match fs::symlink_metadata(target) {
|
||||
Ok(_) => {
|
||||
fs::rename(target, backup)?;
|
||||
Some(backup.to_path_buf())
|
||||
}
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => None,
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
if let Err(err) = fs::rename(staged, target) {
|
||||
if let Some(old_file) = old_file.as_deref() {
|
||||
if let Err(restore_err) = fs::rename(old_file, target) {
|
||||
return Err(io::Error::new(
|
||||
restore_err.kind(),
|
||||
format!(
|
||||
"{err}; additionally failed to restore {}: {restore_err}",
|
||||
target.display()
|
||||
),
|
||||
));
|
||||
}
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
Ok(old_file)
|
||||
}
|
||||
|
||||
fn rollback_installed_files(installed: &[(PathBuf, Option<PathBuf>)]) -> io::Result<()> {
|
||||
let mut rollback_error = None;
|
||||
for (target, backup) in installed.iter().rev() {
|
||||
match fs::remove_file(target) {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(err) => {
|
||||
rollback_error.get_or_insert(err);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if let Some(backup) = backup {
|
||||
if let Err(err) = fs::rename(backup, target) {
|
||||
rollback_error.get_or_insert(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
match rollback_error {
|
||||
Some(err) => Err(err),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop page files left by an earlier, longer recording without deleting the
|
||||
/// directory itself: `record start` and the `record stop` fallback export the
|
||||
/// same recording concurrently, and a `remove_dir_all` here would fail the
|
||||
/// writer whose files the other one just erased.
|
||||
fn prune_stale_pages(pages_dir: &Path, keep: &HashSet<String>) -> Result<(), CliError> {
|
||||
let entries = match fs::read_dir(pages_dir) {
|
||||
Ok(entries) => entries,
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(err) => {
|
||||
return Err(CliError::Local(
|
||||
anyhow::Error::new(err).context(format!("read pages dir {}", pages_dir.display())),
|
||||
));
|
||||
}
|
||||
};
|
||||
for entry in entries.flatten() {
|
||||
let name = entry.file_name();
|
||||
let Some(name) = name.to_str() else { continue };
|
||||
if !name.ends_with(".vom.txt") || keep.contains(name) {
|
||||
continue;
|
||||
}
|
||||
match fs::remove_file(entry.path()) {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(err) => {
|
||||
return Err(CliError::Local(
|
||||
anyhow::Error::new(err)
|
||||
.context(format!("remove stale page {}", entry.path().display())),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_trace_bundle(output_dir: &Path, trace: &Trace) -> Result<PathBuf, CliError> {
|
||||
let pages_dir = pages_dir_for_output(output_dir);
|
||||
let trace_path = trace_json_path(output_dir);
|
||||
|
||||
if trace.version != TRACE_VERSION {
|
||||
return Err(CliError::Local(anyhow::anyhow!(
|
||||
"trace version {} is not supported (expected {})",
|
||||
trace.version,
|
||||
TRACE_VERSION
|
||||
)));
|
||||
}
|
||||
|
||||
let mut state_ids = HashSet::with_capacity(trace.states.len());
|
||||
for state in &trace.states {
|
||||
if !is_canonical_state_id(&state.id) {
|
||||
return Err(CliError::Local(anyhow::anyhow!(
|
||||
"state id {:?} is not canonical (expected s1, s2, ...)",
|
||||
state.id
|
||||
)));
|
||||
}
|
||||
if !state_ids.insert(state.id.as_str()) {
|
||||
return Err(CliError::Local(anyhow::anyhow!(
|
||||
"duplicate state id {:?}",
|
||||
state.id
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
let mut disk_states: Vec<TraceState> = Vec::with_capacity(trace.states.len());
|
||||
let mut page_writes = Vec::new();
|
||||
let mut keep = HashSet::new();
|
||||
for state in &trace.states {
|
||||
let mut disk = state.clone();
|
||||
if let Some(body) = state.body.as_deref() {
|
||||
let filename = format!("{}.vom.txt", state.id);
|
||||
let page_path = pages_dir.join(&filename);
|
||||
if page_path.parent() != Some(pages_dir.as_path()) {
|
||||
return Err(CliError::Local(anyhow::anyhow!(
|
||||
"page path {} escapes pages directory {}",
|
||||
page_path.display(),
|
||||
pages_dir.display()
|
||||
)));
|
||||
}
|
||||
keep.insert(filename.clone());
|
||||
page_writes.push((filename.clone(), body));
|
||||
disk.body = None;
|
||||
disk.page = Some(filename);
|
||||
}
|
||||
disk_states.push(disk);
|
||||
}
|
||||
|
||||
let mut disk_trace = trace.clone();
|
||||
disk_trace.states = disk_states;
|
||||
let json = serde_json::to_string_pretty(&disk_trace)
|
||||
.context("serialize trace JSON")
|
||||
.map_err(CliError::Local)?;
|
||||
|
||||
validate_bundle_directory(output_dir, "output bundle directory")?;
|
||||
fs::create_dir_all(output_dir)
|
||||
.with_context(|| format!("create output dir {}", output_dir.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
|
||||
validate_bundle_directory(&pages_dir, "pages directory")?;
|
||||
|
||||
let _lock = acquire_export_lock(output_dir)?;
|
||||
|
||||
fs::create_dir_all(&pages_dir)
|
||||
.with_context(|| format!("create pages dir {}", pages_dir.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
|
||||
for (filename, _) in &page_writes {
|
||||
validate_replaceable_file(&pages_dir.join(filename), "page observation")?;
|
||||
}
|
||||
validate_replaceable_file(&trace_path, "trace JSON")?;
|
||||
|
||||
let transaction_id = uuid::Uuid::new_v4();
|
||||
let staging_dir = output_dir.join(format!(".bsk-record-stage-{transaction_id}"));
|
||||
let staging_pages = staging_dir.join("pages");
|
||||
fs::create_dir_all(&staging_pages)
|
||||
.with_context(|| format!("create staging dir {}", staging_pages.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
|
||||
let stage_result: Result<(), CliError> = (|| {
|
||||
for (filename, body) in &page_writes {
|
||||
let staged_path = staging_pages.join(filename);
|
||||
fs::write(&staged_path, body)
|
||||
.with_context(|| format!("write staged page observation {}", staged_path.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
}
|
||||
let staged_trace = staging_dir.join("trace.json");
|
||||
fs::write(&staged_trace, format!("{json}\n"))
|
||||
.with_context(|| format!("write staged trace to {}", staged_trace.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
Ok(())
|
||||
})();
|
||||
if let Err(err) = stage_result {
|
||||
let _ = fs::remove_dir_all(&staging_dir);
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let mut replacements: Vec<(PathBuf, PathBuf, PathBuf)> =
|
||||
Vec::with_capacity(page_writes.len() + 1);
|
||||
for (filename, _) in &page_writes {
|
||||
replacements.push((
|
||||
staging_pages.join(filename),
|
||||
pages_dir.join(filename),
|
||||
pages_dir.join(format!(".{filename}.{transaction_id}.bak")),
|
||||
));
|
||||
}
|
||||
replacements.push((
|
||||
staging_dir.join("trace.json"),
|
||||
trace_path.clone(),
|
||||
output_dir.join(format!(".trace.json.{transaction_id}.bak")),
|
||||
));
|
||||
|
||||
let mut installed = Vec::with_capacity(replacements.len());
|
||||
for (staged, target, backup) in replacements {
|
||||
match commit_staged_file(&staged, &target, &backup) {
|
||||
Ok(old_file) => installed.push((target, old_file)),
|
||||
Err(err) => {
|
||||
let rollback_result = rollback_installed_files(&installed);
|
||||
let _ = fs::remove_dir_all(&staging_dir);
|
||||
let message = match rollback_result {
|
||||
Ok(()) => format!("commit replacement {}: {err}", target.display()),
|
||||
Err(rollback_err) => format!(
|
||||
"commit replacement {}: {err}; rollback also failed: {rollback_err}",
|
||||
target.display()
|
||||
),
|
||||
};
|
||||
return Err(CliError::Local(anyhow::anyhow!(message)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (_, backup) in &installed {
|
||||
if let Some(backup) = backup {
|
||||
let _ = fs::remove_file(backup);
|
||||
}
|
||||
}
|
||||
let _ = fs::remove_dir_all(&staging_dir);
|
||||
prune_stale_pages(&pages_dir, &keep)?;
|
||||
|
||||
Ok(pages_dir)
|
||||
}
|
||||
|
||||
fn render_finish(
|
||||
trace: &RecordedTrace,
|
||||
output_dir: &PathBuf,
|
||||
exported: &ExportMeta,
|
||||
format: Format,
|
||||
) -> Result<(), CliError> {
|
||||
let trace_path = trace_json_path(output_dir);
|
||||
match format {
|
||||
Format::Json => {
|
||||
let payload = match trace {
|
||||
RecordedTrace::V3(t) => serde_json::json!({
|
||||
"output": output_dir,
|
||||
"trace_json": trace_path,
|
||||
"trace_version": exported.trace_version,
|
||||
"pages_dir": exported.pages_dir,
|
||||
"trace": t,
|
||||
"window_closed": true,
|
||||
}),
|
||||
RecordedTrace::V2(t) => serde_json::json!({
|
||||
"output": output_dir,
|
||||
"trace_json": trace_path,
|
||||
"trace_version": exported.trace_version,
|
||||
"pages_dir": null,
|
||||
"trace": t,
|
||||
"window_closed": true,
|
||||
}),
|
||||
};
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string(&serde_json::json!({
|
||||
"output": output,
|
||||
"trace": trace,
|
||||
"window_closed": true,
|
||||
}))
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
|
||||
serde_json::to_string(&payload).map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
|
||||
);
|
||||
}
|
||||
Format::Human => {
|
||||
println!("saved {} steps to {}", trace.steps.len(), output.display());
|
||||
}
|
||||
Format::Human => match trace {
|
||||
RecordedTrace::V3(t) => {
|
||||
let pages_dir = exported
|
||||
.pages_dir
|
||||
.as_ref()
|
||||
.map(|p| p.display().to_string())
|
||||
.unwrap_or_else(|| pages_dir_for_output(output_dir).display().to_string());
|
||||
println!(
|
||||
"saved {} steps and {} states to {} (+ {})",
|
||||
t.steps.len(),
|
||||
t.states.len(),
|
||||
trace_path.display(),
|
||||
pages_dir
|
||||
);
|
||||
}
|
||||
RecordedTrace::V2(t) => {
|
||||
if exported.v2_fallback {
|
||||
eprintln!(
|
||||
"note: extension returned trace v2 (no page observations); update the BrowserSkill extension for v3 bundles"
|
||||
);
|
||||
}
|
||||
println!(
|
||||
"saved {} steps to {} (trace v2)",
|
||||
t.steps.len(),
|
||||
trace_path.display()
|
||||
);
|
||||
}
|
||||
},
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -230,33 +675,297 @@ fn render_finish(trace: &Trace, output: &PathBuf, format: Format) -> Result<(),
|
||||
fn render_stop(
|
||||
result: &RecordStopResult,
|
||||
output: &PathBuf,
|
||||
exported: &ExportMeta,
|
||||
format: Format,
|
||||
) -> Result<(), CliError> {
|
||||
render_finish(&result.trace, output, format)
|
||||
render_finish(&result.trace, output, exported, format)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use bsk_protocol::tools::{
|
||||
NavigationCause, RecorderInfo, Step, StepCommon, StepResult, StopReason, TraceEntry,
|
||||
VOM_FORMAT_VERSION,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn default_output_is_trace_json() {
|
||||
let args = RecordStopArgs {
|
||||
output: PathBuf::from("trace.json"),
|
||||
};
|
||||
assert_eq!(args.output, PathBuf::from("trace.json"));
|
||||
fn sample_trace(state_id: &str, body: &str) -> Trace {
|
||||
Trace {
|
||||
version: TRACE_VERSION,
|
||||
recorded_at: "2026-08-10T02:12:55.360Z".into(),
|
||||
stopped_by: StopReason::UserFinish,
|
||||
entry: TraceEntry {
|
||||
start_url: "https://example.com/".into(),
|
||||
},
|
||||
recorder: RecorderInfo {
|
||||
bsk: "0.1.10".into(),
|
||||
vom: VOM_FORMAT_VERSION,
|
||||
},
|
||||
states: vec![TraceState {
|
||||
id: state_id.into(),
|
||||
url: "https://example.com/".into(),
|
||||
title: Some("Example".into()),
|
||||
body: Some(body.into()),
|
||||
page: None,
|
||||
truncated: false,
|
||||
}],
|
||||
steps: vec![Step::Navigate {
|
||||
common: StepCommon {
|
||||
id: 1,
|
||||
state: state_id.into(),
|
||||
result: StepResult {
|
||||
state: state_id.into(),
|
||||
},
|
||||
},
|
||||
to: "https://example.com/".into(),
|
||||
cause: NavigationCause::UserTyped,
|
||||
}],
|
||||
purpose: None,
|
||||
started_at: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn start_args_default_output_is_trace_json() {
|
||||
fn concurrent_teardown_is_not_a_record_failure() {
|
||||
let stopping = CliError::from_rpc(bsk_protocol::RpcError {
|
||||
code: ErrorCode::Timeout,
|
||||
message: "session stop is already in progress".into(),
|
||||
data: Some(serde_json::json!({ "reason": STOP_IN_PROGRESS_REASON })),
|
||||
});
|
||||
assert!(session_teardown_raced(&stopping));
|
||||
|
||||
let gone = CliError::from_rpc(bsk_protocol::RpcError {
|
||||
code: ErrorCode::NotFound,
|
||||
message: "session is not registered".into(),
|
||||
data: None,
|
||||
});
|
||||
assert!(session_teardown_raced(&gone));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unrelated_stop_timeout_still_fails() {
|
||||
let stalled = CliError::from_rpc(bsk_protocol::RpcError {
|
||||
code: ErrorCode::Timeout,
|
||||
message: "tool.session_stop timed out".into(),
|
||||
data: None,
|
||||
});
|
||||
assert!(!session_teardown_raced(&stalled));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_output_is_trace_dir() {
|
||||
let args = RecordStopArgs {
|
||||
output: PathBuf::from("trace"),
|
||||
};
|
||||
assert_eq!(args.output, PathBuf::from("trace"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn start_args_default_output_is_trace_dir() {
|
||||
let args = RecordStartArgs {
|
||||
browser: None,
|
||||
tab_id: None,
|
||||
url: None,
|
||||
purpose: None,
|
||||
output: PathBuf::from("trace.json"),
|
||||
max_page_tokens: None,
|
||||
redact_values: false,
|
||||
output: PathBuf::from("trace"),
|
||||
};
|
||||
assert_eq!(args.output, PathBuf::from("trace.json"));
|
||||
assert_eq!(args.output, PathBuf::from("trace"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_trace_bundle_writes_dir_layout() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let output = dir.path().join("flow");
|
||||
let trace = sample_trace("s1", "@vom 1\nL1 page");
|
||||
let pages_dir = write_trace_bundle(&output, &trace).unwrap();
|
||||
let trace_path = output.join("trace.json");
|
||||
assert!(trace_path.exists());
|
||||
assert_eq!(pages_dir, output.join("pages"));
|
||||
assert!(pages_dir.join("s1.vom.txt").exists());
|
||||
let disk: serde_json::Value =
|
||||
serde_json::from_str(&fs::read_to_string(&trace_path).unwrap()).unwrap();
|
||||
assert_eq!(disk["states"][0]["page"], "s1.vom.txt");
|
||||
assert!(disk["states"][0].get("body").is_none());
|
||||
assert_eq!(
|
||||
fs::read_to_string(pages_dir.join("s1.vom.txt")).unwrap(),
|
||||
"@vom 1\nL1 page"
|
||||
);
|
||||
|
||||
// Re-exporting drops pages from the previous run but keeps the
|
||||
// directory in place for the concurrent writer.
|
||||
fs::write(pages_dir.join("s9.vom.txt"), "stale").unwrap();
|
||||
fs::write(pages_dir.join("notes.md"), "keep me").unwrap();
|
||||
write_trace_bundle(&output, &trace).unwrap();
|
||||
assert!(!pages_dir.join("s9.vom.txt").exists());
|
||||
assert!(pages_dir.join("s1.vom.txt").exists());
|
||||
assert!(pages_dir.join("notes.md").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_trace_bundle_rejects_traversal_state_id() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let output = dir.path().join("flow");
|
||||
let trace = sample_trace("../escaped", "@vom 1\nL1 page");
|
||||
|
||||
let err = write_trace_bundle(&output, &trace).unwrap_err();
|
||||
|
||||
assert!(err.to_string().contains("state id"));
|
||||
assert!(!output.join("escaped.vom.txt").exists());
|
||||
assert!(!output.join("trace.json").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_trace_bundle_rejects_duplicate_state_id() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let output = dir.path().join("flow");
|
||||
let mut trace = sample_trace("s1", "@vom 1\nL1 first");
|
||||
trace.states.push(TraceState {
|
||||
body: Some("@vom 1\nL1 duplicate".into()),
|
||||
..trace.states[0].clone()
|
||||
});
|
||||
|
||||
let err = write_trace_bundle(&output, &trace).unwrap_err();
|
||||
|
||||
assert!(err.to_string().contains("duplicate state id"));
|
||||
assert!(!output.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsupported_version_preserves_existing_bundle() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let output = dir.path().join("flow");
|
||||
write_trace_bundle(&output, &sample_trace("s1", "original")).unwrap();
|
||||
let original_json = fs::read(output.join("trace.json")).unwrap();
|
||||
|
||||
let mut replacement = sample_trace("s2", "replacement");
|
||||
replacement.version += 1;
|
||||
let err = write_trace_bundle(&output, &replacement).unwrap_err();
|
||||
|
||||
assert!(err.to_string().contains("not supported"));
|
||||
assert_eq!(fs::read(output.join("trace.json")).unwrap(), original_json);
|
||||
assert_eq!(
|
||||
fs::read_to_string(output.join("pages/s1.vom.txt")).unwrap(),
|
||||
"original"
|
||||
);
|
||||
assert!(!output.join("pages/s2.vom.txt").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_replacement_failure_preserves_existing_bundle() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let output = dir.path().join("flow");
|
||||
write_trace_bundle(&output, &sample_trace("s1", "original")).unwrap();
|
||||
let original_json = fs::read(output.join("trace.json")).unwrap();
|
||||
fs::write(output.join("pages/notes.md"), "keep me").unwrap();
|
||||
fs::create_dir(output.join("pages/s2.vom.txt")).unwrap();
|
||||
|
||||
let err = write_trace_bundle(&output, &sample_trace("s2", "replacement")).unwrap_err();
|
||||
|
||||
assert!(err.to_string().contains("page"));
|
||||
assert_eq!(fs::read(output.join("trace.json")).unwrap(), original_json);
|
||||
assert_eq!(
|
||||
fs::read_to_string(output.join("pages/s1.vom.txt")).unwrap(),
|
||||
"original"
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read_to_string(output.join("pages/notes.md")).unwrap(),
|
||||
"keep me"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn write_trace_bundle_rejects_symlinked_pages_directory() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let output = dir.path().join("flow");
|
||||
let external_pages = dir.path().join("external-pages");
|
||||
fs::create_dir_all(&output).unwrap();
|
||||
fs::create_dir_all(&external_pages).unwrap();
|
||||
fs::write(external_pages.join("s9.vom.txt"), "external").unwrap();
|
||||
symlink(&external_pages, output.join("pages")).unwrap();
|
||||
|
||||
let err = write_trace_bundle(&output, &sample_trace("s1", "replacement")).unwrap_err();
|
||||
|
||||
assert!(err.to_string().contains("symlink"));
|
||||
assert_eq!(
|
||||
fs::read_to_string(external_pages.join("s9.vom.txt")).unwrap(),
|
||||
"external"
|
||||
);
|
||||
assert!(!external_pages.join("s1.vom.txt").exists());
|
||||
assert!(!output.join("trace.json").exists());
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn write_trace_bundle_rejects_symlinked_output_directory() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let external_output = dir.path().join("external-output");
|
||||
let output = dir.path().join("flow");
|
||||
fs::create_dir_all(&external_output).unwrap();
|
||||
fs::write(external_output.join("notes.md"), "external").unwrap();
|
||||
symlink(&external_output, &output).unwrap();
|
||||
|
||||
let err = write_trace_bundle(&output, &sample_trace("s1", "replacement")).unwrap_err();
|
||||
|
||||
assert!(err.to_string().contains("symlink"));
|
||||
assert_eq!(
|
||||
fs::read_to_string(external_output.join("notes.md")).unwrap(),
|
||||
"external"
|
||||
);
|
||||
assert!(!external_output.join("trace.json").exists());
|
||||
assert!(!external_output.join("pages").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_trace_v2_writes_single_json_and_prunes_stale_pages() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let output = dir.path().join("flow");
|
||||
fs::create_dir_all(output.join("pages")).unwrap();
|
||||
fs::write(output.join("pages/s1.vom.txt"), "stale").unwrap();
|
||||
|
||||
let trace = TraceV2 {
|
||||
recorded_at: "2026-08-10T02:12:55.360Z".into(),
|
||||
started_at: None,
|
||||
purpose: None,
|
||||
entry: bsk_protocol::tools::TraceEntry {
|
||||
start_url: "https://example.com/".into(),
|
||||
},
|
||||
pages: vec![bsk_protocol::tools::PageRef {
|
||||
id: "p1".into(),
|
||||
url: "https://example.com/".into(),
|
||||
title: None,
|
||||
}],
|
||||
steps: vec![],
|
||||
};
|
||||
write_trace_v2(&output, &trace).unwrap();
|
||||
assert!(output.join("trace.json").exists());
|
||||
assert!(!output.join("pages/s1.vom.txt").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn export_recorded_trace_accepts_v2() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let output = dir.path().join("flow");
|
||||
let trace = RecordedTrace::V2(TraceV2 {
|
||||
recorded_at: "2026-08-10T02:12:55.360Z".into(),
|
||||
started_at: None,
|
||||
purpose: None,
|
||||
entry: bsk_protocol::tools::TraceEntry {
|
||||
start_url: "https://example.com/".into(),
|
||||
},
|
||||
pages: vec![],
|
||||
steps: vec![],
|
||||
});
|
||||
let exported = export_recorded_trace(&output, &trace).unwrap();
|
||||
assert_eq!(exported.trace_version, TRACE_VERSION_V2);
|
||||
assert!(exported.pages_dir.is_none());
|
||||
assert!(output.join("trace.json").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -141,9 +141,9 @@ pub fn info_for(code: ErrorCode) -> RenderInfo {
|
||||
exit_code: 2,
|
||||
},
|
||||
ErrorCode::VersionTooOld => RenderInfo {
|
||||
summary: "peer version is too old to communicate with this build",
|
||||
summary: "the installed bsk CLI and BrowserSkill extension are incompatible",
|
||||
hint: Some(
|
||||
"upgrade both bsk CLI and the browser-skill extension so both sides satisfy `min_compatible_protocol`",
|
||||
"update bsk and the BrowserSkill extension, reload the extension, then run `bsk daemon restart`",
|
||||
),
|
||||
exit_code: 5,
|
||||
},
|
||||
@@ -157,7 +157,7 @@ pub fn info_for(code: ErrorCode) -> RenderInfo {
|
||||
ErrorCode::NoBrowserConnected => RenderInfo {
|
||||
summary: "no browser is connected to the daemon",
|
||||
hint: Some(
|
||||
"open the browser-skill extension in your browser and wait for the popup to show \"connected\"",
|
||||
"open the BrowserSkill extension and wait for it to show \"connected\"; if the extension is already open, update and reload it, then run `bsk daemon restart`",
|
||||
),
|
||||
exit_code: 1,
|
||||
},
|
||||
@@ -310,6 +310,30 @@ mod tests {
|
||||
assert_eq!(exit_code_for(ErrorCode::VersionTooOld), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_error_names_the_components_and_gives_update_steps() {
|
||||
let info = info_for(ErrorCode::VersionTooOld);
|
||||
assert!(info.summary.contains("bsk CLI"));
|
||||
assert!(info.summary.contains("BrowserSkill extension"));
|
||||
let hint = info
|
||||
.hint
|
||||
.expect("version mismatch should have a repair hint");
|
||||
assert!(hint.contains("update bsk"));
|
||||
assert!(hint.contains("reload the extension"));
|
||||
assert!(hint.contains("bsk daemon restart"));
|
||||
assert!(!hint.contains("min_compatible_protocol"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_browser_hint_explains_version_mismatch_recovery() {
|
||||
let hint = info_for(ErrorCode::NoBrowserConnected)
|
||||
.hint
|
||||
.expect("no browser should have a repair hint");
|
||||
assert!(hint.contains("extension is already open"));
|
||||
assert!(hint.contains("update"));
|
||||
assert!(hint.contains("bsk daemon restart"));
|
||||
}
|
||||
|
||||
/// Every code must carry a non-empty summary; an empty `summary`
|
||||
/// would render `error: ` with nothing after it.
|
||||
#[test]
|
||||
|
||||
@@ -786,7 +786,19 @@ async fn handle_session_stop(state: &Arc<DaemonState>, params: Value) -> Result<
|
||||
}
|
||||
}
|
||||
|
||||
/// Marks a `session.stop` rejected because another teardown already owns the
|
||||
/// session, so callers that race on purpose (`bsk record`) can tell it apart
|
||||
/// from a browser that stopped responding.
|
||||
pub const STOP_IN_PROGRESS_REASON: &str = "session_stop_in_progress";
|
||||
|
||||
fn map_stop_error(err: StopSessionError) -> RpcError {
|
||||
if matches!(err, StopSessionError::Stopping) {
|
||||
return RpcError {
|
||||
code: ErrorCode::Timeout,
|
||||
message: err.to_string(),
|
||||
data: Some(serde_json::json!({ "reason": STOP_IN_PROGRESS_REASON })),
|
||||
};
|
||||
}
|
||||
let code = match &err {
|
||||
StopSessionError::NotFound => ErrorCode::NotFound,
|
||||
StopSessionError::Stopping => ErrorCode::Timeout,
|
||||
|
||||
@@ -27,7 +27,7 @@ use crate::daemon::{
|
||||
browsers::{BROWSER_LIVENESS_TICK, BROWSER_LIVENESS_TIMEOUT, EXTENSION_CONNECT_WAIT},
|
||||
info as daemon_info, ipc, lockfile, paths,
|
||||
sessions::{StopSessionError, forget_session, stop_session},
|
||||
state::DaemonState,
|
||||
state::{DaemonState, PROTOCOL_VERSION},
|
||||
ws,
|
||||
};
|
||||
|
||||
@@ -299,7 +299,7 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
|
||||
ws_port,
|
||||
sock_path: sock_path.clone(),
|
||||
daemon_version: env!("CARGO_PKG_VERSION"),
|
||||
protocol_version: "1.0",
|
||||
protocol_version: PROTOCOL_VERSION,
|
||||
};
|
||||
let handler = ipc::full_handler(status, Arc::clone(&state));
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@ use super::start::DaemonConfig;
|
||||
use super::ws::WsHandle;
|
||||
|
||||
pub const DAEMON_VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
pub const PROTOCOL_VERSION: &str = "1.0";
|
||||
pub const PROTOCOL_VERSION: &str = "1.1";
|
||||
/// Lowest **protocol** version peers must speak (e.g. `"1.0"`).
|
||||
pub const MIN_COMPATIBLE_PROTOCOL: &str = "1.0";
|
||||
/// Legacy app-semver floor used only when `HandshakeResult.min_compatible_peer`
|
||||
|
||||
@@ -112,14 +112,14 @@ async fn handshake_as_ext(
|
||||
let params = HandshakeParams {
|
||||
client: "browser-skill-extension".into(),
|
||||
version: "0.1.0-dev.0".parse().unwrap(),
|
||||
protocol_version: "1.0".into(),
|
||||
protocol_version: "1.1".into(),
|
||||
instance_id: TEST_EXT_ID.into(),
|
||||
browser: BrowserPeerInfo {
|
||||
name: "chrome".into(),
|
||||
version: "131.0".into(),
|
||||
},
|
||||
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
||||
min_compatible_protocol: Some("1.0".into()),
|
||||
min_compatible_protocol: Some("1.1".into()),
|
||||
label: "Test".into(),
|
||||
};
|
||||
let req = RequestFrame {
|
||||
|
||||
@@ -75,7 +75,7 @@ async fn handshake_as_ext(
|
||||
let params = HandshakeParams {
|
||||
client: "browser-skill-extension".into(),
|
||||
version: "0.1.0-dev.0".parse().unwrap(),
|
||||
protocol_version: "1.0".into(),
|
||||
protocol_version: "1.1".into(),
|
||||
instance_id: TEST_EXT_ID.into(),
|
||||
browser: BrowserPeerInfo {
|
||||
name: "chrome".into(),
|
||||
@@ -83,7 +83,7 @@ async fn handshake_as_ext(
|
||||
},
|
||||
label: "Test".into(),
|
||||
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
||||
min_compatible_protocol: Some("1.0".into()),
|
||||
min_compatible_protocol: Some("1.1".into()),
|
||||
};
|
||||
let req = RequestFrame {
|
||||
id: "hs".into(),
|
||||
|
||||
@@ -108,12 +108,12 @@ async fn send_handshake_with_floors(
|
||||
async fn handshake_ok_when_protocol_matches() {
|
||||
let (handle, _sock) = spawn_daemon().await;
|
||||
let mut ws = open_ws(handle.ws_addr()).await;
|
||||
let resp = send_handshake(&mut ws, "1.0", env!("CARGO_PKG_VERSION")).await;
|
||||
let resp = send_handshake(&mut ws, "1.1", env!("CARGO_PKG_VERSION")).await;
|
||||
let result: HandshakeResult = match resp.body {
|
||||
ResponseBody::Ok(v) => serde_json::from_value(v).unwrap(),
|
||||
ResponseBody::Err(e) => panic!("expected ok handshake, got {e:?}"),
|
||||
};
|
||||
assert_eq!(result.protocol_version, "1.0");
|
||||
assert_eq!(result.protocol_version, "1.1");
|
||||
assert_eq!(
|
||||
result
|
||||
.min_compatible_peer
|
||||
@@ -135,7 +135,7 @@ async fn handshake_ok_when_app_versions_differ_but_protocol_matches() {
|
||||
let (handle, _sock) = spawn_daemon().await;
|
||||
let mut ws = open_ws(handle.ws_addr()).await;
|
||||
let resp =
|
||||
send_handshake_with_floors(&mut ws, "1.0", "9.9.9", Some("0.0.0"), Some("1.0")).await;
|
||||
send_handshake_with_floors(&mut ws, "1.1", "9.9.9", Some("0.0.0"), Some("1.1")).await;
|
||||
match resp.body {
|
||||
ResponseBody::Ok(_) => {}
|
||||
other => panic!("expected ok when protocol matches, got {other:?}"),
|
||||
@@ -149,7 +149,32 @@ async fn handshake_skew_when_protocol_minor_differs() {
|
||||
let mut ws = open_ws(handle.ws_addr()).await;
|
||||
let resp = send_handshake_with_floors(
|
||||
&mut ws,
|
||||
"1.1",
|
||||
"1.2",
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
Some("0.0.0"),
|
||||
Some("1.1"),
|
||||
)
|
||||
.await;
|
||||
match resp.body {
|
||||
ResponseBody::Ok(_) => {}
|
||||
other => panic!("minor protocol drift should warn-but-allow, got {other:?}"),
|
||||
}
|
||||
let state = handle.state();
|
||||
let client = state
|
||||
.browsers
|
||||
.get(&bsk::daemon::browsers::BrowserId(TEST_EXT_ID.into()))
|
||||
.expect("browser registered");
|
||||
assert!(client.version_skew);
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn handshake_skew_when_protocol_1_0_peer() {
|
||||
let (handle, _sock) = spawn_daemon().await;
|
||||
let mut ws = open_ws(handle.ws_addr()).await;
|
||||
let resp = send_handshake_with_floors(
|
||||
&mut ws,
|
||||
"1.0",
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
Some("0.0.0"),
|
||||
Some("1.0"),
|
||||
@@ -157,7 +182,7 @@ async fn handshake_skew_when_protocol_minor_differs() {
|
||||
.await;
|
||||
match resp.body {
|
||||
ResponseBody::Ok(_) => {}
|
||||
other => panic!("minor protocol drift should warn-but-allow, got {other:?}"),
|
||||
other => panic!("protocol 1.0 peer should connect with skew, got {other:?}"),
|
||||
}
|
||||
let state = handle.state();
|
||||
let client = state
|
||||
@@ -210,7 +235,7 @@ async fn handshake_legacy_ext_without_protocol_floor_still_ok() {
|
||||
let mut ws = open_ws(handle.ws_addr()).await;
|
||||
let resp = send_handshake_with_floors(
|
||||
&mut ws,
|
||||
"1.0",
|
||||
"1.1",
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
Some("0.1.0"),
|
||||
None,
|
||||
@@ -235,7 +260,7 @@ async fn status_surfaces_version_skew_for_skewed_browser() {
|
||||
browser_name: "chrome".into(),
|
||||
browser_version: "131.0".into(),
|
||||
extension_version: "9.9.9".into(),
|
||||
extension_protocol_version: "1.1".into(),
|
||||
extension_protocol_version: "1.2".into(),
|
||||
label: "Older".into(),
|
||||
sink: bsk::daemon::browsers::BrowserSink { tx },
|
||||
pending: Mutex::new(bsk::daemon::browsers::Pending::default()),
|
||||
@@ -263,8 +288,8 @@ async fn status_surfaces_version_skew_for_skewed_browser() {
|
||||
.iter()
|
||||
.find(|s| s.instance_id == "skew-only-test")
|
||||
.expect("status must list our skew client");
|
||||
assert_eq!(skew.client_protocol_version, "1.1");
|
||||
assert_eq!(skew.server_protocol_version, "1.0");
|
||||
assert_eq!(skew.client_protocol_version, "1.2");
|
||||
assert_eq!(skew.server_protocol_version, "1.1");
|
||||
assert_eq!(skew.client_version, "9.9.9");
|
||||
let entry = status
|
||||
.browsers
|
||||
@@ -281,7 +306,7 @@ async fn handshake_rejects_when_local_below_peer_min_compatible_protocol() {
|
||||
let mut ws = open_ws(handle.ws_addr()).await;
|
||||
let resp = send_handshake_with_floors(
|
||||
&mut ws,
|
||||
"1.0",
|
||||
"1.1",
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
Some("0.0.0"),
|
||||
Some("99.0.0"),
|
||||
|
||||
@@ -81,14 +81,14 @@ async fn handshake_as_ext(
|
||||
let params = HandshakeParams {
|
||||
client: "browser-skill-extension".into(),
|
||||
version: "0.1.0-dev.0".parse().unwrap(),
|
||||
protocol_version: "1.0".into(),
|
||||
protocol_version: "1.1".into(),
|
||||
instance_id: TEST_EXT_ID.into(),
|
||||
browser: BrowserPeerInfo {
|
||||
name: "chrome".into(),
|
||||
version: "131.0".into(),
|
||||
},
|
||||
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
||||
min_compatible_protocol: Some("1.0".into()),
|
||||
min_compatible_protocol: Some("1.1".into()),
|
||||
label: "Test".into(),
|
||||
};
|
||||
let req = RequestFrame {
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
use bsk::cli::record_state;
|
||||
use bsk::daemon::info::DaemonInfo;
|
||||
use bsk::daemon::paths::BSK_HOME_ENV;
|
||||
use bsk::daemon::{self, DaemonConfig};
|
||||
use bsk::ipc_client::IpcClient;
|
||||
use bsk_protocol::system::{HandshakeParams, HandshakeResult};
|
||||
use bsk_protocol::tools::{
|
||||
RecordStopResult, RecordedTrace, RecorderInfo, SessionStartParams, SessionStartResult,
|
||||
StopReason, Trace, TraceEntry, TraceState, VOM_FORMAT_VERSION,
|
||||
};
|
||||
use bsk_protocol::{
|
||||
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
|
||||
};
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use serde_json::json;
|
||||
use tokio::process::Command;
|
||||
use tokio_tungstenite::tungstenite::handshake::client::generate_key;
|
||||
use tokio_tungstenite::tungstenite::http::Request;
|
||||
use tokio_tungstenite::tungstenite::protocol::Message;
|
||||
|
||||
const TEST_EXT_ID: &str = "abcdefghijklmnopabcdefghijklmnop";
|
||||
|
||||
type Ws =
|
||||
tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>>;
|
||||
|
||||
fn bsk_bin() -> PathBuf {
|
||||
PathBuf::from(env!("CARGO_BIN_EXE_bsk"))
|
||||
}
|
||||
|
||||
async fn connect_extension(addr: std::net::SocketAddr) -> Ws {
|
||||
let origin = format!("chrome-extension://{TEST_EXT_ID}");
|
||||
let request = Request::builder()
|
||||
.method("GET")
|
||||
.uri(format!("ws://{addr}/"))
|
||||
.header("Host", addr.to_string())
|
||||
.header("Upgrade", "websocket")
|
||||
.header("Connection", "Upgrade")
|
||||
.header("Sec-WebSocket-Version", "13")
|
||||
.header("Sec-WebSocket-Key", generate_key())
|
||||
.header("Origin", origin)
|
||||
.body(())
|
||||
.unwrap();
|
||||
tokio_tungstenite::connect_async(request).await.unwrap().0
|
||||
}
|
||||
|
||||
async fn handshake(ws: &mut Ws) {
|
||||
let request = RequestFrame {
|
||||
id: "handshake".into(),
|
||||
method: Method::SystemHandshake,
|
||||
params: Some(
|
||||
serde_json::to_value(HandshakeParams {
|
||||
client: "browser-skill-extension".into(),
|
||||
version: "0.1.10".parse().unwrap(),
|
||||
protocol_version: "1.1".into(),
|
||||
instance_id: TEST_EXT_ID.into(),
|
||||
browser: BrowserPeerInfo {
|
||||
name: "chrome".into(),
|
||||
version: "131.0".into(),
|
||||
},
|
||||
min_compatible_peer: Some("0.1.10".parse().unwrap()),
|
||||
min_compatible_protocol: Some("1.1".into()),
|
||||
label: "Record stop retry test".into(),
|
||||
})
|
||||
.unwrap(),
|
||||
),
|
||||
};
|
||||
ws.send(Message::Text(serde_json::to_string(&request).unwrap()))
|
||||
.await
|
||||
.unwrap();
|
||||
let Message::Text(response) = ws.next().await.unwrap().unwrap() else {
|
||||
panic!("expected text handshake response");
|
||||
};
|
||||
let response: ResponseFrame = serde_json::from_str(&response).unwrap();
|
||||
let ResponseBody::Ok(value) = response.body else {
|
||||
panic!("handshake failed");
|
||||
};
|
||||
let _: HandshakeResult = serde_json::from_value(value).unwrap();
|
||||
}
|
||||
|
||||
fn sample_trace() -> Trace {
|
||||
Trace {
|
||||
version: 3,
|
||||
purpose: Some("retry record stop".into()),
|
||||
started_at: Some("2026-08-12T10:00:00Z".into()),
|
||||
recorded_at: "2026-08-12T10:01:00Z".into(),
|
||||
stopped_by: StopReason::CliStop,
|
||||
entry: TraceEntry {
|
||||
start_url: "https://example.com/".into(),
|
||||
},
|
||||
recorder: RecorderInfo {
|
||||
bsk: "0.1.10".into(),
|
||||
vom: VOM_FORMAT_VERSION,
|
||||
},
|
||||
states: vec![TraceState {
|
||||
id: "s1".into(),
|
||||
url: "https://example.com/".into(),
|
||||
title: Some("Example".into()),
|
||||
body: Some("@vom 1\nL1 page".into()),
|
||||
page: None,
|
||||
truncated: false,
|
||||
}],
|
||||
steps: vec![],
|
||||
}
|
||||
}
|
||||
|
||||
fn run_extension(
|
||||
mut ws: Ws,
|
||||
record_stop_calls: Arc<AtomicUsize>,
|
||||
session_stop_calls: Arc<AtomicUsize>,
|
||||
) {
|
||||
tokio::spawn(async move {
|
||||
while let Some(Ok(Message::Text(text))) = ws.next().await {
|
||||
let Ok(Frame::Request(request)) = serde_json::from_str::<Frame>(&text) else {
|
||||
continue;
|
||||
};
|
||||
let body = match request.method {
|
||||
Method::ToolSessionStart => {
|
||||
let _: SessionStartParams =
|
||||
serde_json::from_value(request.params.clone().unwrap()).unwrap();
|
||||
ResponseBody::Ok(
|
||||
serde_json::to_value(SessionStartResult {
|
||||
agent_window_id: Some(100),
|
||||
})
|
||||
.unwrap(),
|
||||
)
|
||||
}
|
||||
Method::ToolRecordStop => {
|
||||
if record_stop_calls.fetch_add(1, Ordering::SeqCst) == 0 {
|
||||
ResponseBody::Err(RpcError {
|
||||
code: ErrorCode::ProtocolError,
|
||||
message: "recording remains active; retry record stop".into(),
|
||||
data: Some(json!({ "recording": true, "retryable": true })),
|
||||
})
|
||||
} else {
|
||||
ResponseBody::Ok(
|
||||
serde_json::to_value(RecordStopResult {
|
||||
trace: RecordedTrace::V3(sample_trace()),
|
||||
})
|
||||
.unwrap(),
|
||||
)
|
||||
}
|
||||
}
|
||||
Method::ToolSessionStop => {
|
||||
session_stop_calls.fetch_add(1, Ordering::SeqCst);
|
||||
ResponseBody::Ok(json!({}))
|
||||
}
|
||||
_ => ResponseBody::Err(RpcError {
|
||||
code: ErrorCode::ProtocolError,
|
||||
message: format!("unexpected method {:?}", request.method),
|
||||
data: None,
|
||||
}),
|
||||
};
|
||||
let response = ResponseFrame {
|
||||
id: request.id,
|
||||
body,
|
||||
};
|
||||
ws.send(Message::Text(serde_json::to_string(&response).unwrap()))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
async fn start_session(sock: &Path) -> String {
|
||||
#[derive(serde::Deserialize)]
|
||||
struct StartResult {
|
||||
session_id: String,
|
||||
}
|
||||
|
||||
let mut client = IpcClient::connect(sock).await.unwrap();
|
||||
client
|
||||
.call::<(), StartResult>(
|
||||
"session-start",
|
||||
Method::SessionStart,
|
||||
None,
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.session_id
|
||||
}
|
||||
|
||||
async fn run_record_stop(home: &Path, output: &Path) -> std::process::Output {
|
||||
Command::new(bsk_bin())
|
||||
.arg("record")
|
||||
.arg("stop")
|
||||
.arg("--output")
|
||||
.arg(output)
|
||||
.env(BSK_HOME_ENV, home)
|
||||
.output()
|
||||
.await
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn failed_record_stop_keeps_state_and_session_for_retry() {
|
||||
let temp = tempfile::Builder::new()
|
||||
.prefix(".record-stop-retry-")
|
||||
.tempdir_in(env!("CARGO_MANIFEST_DIR"))
|
||||
.unwrap();
|
||||
let home = temp.path().join("bsk-home");
|
||||
let sock = temp.path().join("daemon.sock");
|
||||
let output = temp.path().join("trace");
|
||||
|
||||
let daemon = daemon::run(DaemonConfig::new(0), Some(sock.clone()))
|
||||
.await
|
||||
.unwrap();
|
||||
bsk::daemon::info::write_to_path(
|
||||
&DaemonInfo::now(
|
||||
std::process::id(),
|
||||
sock.clone(),
|
||||
daemon.ws_addr().port(),
|
||||
env!("CARGO_PKG_VERSION"),
|
||||
),
|
||||
&home.join("daemon.json"),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let mut extension = connect_extension(daemon.ws_addr()).await;
|
||||
handshake(&mut extension).await;
|
||||
let record_stop_calls = Arc::new(AtomicUsize::new(0));
|
||||
let session_stop_calls = Arc::new(AtomicUsize::new(0));
|
||||
run_extension(
|
||||
extension,
|
||||
Arc::clone(&record_stop_calls),
|
||||
Arc::clone(&session_stop_calls),
|
||||
);
|
||||
|
||||
let session_id = start_session(&sock).await;
|
||||
unsafe {
|
||||
std::env::set_var(BSK_HOME_ENV, &home);
|
||||
}
|
||||
record_state::write(&session_id).unwrap();
|
||||
|
||||
let first = run_record_stop(&home, &output).await;
|
||||
assert!(
|
||||
!first.status.success(),
|
||||
"first stop unexpectedly succeeded: {}",
|
||||
String::from_utf8_lossy(&first.stdout)
|
||||
);
|
||||
assert_eq!(record_state::read().unwrap().session_id, session_id);
|
||||
assert_eq!(record_stop_calls.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(
|
||||
session_stop_calls.load(Ordering::SeqCst),
|
||||
0,
|
||||
"failed record stop must leave the session active"
|
||||
);
|
||||
|
||||
let second = run_record_stop(&home, &output).await;
|
||||
assert!(
|
||||
second.status.success(),
|
||||
"retry failed: {}",
|
||||
String::from_utf8_lossy(&second.stderr)
|
||||
);
|
||||
assert!(record_state::read().is_err());
|
||||
assert_eq!(record_stop_calls.load(Ordering::SeqCst), 2);
|
||||
assert_eq!(session_stop_calls.load(Ordering::SeqCst), 1);
|
||||
assert!(output.join("trace.json").exists());
|
||||
assert!(output.join("pages/s1.vom.txt").exists());
|
||||
|
||||
unsafe {
|
||||
std::env::remove_var(BSK_HOME_ENV);
|
||||
}
|
||||
daemon.shutdown().await;
|
||||
}
|
||||
@@ -81,7 +81,7 @@ async fn handshake_as_ext(
|
||||
let params = HandshakeParams {
|
||||
client: "browser-skill-extension".into(),
|
||||
version: "0.1.0-dev.0".parse().unwrap(),
|
||||
protocol_version: "1.0".into(),
|
||||
protocol_version: "1.1".into(),
|
||||
instance_id: TEST_EXT_ID.into(),
|
||||
browser: BrowserPeerInfo {
|
||||
name: "chrome".into(),
|
||||
@@ -89,7 +89,7 @@ async fn handshake_as_ext(
|
||||
},
|
||||
label: "Test".into(),
|
||||
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
||||
min_compatible_protocol: Some("1.0".into()),
|
||||
min_compatible_protocol: Some("1.1".into()),
|
||||
};
|
||||
let req = RequestFrame {
|
||||
id: "hs".into(),
|
||||
|
||||
@@ -84,14 +84,14 @@ async fn handshake_as_ext(
|
||||
let params = HandshakeParams {
|
||||
client: "browser-skill-extension".into(),
|
||||
version: "0.1.0-dev.0".parse().unwrap(),
|
||||
protocol_version: "1.0".into(),
|
||||
protocol_version: "1.1".into(),
|
||||
instance_id: TEST_EXT_ID.into(),
|
||||
browser: BrowserPeerInfo {
|
||||
name: "chrome".into(),
|
||||
version: "131.0".into(),
|
||||
},
|
||||
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
||||
min_compatible_protocol: Some("1.0".into()),
|
||||
min_compatible_protocol: Some("1.1".into()),
|
||||
label: "Test".into(),
|
||||
};
|
||||
let req = RequestFrame {
|
||||
@@ -400,14 +400,14 @@ async fn connect_second_ext(
|
||||
let params = HandshakeParams {
|
||||
client: "browser-skill-extension".into(),
|
||||
version: "0.1.0-dev.0".parse().unwrap(),
|
||||
protocol_version: "1.0".into(),
|
||||
protocol_version: "1.1".into(),
|
||||
instance_id: instance_id.into(),
|
||||
browser: BrowserPeerInfo {
|
||||
name: "edge".into(),
|
||||
version: "130".into(),
|
||||
},
|
||||
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
||||
min_compatible_protocol: Some("1.0".into()),
|
||||
min_compatible_protocol: Some("1.1".into()),
|
||||
label: label.into(),
|
||||
};
|
||||
let hs = RequestFrame {
|
||||
|
||||
@@ -61,7 +61,7 @@ fn bsk_status_json_returns_structured_payload() {
|
||||
|
||||
assert!(parsed["pid"].as_u64().unwrap() > 0);
|
||||
assert!(!parsed["daemon_version"].as_str().unwrap().is_empty());
|
||||
assert_eq!(parsed["protocol_version"], "1.0");
|
||||
assert_eq!(parsed["protocol_version"], "1.1");
|
||||
assert!(parsed["sock_path"].as_str().is_some());
|
||||
assert_eq!(parsed["browsers"], serde_json::json!([]));
|
||||
assert_eq!(parsed["sessions"], serde_json::json!([]));
|
||||
|
||||
@@ -73,14 +73,14 @@ async fn do_handshake(ws: &mut Ws) -> HandshakeResult {
|
||||
let params = HandshakeParams {
|
||||
client: "browser-skill-extension".into(),
|
||||
version: "0.1.0-dev.0".parse().unwrap(),
|
||||
protocol_version: "1.0".into(),
|
||||
protocol_version: "1.1".into(),
|
||||
instance_id: TEST_EXT_ID.into(),
|
||||
browser: BrowserPeerInfo {
|
||||
name: "chrome".into(),
|
||||
version: "131.0".into(),
|
||||
},
|
||||
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
||||
min_compatible_protocol: Some("1.0".into()),
|
||||
min_compatible_protocol: Some("1.1".into()),
|
||||
label: "Test".into(),
|
||||
};
|
||||
let req = RequestFrame {
|
||||
|
||||
@@ -76,14 +76,14 @@ async fn do_handshake(ws: &mut Ws) -> HandshakeResult {
|
||||
let params = HandshakeParams {
|
||||
client: "browser-skill-extension".into(),
|
||||
version: "0.1.0-dev.0".parse().unwrap(),
|
||||
protocol_version: "1.0".into(),
|
||||
protocol_version: "1.1".into(),
|
||||
instance_id: TEST_EXT_ID.into(),
|
||||
browser: BrowserPeerInfo {
|
||||
name: "chrome".into(),
|
||||
version: "131.0".into(),
|
||||
},
|
||||
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
||||
min_compatible_protocol: Some("1.0".into()),
|
||||
min_compatible_protocol: Some("1.1".into()),
|
||||
label: "Test".into(),
|
||||
};
|
||||
let req = RequestFrame {
|
||||
|
||||
@@ -75,14 +75,14 @@ async fn do_handshake(ws: &mut Ws) -> HandshakeResult {
|
||||
let params = HandshakeParams {
|
||||
client: "browser-skill-extension".into(),
|
||||
version: "0.1.0-dev.0".parse().unwrap(),
|
||||
protocol_version: "1.0".into(),
|
||||
protocol_version: "1.1".into(),
|
||||
instance_id: TEST_EXT_ID.into(),
|
||||
browser: BrowserPeerInfo {
|
||||
name: "chrome".into(),
|
||||
version: "131.0".into(),
|
||||
},
|
||||
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
||||
min_compatible_protocol: Some("1.0".into()),
|
||||
min_compatible_protocol: Some("1.1".into()),
|
||||
label: "Test".into(),
|
||||
};
|
||||
let req = RequestFrame {
|
||||
|
||||
@@ -85,14 +85,14 @@ async fn do_handshake(ws: &mut Ws) -> HandshakeResult {
|
||||
let params = HandshakeParams {
|
||||
client: "browser-skill-extension".into(),
|
||||
version: "0.1.0-dev.0".parse().unwrap(),
|
||||
protocol_version: "1.0".into(),
|
||||
protocol_version: "1.1".into(),
|
||||
instance_id: TEST_EXT_ID.into(),
|
||||
browser: BrowserPeerInfo {
|
||||
name: "chrome".into(),
|
||||
version: "131.0".into(),
|
||||
},
|
||||
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
||||
min_compatible_protocol: Some("1.0".into()),
|
||||
min_compatible_protocol: Some("1.1".into()),
|
||||
label: "Test".into(),
|
||||
};
|
||||
let req = RequestFrame {
|
||||
|
||||
@@ -53,14 +53,14 @@ pub async fn send_handshake(
|
||||
let params = HandshakeParams {
|
||||
client: "browser-skill-extension".into(),
|
||||
version: "0.1.0-dev.0".parse().unwrap(),
|
||||
protocol_version: "1.0".into(),
|
||||
protocol_version: "1.1".into(),
|
||||
instance_id: instance_id.into(),
|
||||
browser: BrowserPeerInfo {
|
||||
name: "chrome".into(),
|
||||
version: "131.0".into(),
|
||||
},
|
||||
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
||||
min_compatible_protocol: Some("1.0".into()),
|
||||
min_compatible_protocol: Some("1.1".into()),
|
||||
label: "Test Chrome".into(),
|
||||
};
|
||||
let req = RequestFrame {
|
||||
@@ -91,7 +91,7 @@ async fn ws_handshake_registers_browser_in_state() {
|
||||
let mut ws = connect_ext(handle.ws_addr(), &origin).await;
|
||||
let result = send_handshake(&mut ws, TEST_EXT_ID).await;
|
||||
assert_eq!(result.server, "browser-skill-daemon");
|
||||
assert_eq!(result.protocol_version, "1.0");
|
||||
assert_eq!(result.protocol_version, "1.1");
|
||||
|
||||
let state = handle.state();
|
||||
let browsers = state.browsers.snapshot();
|
||||
|
||||
@@ -7,10 +7,18 @@
|
||||
],
|
||||
"properties": {
|
||||
"trace": {
|
||||
"$ref": "#/definitions/Trace"
|
||||
"$ref": "#/definitions/RecordedTrace"
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"FillCommit": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"enter",
|
||||
"suggestion",
|
||||
"blur"
|
||||
]
|
||||
},
|
||||
"KeyModifier": {
|
||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||
"type": "string",
|
||||
@@ -21,6 +29,18 @@
|
||||
"shift"
|
||||
]
|
||||
},
|
||||
"NavigationCause": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"user_typed",
|
||||
"link",
|
||||
"form_submit",
|
||||
"reload",
|
||||
"history",
|
||||
"script",
|
||||
"browser"
|
||||
]
|
||||
},
|
||||
"PageRef": {
|
||||
"description": "Page context dictionary entry — referenced by steps via `page` id.",
|
||||
"type": "object",
|
||||
@@ -43,6 +63,33 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"RecordedTrace": {
|
||||
"oneOf": [
|
||||
{
|
||||
"$ref": "#/definitions/TraceV2"
|
||||
},
|
||||
{
|
||||
"$ref": "#/definitions/Trace"
|
||||
}
|
||||
]
|
||||
},
|
||||
"RecorderInfo": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"bsk",
|
||||
"vom"
|
||||
],
|
||||
"properties": {
|
||||
"bsk": {
|
||||
"type": "string"
|
||||
},
|
||||
"vom": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
}
|
||||
}
|
||||
},
|
||||
"SelectedOption": {
|
||||
"description": "One selected option (`select` op).",
|
||||
"type": "object",
|
||||
@@ -61,12 +108,333 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"SelectedOptionV2": {
|
||||
"description": "One selected option (`select` op).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"label": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"Step": {
|
||||
"description": "One recorded user action — discriminated union by `op`.",
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"cause",
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"to"
|
||||
],
|
||||
"properties": {
|
||||
"cause": {
|
||||
"$ref": "#/definitions/NavigationCause"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"navigate"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"to": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"click"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"hover"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"commit",
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target",
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"commit": {
|
||||
"$ref": "#/definitions/FillCommit"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"fill"
|
||||
]
|
||||
},
|
||||
"redacted": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"select"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"selection": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/SelectedOption"
|
||||
}
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"key",
|
||||
"op",
|
||||
"result",
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"key": {
|
||||
"type": "string"
|
||||
},
|
||||
"modifiers": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/KeyModifier"
|
||||
}
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"press"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"scroll"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"StepEffectV2": {
|
||||
"description": "Observed navigation after a step (objective fact only).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"navigated_to"
|
||||
],
|
||||
"properties": {
|
||||
"navigated_to": {
|
||||
"description": "Reference into `pages[]` for the destination page.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepResult": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"state": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepV2": {
|
||||
"description": "One recorded user action — discriminated union by `op` (v2).",
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
@@ -77,7 +445,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -105,7 +473,7 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -117,7 +485,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -140,52 +508,12 @@
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"hover"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -198,7 +526,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -227,7 +555,7 @@
|
||||
]
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
@@ -235,7 +563,7 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -248,7 +576,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -273,16 +601,16 @@
|
||||
"selection": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/SelectedOption"
|
||||
"$ref": "#/definitions/SelectedOptionV2"
|
||||
}
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -294,7 +622,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -331,7 +659,7 @@
|
||||
"target": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -342,21 +670,48 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"StepEffect": {
|
||||
"description": "Observed navigation after a step (objective fact only).",
|
||||
"StopReason": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"user_finish",
|
||||
"cli_stop"
|
||||
]
|
||||
},
|
||||
"TargetDescriptor": {
|
||||
"description": "Stable semantic handle for an interacted element within a page observation.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"navigated_to"
|
||||
],
|
||||
"properties": {
|
||||
"navigated_to": {
|
||||
"description": "Reference into `pages[]` for the destination page.",
|
||||
"type": "string"
|
||||
"ctx": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"name": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"ref": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"role": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"unmatched": {
|
||||
"type": "boolean"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TargetDescriptor": {
|
||||
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
|
||||
"TargetDescriptorV2": {
|
||||
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"tag"
|
||||
@@ -400,6 +755,113 @@
|
||||
"Trace": {
|
||||
"description": "Persisted user-action trace exported by `tool.record_stop` / `await`.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"entry",
|
||||
"recorded_at",
|
||||
"recorder",
|
||||
"states",
|
||||
"steps",
|
||||
"stopped_by",
|
||||
"version"
|
||||
],
|
||||
"properties": {
|
||||
"entry": {
|
||||
"$ref": "#/definitions/TraceEntry"
|
||||
},
|
||||
"purpose": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"recorded_at": {
|
||||
"type": "string"
|
||||
},
|
||||
"recorder": {
|
||||
"$ref": "#/definitions/RecorderInfo"
|
||||
},
|
||||
"started_at": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"states": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/TraceState"
|
||||
}
|
||||
},
|
||||
"steps": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/Step"
|
||||
}
|
||||
},
|
||||
"stopped_by": {
|
||||
"$ref": "#/definitions/StopReason"
|
||||
},
|
||||
"version": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceEntry": {
|
||||
"description": "Recording entry point — first URL the flow starts from.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"start_url"
|
||||
],
|
||||
"properties": {
|
||||
"start_url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceState": {
|
||||
"description": "Page observation dictionary entry — referenced by steps via `state` / `result.state`.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"url"
|
||||
],
|
||||
"properties": {
|
||||
"body": {
|
||||
"description": "Wire-only: full page observation (front matter + VOM body + annotations).",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"page": {
|
||||
"description": "Disk-only: filename under the bundle `pages/` directory.",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"title": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"truncated": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceV2": {
|
||||
"description": "Persisted user-action trace exported by legacy `tool.record_stop` / `await`.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"entry",
|
||||
"pages",
|
||||
@@ -417,7 +879,6 @@
|
||||
}
|
||||
},
|
||||
"purpose": {
|
||||
"description": "Optional user-provided goal from `--purpose` (metadata only).",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
@@ -436,22 +897,10 @@
|
||||
"steps": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/Step"
|
||||
"$ref": "#/definitions/StepV2"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceEntry": {
|
||||
"description": "Recording entry point — first URL the flow starts from.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"start_url"
|
||||
],
|
||||
"properties": {
|
||||
"start_url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,12 +6,26 @@
|
||||
"session_id"
|
||||
],
|
||||
"properties": {
|
||||
"max_page_tokens": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"purpose": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"redact_values": {
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"session_id": {
|
||||
"type": "string"
|
||||
},
|
||||
@@ -22,6 +36,15 @@
|
||||
],
|
||||
"format": "int64"
|
||||
},
|
||||
"trace_version": {
|
||||
"description": "Desired trace export format. Omitted ⇒ v2; `3` ⇒ state-linked v3 bundle.",
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"url": {
|
||||
"type": [
|
||||
"string",
|
||||
|
||||
@@ -7,10 +7,18 @@
|
||||
],
|
||||
"properties": {
|
||||
"trace": {
|
||||
"$ref": "#/definitions/Trace"
|
||||
"$ref": "#/definitions/RecordedTrace"
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"FillCommit": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"enter",
|
||||
"suggestion",
|
||||
"blur"
|
||||
]
|
||||
},
|
||||
"KeyModifier": {
|
||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||
"type": "string",
|
||||
@@ -21,6 +29,18 @@
|
||||
"shift"
|
||||
]
|
||||
},
|
||||
"NavigationCause": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"user_typed",
|
||||
"link",
|
||||
"form_submit",
|
||||
"reload",
|
||||
"history",
|
||||
"script",
|
||||
"browser"
|
||||
]
|
||||
},
|
||||
"PageRef": {
|
||||
"description": "Page context dictionary entry — referenced by steps via `page` id.",
|
||||
"type": "object",
|
||||
@@ -43,6 +63,33 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"RecordedTrace": {
|
||||
"oneOf": [
|
||||
{
|
||||
"$ref": "#/definitions/TraceV2"
|
||||
},
|
||||
{
|
||||
"$ref": "#/definitions/Trace"
|
||||
}
|
||||
]
|
||||
},
|
||||
"RecorderInfo": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"bsk",
|
||||
"vom"
|
||||
],
|
||||
"properties": {
|
||||
"bsk": {
|
||||
"type": "string"
|
||||
},
|
||||
"vom": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
}
|
||||
}
|
||||
},
|
||||
"SelectedOption": {
|
||||
"description": "One selected option (`select` op).",
|
||||
"type": "object",
|
||||
@@ -61,12 +108,333 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"SelectedOptionV2": {
|
||||
"description": "One selected option (`select` op).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"label": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"Step": {
|
||||
"description": "One recorded user action — discriminated union by `op`.",
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"cause",
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"to"
|
||||
],
|
||||
"properties": {
|
||||
"cause": {
|
||||
"$ref": "#/definitions/NavigationCause"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"navigate"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"to": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"click"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"hover"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"commit",
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target",
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"commit": {
|
||||
"$ref": "#/definitions/FillCommit"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"fill"
|
||||
]
|
||||
},
|
||||
"redacted": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"select"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"selection": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/SelectedOption"
|
||||
}
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"key",
|
||||
"op",
|
||||
"result",
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"key": {
|
||||
"type": "string"
|
||||
},
|
||||
"modifiers": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/KeyModifier"
|
||||
}
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"press"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"scroll"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"StepEffectV2": {
|
||||
"description": "Observed navigation after a step (objective fact only).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"navigated_to"
|
||||
],
|
||||
"properties": {
|
||||
"navigated_to": {
|
||||
"description": "Reference into `pages[]` for the destination page.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepResult": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"state": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepV2": {
|
||||
"description": "One recorded user action — discriminated union by `op` (v2).",
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
@@ -77,7 +445,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -105,7 +473,7 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -117,7 +485,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -140,52 +508,12 @@
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"hover"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -198,7 +526,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -227,7 +555,7 @@
|
||||
]
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
@@ -235,7 +563,7 @@
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -248,7 +576,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -273,16 +601,16 @@
|
||||
"selection": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/SelectedOption"
|
||||
"$ref": "#/definitions/SelectedOptionV2"
|
||||
}
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
@@ -294,7 +622,7 @@
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -331,7 +659,7 @@
|
||||
"target": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
@@ -342,21 +670,48 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"StepEffect": {
|
||||
"description": "Observed navigation after a step (objective fact only).",
|
||||
"StopReason": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"user_finish",
|
||||
"cli_stop"
|
||||
]
|
||||
},
|
||||
"TargetDescriptor": {
|
||||
"description": "Stable semantic handle for an interacted element within a page observation.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"navigated_to"
|
||||
],
|
||||
"properties": {
|
||||
"navigated_to": {
|
||||
"description": "Reference into `pages[]` for the destination page.",
|
||||
"type": "string"
|
||||
"ctx": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"name": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"ref": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"role": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"unmatched": {
|
||||
"type": "boolean"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TargetDescriptor": {
|
||||
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
|
||||
"TargetDescriptorV2": {
|
||||
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"tag"
|
||||
@@ -400,6 +755,113 @@
|
||||
"Trace": {
|
||||
"description": "Persisted user-action trace exported by `tool.record_stop` / `await`.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"entry",
|
||||
"recorded_at",
|
||||
"recorder",
|
||||
"states",
|
||||
"steps",
|
||||
"stopped_by",
|
||||
"version"
|
||||
],
|
||||
"properties": {
|
||||
"entry": {
|
||||
"$ref": "#/definitions/TraceEntry"
|
||||
},
|
||||
"purpose": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"recorded_at": {
|
||||
"type": "string"
|
||||
},
|
||||
"recorder": {
|
||||
"$ref": "#/definitions/RecorderInfo"
|
||||
},
|
||||
"started_at": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"states": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/TraceState"
|
||||
}
|
||||
},
|
||||
"steps": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/Step"
|
||||
}
|
||||
},
|
||||
"stopped_by": {
|
||||
"$ref": "#/definitions/StopReason"
|
||||
},
|
||||
"version": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceEntry": {
|
||||
"description": "Recording entry point — first URL the flow starts from.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"start_url"
|
||||
],
|
||||
"properties": {
|
||||
"start_url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceState": {
|
||||
"description": "Page observation dictionary entry — referenced by steps via `state` / `result.state`.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"url"
|
||||
],
|
||||
"properties": {
|
||||
"body": {
|
||||
"description": "Wire-only: full page observation (front matter + VOM body + annotations).",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"page": {
|
||||
"description": "Disk-only: filename under the bundle `pages/` directory.",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"title": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"truncated": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceV2": {
|
||||
"description": "Persisted user-action trace exported by legacy `tool.record_stop` / `await`.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"entry",
|
||||
"pages",
|
||||
@@ -417,7 +879,6 @@
|
||||
}
|
||||
},
|
||||
"purpose": {
|
||||
"description": "Optional user-provided goal from `--purpose` (metadata only).",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
@@ -436,22 +897,10 @@
|
||||
"steps": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/Step"
|
||||
"$ref": "#/definitions/StepV2"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceEntry": {
|
||||
"description": "Recording entry point — first URL the flow starts from.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"start_url"
|
||||
],
|
||||
"properties": {
|
||||
"start_url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,45 +5,65 @@
|
||||
"type": "object",
|
||||
"required": [
|
||||
"entry",
|
||||
"pages",
|
||||
"recorded_at",
|
||||
"steps"
|
||||
"recorder",
|
||||
"states",
|
||||
"steps",
|
||||
"stopped_by",
|
||||
"version"
|
||||
],
|
||||
"properties": {
|
||||
"entry": {
|
||||
"$ref": "#/definitions/TraceEntry"
|
||||
},
|
||||
"pages": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/PageRef"
|
||||
}
|
||||
},
|
||||
"purpose": {
|
||||
"description": "Optional user-provided goal from `--purpose` (metadata only).",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"recorded_at": {
|
||||
"description": "RFC 3339 timestamp when recording stopped.",
|
||||
"type": "string"
|
||||
},
|
||||
"recorder": {
|
||||
"$ref": "#/definitions/RecorderInfo"
|
||||
},
|
||||
"started_at": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"states": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/TraceState"
|
||||
}
|
||||
},
|
||||
"steps": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/Step"
|
||||
}
|
||||
},
|
||||
"stopped_by": {
|
||||
"$ref": "#/definitions/StopReason"
|
||||
},
|
||||
"version": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"FillCommit": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"enter",
|
||||
"suggestion",
|
||||
"blur"
|
||||
]
|
||||
},
|
||||
"KeyModifier": {
|
||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||
"type": "string",
|
||||
@@ -54,25 +74,32 @@
|
||||
"shift"
|
||||
]
|
||||
},
|
||||
"PageRef": {
|
||||
"description": "Page context dictionary entry — referenced by steps via `page` id.",
|
||||
"NavigationCause": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"user_typed",
|
||||
"link",
|
||||
"form_submit",
|
||||
"reload",
|
||||
"history",
|
||||
"script",
|
||||
"browser"
|
||||
]
|
||||
},
|
||||
"RecorderInfo": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"url"
|
||||
"bsk",
|
||||
"vom"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"bsk": {
|
||||
"type": "string"
|
||||
},
|
||||
"title": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
"vom": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -101,21 +128,16 @@
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"cause",
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"result",
|
||||
"state",
|
||||
"to"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
"cause": {
|
||||
"$ref": "#/definitions/NavigationCause"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
@@ -128,8 +150,11 @@
|
||||
"navigate"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"to": {
|
||||
@@ -143,20 +168,11 @@
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
@@ -168,8 +184,11 @@
|
||||
"click"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
@@ -183,20 +202,11 @@
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
@@ -208,8 +218,11 @@
|
||||
"hover"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
@@ -221,22 +234,17 @@
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"commit",
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"result",
|
||||
"state",
|
||||
"target",
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
"commit": {
|
||||
"$ref": "#/definitions/FillCommit"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
@@ -249,15 +257,15 @@
|
||||
"fill"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"redacted": {
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
"type": "boolean"
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
@@ -273,21 +281,11 @@
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"selection",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
@@ -299,9 +297,8 @@
|
||||
"select"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"selection": {
|
||||
"type": "array",
|
||||
@@ -309,6 +306,10 @@
|
||||
"$ref": "#/definitions/SelectedOption"
|
||||
}
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
@@ -321,19 +322,10 @@
|
||||
"id",
|
||||
"key",
|
||||
"op",
|
||||
"page"
|
||||
"result",
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
@@ -357,8 +349,11 @@
|
||||
"press"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
@@ -372,48 +367,74 @@
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"scroll"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"StepEffect": {
|
||||
"description": "Observed navigation after a step (objective fact only).",
|
||||
"StepResult": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"navigated_to"
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"navigated_to": {
|
||||
"description": "Reference into `pages[]` for the destination page.",
|
||||
"state": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"StopReason": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"user_finish",
|
||||
"cli_stop"
|
||||
]
|
||||
},
|
||||
"TargetDescriptor": {
|
||||
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
|
||||
"description": "Stable semantic handle for an interacted element within a page observation.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"tag"
|
||||
],
|
||||
"properties": {
|
||||
"ctx": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"name": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"name_attr": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"nearby_label": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"placeholder": {
|
||||
"ref": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
@@ -425,8 +446,8 @@
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"tag": {
|
||||
"type": "string"
|
||||
"unmatched": {
|
||||
"type": "boolean"
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -441,6 +462,45 @@
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceState": {
|
||||
"description": "Page observation dictionary entry — referenced by steps via `state` / `result.state`.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"url"
|
||||
],
|
||||
"properties": {
|
||||
"body": {
|
||||
"description": "Wire-only: full page observation (front matter + VOM body + annotations).",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"page": {
|
||||
"description": "Disk-only: filename under the bundle `pages/` directory.",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"title": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"truncated": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,21 +7,16 @@
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"cause",
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"result",
|
||||
"state",
|
||||
"to"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
"cause": {
|
||||
"$ref": "#/definitions/NavigationCause"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
@@ -34,8 +29,11 @@
|
||||
"navigate"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"to": {
|
||||
@@ -49,20 +47,11 @@
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
@@ -74,8 +63,11 @@
|
||||
"click"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
@@ -89,20 +81,11 @@
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
@@ -114,8 +97,11 @@
|
||||
"hover"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
@@ -127,22 +113,17 @@
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"commit",
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"result",
|
||||
"state",
|
||||
"target",
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
"commit": {
|
||||
"$ref": "#/definitions/FillCommit"
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
@@ -155,15 +136,15 @@
|
||||
"fill"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"redacted": {
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
"type": "boolean"
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
@@ -179,21 +160,11 @@
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"selection",
|
||||
"result",
|
||||
"state",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
@@ -205,9 +176,8 @@
|
||||
"select"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"selection": {
|
||||
"type": "array",
|
||||
@@ -215,6 +185,10 @@
|
||||
"$ref": "#/definitions/SelectedOption"
|
||||
}
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
@@ -227,19 +201,10 @@
|
||||
"id",
|
||||
"key",
|
||||
"op",
|
||||
"page"
|
||||
"result",
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
@@ -263,8 +228,11 @@
|
||||
"press"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
@@ -278,9 +246,47 @@
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"result",
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"scroll"
|
||||
]
|
||||
},
|
||||
"result": {
|
||||
"$ref": "#/definitions/StepResult"
|
||||
},
|
||||
"state": {
|
||||
"description": "Observation id immediately before this action.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"definitions": {
|
||||
"FillCommit": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"enter",
|
||||
"suggestion",
|
||||
"blur"
|
||||
]
|
||||
},
|
||||
"KeyModifier": {
|
||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||
"type": "string",
|
||||
@@ -291,6 +297,18 @@
|
||||
"shift"
|
||||
]
|
||||
},
|
||||
"NavigationCause": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"user_typed",
|
||||
"link",
|
||||
"form_submit",
|
||||
"reload",
|
||||
"history",
|
||||
"script",
|
||||
"browser"
|
||||
]
|
||||
},
|
||||
"SelectedOption": {
|
||||
"description": "One selected option (`select` op).",
|
||||
"type": "object",
|
||||
@@ -309,45 +327,34 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepEffect": {
|
||||
"description": "Observed navigation after a step (objective fact only).",
|
||||
"StepResult": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"navigated_to"
|
||||
"state"
|
||||
],
|
||||
"properties": {
|
||||
"navigated_to": {
|
||||
"description": "Reference into `pages[]` for the destination page.",
|
||||
"state": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TargetDescriptor": {
|
||||
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
|
||||
"description": "Stable semantic handle for an interacted element within a page observation.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"tag"
|
||||
],
|
||||
"properties": {
|
||||
"ctx": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"name": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"name_attr": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"nearby_label": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"placeholder": {
|
||||
"ref": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
@@ -359,8 +366,8 @@
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"tag": {
|
||||
"type": "string"
|
||||
"unmatched": {
|
||||
"type": "boolean"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,405 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "TraceV2",
|
||||
"description": "Persisted user-action trace exported by legacy `tool.record_stop` / `await`.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"entry",
|
||||
"pages",
|
||||
"recorded_at",
|
||||
"steps"
|
||||
],
|
||||
"properties": {
|
||||
"entry": {
|
||||
"$ref": "#/definitions/TraceEntry"
|
||||
},
|
||||
"pages": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/PageRef"
|
||||
}
|
||||
},
|
||||
"purpose": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"recorded_at": {
|
||||
"description": "RFC 3339 timestamp when recording stopped.",
|
||||
"type": "string"
|
||||
},
|
||||
"started_at": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"steps": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/StepV2"
|
||||
}
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"KeyModifier": {
|
||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"alt",
|
||||
"ctrl",
|
||||
"meta",
|
||||
"shift"
|
||||
]
|
||||
},
|
||||
"PageRef": {
|
||||
"description": "Page context dictionary entry — referenced by steps via `page` id.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"url"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"title": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"SelectedOptionV2": {
|
||||
"description": "One selected option (`select` op).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"label": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepEffectV2": {
|
||||
"description": "Observed navigation after a step (objective fact only).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"navigated_to"
|
||||
],
|
||||
"properties": {
|
||||
"navigated_to": {
|
||||
"description": "Reference into `pages[]` for the destination page.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepV2": {
|
||||
"description": "One recorded user action — discriminated union by `op` (v2).",
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"to"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"navigate"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"to": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"click"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target",
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"fill"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"redacted": {
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"selection",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"select"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"selection": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/SelectedOptionV2"
|
||||
}
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every v2 step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"key",
|
||||
"op",
|
||||
"page"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffectV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"key": {
|
||||
"type": "string"
|
||||
},
|
||||
"modifiers": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/KeyModifier"
|
||||
}
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"press"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/TargetDescriptorV2"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"TargetDescriptorV2": {
|
||||
"description": "Stable semantic handle for an interacted element (v2).\n\n`name` and `nearby_label` are **untrusted page text**.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"tag"
|
||||
],
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"name_attr": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"nearby_label": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"placeholder": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"role": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"tag": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceEntry": {
|
||||
"description": "Recording entry point — first URL the flow starts from.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"start_url"
|
||||
],
|
||||
"properties": {
|
||||
"start_url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -113,6 +113,7 @@ fn main() {
|
||||
dump!(RequestHelpParams, "tool_request_help_params");
|
||||
dump!(RequestHelpResult, "tool_request_help_result");
|
||||
|
||||
dump!(TraceV2, "trace_v2");
|
||||
dump!(Trace, "trace");
|
||||
dump!(Step, "trace_step");
|
||||
dump!(RecordStartParams, "tool_record_start_params");
|
||||
|
||||
@@ -9,6 +9,7 @@ pub mod navigation;
|
||||
pub mod network;
|
||||
pub mod observation;
|
||||
pub mod record;
|
||||
pub mod record_v2;
|
||||
pub mod script;
|
||||
pub mod session;
|
||||
pub mod tabs;
|
||||
|
||||
@@ -1,35 +1,39 @@
|
||||
//! Semantic user-action recording (`tool.record_start` / `stop` / `await`).
|
||||
//!
|
||||
//! Trace v2 is a **record-only** log of user actions: what was clicked, filled,
|
||||
//! selected, and where navigation occurred. No LLM runs during recording, so
|
||||
//! variable-vs-constant classification is **not** stored — executing agents
|
||||
//! infer that at run time from raw values and control names.
|
||||
//! Trace v3 is a **state-action-state** chain: each step binds to page
|
||||
//! observations (VOM) captured before and after the action.
|
||||
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::interaction::KeyModifier;
|
||||
|
||||
pub use crate::record_v2::{
|
||||
PageRef, SelectedOptionV2, StepCommonV2, StepEffectV2, StepV2, TargetDescriptorV2, TraceV2,
|
||||
};
|
||||
|
||||
pub const TRACE_VERSION: u32 = 3;
|
||||
pub const TRACE_VERSION_V2: u32 = 2;
|
||||
pub const DEFAULT_TRACE_VERSION: u32 = 2;
|
||||
pub const VOM_FORMAT_VERSION: u32 = 1;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Target
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Stable semantic handle for an interacted element.
|
||||
///
|
||||
/// `name` and `nearby_label` are **untrusted page text**.
|
||||
/// Stable semantic handle for an interacted element within a page observation.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct TargetDescriptor {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none", rename = "ref")]
|
||||
pub element_ref: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub role: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub name: Option<String>,
|
||||
pub tag: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub name_attr: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub placeholder: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub nearby_label: Option<String>,
|
||||
pub ctx: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
pub unmatched: bool,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -42,15 +46,54 @@ pub struct TraceEntry {
|
||||
pub start_url: String,
|
||||
}
|
||||
|
||||
/// Page context dictionary entry — referenced by steps via `page` id.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct PageRef {
|
||||
pub struct RecorderInfo {
|
||||
pub bsk: String,
|
||||
pub vom: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum StopReason {
|
||||
UserFinish,
|
||||
CliStop,
|
||||
}
|
||||
|
||||
/// Page observation dictionary entry — referenced by steps via `state` / `result.state`.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct TraceState {
|
||||
pub id: String,
|
||||
pub url: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub title: Option<String>,
|
||||
/// Wire-only: full page observation (front matter + VOM body + annotations).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub body: Option<String>,
|
||||
/// Disk-only: filename under the bundle `pages/` directory.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub page: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
pub truncated: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct StepResult {
|
||||
pub state: String,
|
||||
}
|
||||
|
||||
/// Fields shared by every step variant (flattened in JSON).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct StepCommon {
|
||||
pub id: u32,
|
||||
/// Observation id immediately before this action.
|
||||
pub state: String,
|
||||
pub result: StepResult,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Step op-specific payloads
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// One selected option (`select` op).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct SelectedOption {
|
||||
@@ -59,27 +102,26 @@ pub struct SelectedOption {
|
||||
pub label: Option<String>,
|
||||
}
|
||||
|
||||
/// Observed navigation after a step (objective fact only).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct StepEffect {
|
||||
/// Reference into `pages[]` for the destination page.
|
||||
pub navigated_to: String,
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum NavigationCause {
|
||||
UserTyped,
|
||||
Link,
|
||||
FormSubmit,
|
||||
Reload,
|
||||
History,
|
||||
Script,
|
||||
Browser,
|
||||
}
|
||||
|
||||
/// Fields shared by every step variant (flattened in JSON).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct StepCommon {
|
||||
pub id: u32,
|
||||
/// Reference into `pages[]`.
|
||||
pub page: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub effect: Option<StepEffect>,
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum FillCommit {
|
||||
Enter,
|
||||
Suggestion,
|
||||
Blur,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Step op-specific payloads
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// One recorded user action — discriminated union by `op`.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(tag = "op", rename_all = "snake_case")]
|
||||
@@ -88,6 +130,7 @@ pub enum Step {
|
||||
#[serde(flatten)]
|
||||
common: StepCommon,
|
||||
to: String,
|
||||
cause: NavigationCause,
|
||||
},
|
||||
Click {
|
||||
#[serde(flatten)]
|
||||
@@ -104,13 +147,15 @@ pub enum Step {
|
||||
common: StepCommon,
|
||||
target: TargetDescriptor,
|
||||
value: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
redacted: Option<bool>,
|
||||
commit: FillCommit,
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
redacted: bool,
|
||||
},
|
||||
Select {
|
||||
#[serde(flatten)]
|
||||
common: StepCommon,
|
||||
target: TargetDescriptor,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
selection: Vec<SelectedOption>,
|
||||
},
|
||||
Press {
|
||||
@@ -122,6 +167,10 @@ pub enum Step {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
target: Option<TargetDescriptor>,
|
||||
},
|
||||
Scroll {
|
||||
#[serde(flatten)]
|
||||
common: StepCommon,
|
||||
},
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -131,15 +180,16 @@ pub enum Step {
|
||||
/// Persisted user-action trace exported by `tool.record_stop` / `await`.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Trace {
|
||||
/// RFC 3339 timestamp when recording stopped.
|
||||
pub recorded_at: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub started_at: Option<String>,
|
||||
/// Optional user-provided goal from `--purpose` (metadata only).
|
||||
pub version: u32,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub purpose: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub started_at: Option<String>,
|
||||
pub recorded_at: String,
|
||||
pub stopped_by: StopReason,
|
||||
pub entry: TraceEntry,
|
||||
pub pages: Vec<PageRef>,
|
||||
pub recorder: RecorderInfo,
|
||||
pub states: Vec<TraceState>,
|
||||
pub steps: Vec<Step>,
|
||||
}
|
||||
|
||||
@@ -156,6 +206,13 @@ pub struct RecordStartParams {
|
||||
pub url: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub purpose: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_page_tokens: Option<u32>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub redact_values: Option<bool>,
|
||||
/// Desired trace export format. Omitted ⇒ v2; `3` ⇒ state-linked v3 bundle.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub trace_version: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
@@ -169,9 +226,100 @@ pub struct RecordStopParams {
|
||||
pub session_id: String,
|
||||
}
|
||||
|
||||
/// Wire trace payload — v2 (legacy `pages[]`) or v3 (`version: 3`, `states[]`).
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub enum RecordedTrace {
|
||||
V2(TraceV2),
|
||||
V3(Trace),
|
||||
}
|
||||
|
||||
impl RecordedTrace {
|
||||
pub fn classify_value(v: &serde_json::Value) -> Result<Self, String> {
|
||||
if let Some(ver) = v.get("version").and_then(|x| x.as_u64()) {
|
||||
if ver == u64::from(TRACE_VERSION) {
|
||||
return serde_json::from_value(v.clone())
|
||||
.map(RecordedTrace::V3)
|
||||
.map_err(|e| e.to_string());
|
||||
}
|
||||
return Err(format!("unsupported trace version {ver}"));
|
||||
}
|
||||
if v.get("pages").is_some() && v.get("states").is_none() {
|
||||
return serde_json::from_value(v.clone())
|
||||
.map(RecordedTrace::V2)
|
||||
.map_err(|e| e.to_string());
|
||||
}
|
||||
if v.get("states").is_some() {
|
||||
return serde_json::from_value(v.clone())
|
||||
.map(RecordedTrace::V3)
|
||||
.map_err(|e| e.to_string());
|
||||
}
|
||||
Err("ambiguous or unparseable trace".into())
|
||||
}
|
||||
|
||||
pub fn is_v3(&self) -> bool {
|
||||
matches!(self, RecordedTrace::V3(_))
|
||||
}
|
||||
|
||||
pub fn as_v3(&self) -> Option<&Trace> {
|
||||
match self {
|
||||
RecordedTrace::V3(t) => Some(t),
|
||||
RecordedTrace::V2(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn as_v2(&self) -> Option<&TraceV2> {
|
||||
match self {
|
||||
RecordedTrace::V2(t) => Some(t),
|
||||
RecordedTrace::V3(_) => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for RecordedTrace {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
match self {
|
||||
RecordedTrace::V2(t) => t.serialize(serializer),
|
||||
RecordedTrace::V3(t) => t.serialize(serializer),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for RecordedTrace {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
let value = serde_json::Value::deserialize(deserializer)?;
|
||||
Self::classify_value(&value).map_err(serde::de::Error::custom)
|
||||
}
|
||||
}
|
||||
|
||||
impl JsonSchema for RecordedTrace {
|
||||
fn schema_name() -> String {
|
||||
"RecordedTrace".into()
|
||||
}
|
||||
|
||||
fn json_schema(generator: &mut schemars::SchemaGenerator) -> schemars::schema::Schema {
|
||||
schemars::schema::SchemaObject {
|
||||
subschemas: Some(Box::new(schemars::schema::SubschemaValidation {
|
||||
one_of: Some(vec![
|
||||
generator.subschema_for::<TraceV2>(),
|
||||
generator.subschema_for::<Trace>(),
|
||||
]),
|
||||
..Default::default()
|
||||
})),
|
||||
..Default::default()
|
||||
}
|
||||
.into()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct RecordStopResult {
|
||||
pub trace: Trace,
|
||||
pub trace: RecordedTrace,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
@@ -183,7 +331,7 @@ pub struct RecordAwaitParams {
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct RecordAwaitResult {
|
||||
pub trace: Trace,
|
||||
pub trace: RecordedTrace,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -195,81 +343,79 @@ mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
fn sample_common(id: u32) -> StepCommon {
|
||||
fn sample_common(id: u32, state: &str, result_state: &str) -> StepCommon {
|
||||
StepCommon {
|
||||
id,
|
||||
page: "p1".into(),
|
||||
effect: None,
|
||||
state: state.into(),
|
||||
result: StepResult {
|
||||
state: result_state.into(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_target() -> TargetDescriptor {
|
||||
TargetDescriptor {
|
||||
element_ref: Some("e21".into()),
|
||||
role: Some("button".into()),
|
||||
name: Some("发布".into()),
|
||||
tag: "button".into(),
|
||||
name_attr: None,
|
||||
placeholder: None,
|
||||
nearby_label: None,
|
||||
ctx: Some("金桔柠檬 6 号".into()),
|
||||
unmatched: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_trace() -> Trace {
|
||||
Trace {
|
||||
recorded_at: "2026-07-17T09:01:10Z".into(),
|
||||
started_at: Some("2026-07-17T09:00:00Z".into()),
|
||||
purpose: Some("发布一篇文章".into()),
|
||||
version: TRACE_VERSION,
|
||||
purpose: Some("把草稿商品发布上架".into()),
|
||||
started_at: Some("2026-08-10T02:10:41.080Z".into()),
|
||||
recorded_at: "2026-08-10T02:12:55.360Z".into(),
|
||||
stopped_by: StopReason::UserFinish,
|
||||
entry: TraceEntry {
|
||||
start_url: "https://x.com/editor".into(),
|
||||
start_url: "https://example.com/".into(),
|
||||
},
|
||||
pages: vec![
|
||||
PageRef {
|
||||
id: "p1".into(),
|
||||
url: "https://x.com/editor".into(),
|
||||
title: Some("写文章".into()),
|
||||
recorder: RecorderInfo {
|
||||
bsk: "0.1.10".into(),
|
||||
vom: VOM_FORMAT_VERSION,
|
||||
},
|
||||
states: vec![
|
||||
TraceState {
|
||||
id: "s1".into(),
|
||||
url: "https://example.com/".into(),
|
||||
title: Some("Example Domain".into()),
|
||||
body: None,
|
||||
page: Some("s1.vom.txt".into()),
|
||||
truncated: false,
|
||||
},
|
||||
PageRef {
|
||||
id: "p2".into(),
|
||||
url: "https://x.com/p/99".into(),
|
||||
title: None,
|
||||
TraceState {
|
||||
id: "s2".into(),
|
||||
url: "https://shop.example.com/products?status=draft".into(),
|
||||
title: Some("商品管理".into()),
|
||||
body: None,
|
||||
page: Some("s2.vom.txt".into()),
|
||||
truncated: false,
|
||||
},
|
||||
],
|
||||
steps: vec![
|
||||
Step::Fill {
|
||||
common: sample_common(1),
|
||||
target: TargetDescriptor {
|
||||
role: Some("textbox".into()),
|
||||
name: Some("标题".into()),
|
||||
tag: "input".into(),
|
||||
name_attr: None,
|
||||
placeholder: None,
|
||||
nearby_label: None,
|
||||
},
|
||||
value: "我的第一篇文章".into(),
|
||||
redacted: None,
|
||||
Step::Navigate {
|
||||
common: sample_common(1, "s1", "s2"),
|
||||
to: "https://shop.example.com/products?status=draft".into(),
|
||||
cause: NavigationCause::UserTyped,
|
||||
},
|
||||
Step::Select {
|
||||
common: sample_common(3),
|
||||
Step::Fill {
|
||||
common: sample_common(2, "s2", "s2"),
|
||||
target: TargetDescriptor {
|
||||
role: Some("combobox".into()),
|
||||
name: Some("分类".into()),
|
||||
tag: "select".into(),
|
||||
name_attr: None,
|
||||
placeholder: None,
|
||||
nearby_label: None,
|
||||
element_ref: Some("e12".into()),
|
||||
role: Some("textbox".into()),
|
||||
name: Some("搜索商品".into()),
|
||||
ctx: None,
|
||||
unmatched: false,
|
||||
},
|
||||
selection: vec![SelectedOption {
|
||||
value: "tech".into(),
|
||||
label: Some("技术分享".into()),
|
||||
}],
|
||||
value: "金桔柠檬".into(),
|
||||
commit: FillCommit::Enter,
|
||||
redacted: false,
|
||||
},
|
||||
Step::Click {
|
||||
common: StepCommon {
|
||||
effect: Some(StepEffect {
|
||||
navigated_to: "p2".into(),
|
||||
}),
|
||||
..sample_common(4)
|
||||
},
|
||||
common: sample_common(3, "s2", "s2"),
|
||||
target: sample_target(),
|
||||
},
|
||||
],
|
||||
@@ -279,35 +425,37 @@ mod tests {
|
||||
#[test]
|
||||
fn step_click_round_trips() {
|
||||
let step = Step::Click {
|
||||
common: sample_common(1),
|
||||
common: sample_common(1, "s1", "s2"),
|
||||
target: sample_target(),
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert_eq!(v.get("op").and_then(|v| v.as_str()), Some("click"));
|
||||
assert_eq!(v.get("page").and_then(|v| v.as_str()), Some("p1"));
|
||||
assert!(v.get("key").is_none());
|
||||
assert_eq!(v.get("state").and_then(|v| v.as_str()), Some("s1"));
|
||||
assert_eq!(v["result"]["state"], "s2");
|
||||
assert_eq!(v["target"]["ref"], "e21");
|
||||
let round: Step = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, step);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn step_fill_with_raw_value_round_trips() {
|
||||
fn step_fill_with_commit_round_trips() {
|
||||
let step = Step::Fill {
|
||||
common: sample_common(2),
|
||||
common: sample_common(2, "s2", "s3"),
|
||||
target: TargetDescriptor {
|
||||
element_ref: Some("e12".into()),
|
||||
role: Some("textbox".into()),
|
||||
name: Some("服务名称".into()),
|
||||
tag: "input".into(),
|
||||
name_attr: Some("serviceName".into()),
|
||||
placeholder: None,
|
||||
nearby_label: None,
|
||||
name: Some("搜索商品".into()),
|
||||
ctx: None,
|
||||
unmatched: false,
|
||||
},
|
||||
value: "my-svc".into(),
|
||||
redacted: None,
|
||||
value: "browser skill".into(),
|
||||
commit: FillCommit::Enter,
|
||||
redacted: false,
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert_eq!(v["op"], "fill");
|
||||
assert_eq!(v["value"], "my-svc");
|
||||
assert_eq!(v["commit"], "enter");
|
||||
assert!(v.get("redacted").is_none());
|
||||
let round: Step = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, step);
|
||||
}
|
||||
@@ -315,17 +463,17 @@ mod tests {
|
||||
#[test]
|
||||
fn step_fill_password_is_redacted() {
|
||||
let step = Step::Fill {
|
||||
common: sample_common(1),
|
||||
common: sample_common(1, "s1", "s1"),
|
||||
target: TargetDescriptor {
|
||||
element_ref: Some("e3".into()),
|
||||
role: Some("textbox".into()),
|
||||
name: Some("密码".into()),
|
||||
tag: "input".into(),
|
||||
name_attr: None,
|
||||
placeholder: None,
|
||||
nearby_label: None,
|
||||
ctx: None,
|
||||
unmatched: false,
|
||||
},
|
||||
value: "***".into(),
|
||||
redacted: Some(true),
|
||||
commit: FillCommit::Blur,
|
||||
redacted: true,
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert_eq!(v["value"], "***");
|
||||
@@ -333,131 +481,140 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn step_select_uses_object_array() {
|
||||
let step = Step::Select {
|
||||
common: sample_common(3),
|
||||
target: TargetDescriptor {
|
||||
role: Some("combobox".into()),
|
||||
name: Some("分类".into()),
|
||||
tag: "select".into(),
|
||||
name_attr: None,
|
||||
placeholder: None,
|
||||
nearby_label: None,
|
||||
},
|
||||
selection: vec![SelectedOption {
|
||||
value: "tech".into(),
|
||||
label: Some("技术分享".into()),
|
||||
}],
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert_eq!(v["selection"][0]["value"], "tech");
|
||||
assert_eq!(v["selection"][0]["label"], "技术分享");
|
||||
let round: Step = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, step);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn step_navigate_round_trips() {
|
||||
fn step_navigate_with_cause_round_trips() {
|
||||
let step = Step::Navigate {
|
||||
common: sample_common(1),
|
||||
common: sample_common(1, "s1", "s2"),
|
||||
to: "https://example.com".into(),
|
||||
cause: NavigationCause::UserTyped,
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert_eq!(v["op"], "navigate");
|
||||
assert_eq!(v["to"], "https://example.com");
|
||||
assert_eq!(v["cause"], "user_typed");
|
||||
let round: Step = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, step);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn step_press_with_modifiers_round_trips() {
|
||||
let step = Step::Press {
|
||||
common: StepCommon {
|
||||
effect: Some(StepEffect {
|
||||
navigated_to: "p2".into(),
|
||||
}),
|
||||
..sample_common(2)
|
||||
},
|
||||
key: "Enter".into(),
|
||||
modifiers: Some(vec![KeyModifier::Ctrl, KeyModifier::Shift]),
|
||||
target: Some(sample_target()),
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert_eq!(v["key"], "Enter");
|
||||
assert_eq!(v["modifiers"], json!(["ctrl", "shift"]));
|
||||
assert_eq!(v["effect"]["navigated_to"], "p2");
|
||||
let round: Step = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, step);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_record_only_round_trips() {
|
||||
fn trace_v3_round_trips() {
|
||||
let trace = sample_trace();
|
||||
let v = serde_json::to_value(&trace).unwrap();
|
||||
assert!(v.get("version").is_none());
|
||||
assert_eq!(
|
||||
v.get("purpose").and_then(|v| v.as_str()),
|
||||
Some("发布一篇文章")
|
||||
);
|
||||
assert!(v.get("parameters").is_none());
|
||||
assert!(v.get("site").is_none());
|
||||
assert!(v.get("goal").is_none());
|
||||
assert_eq!(
|
||||
v.get("entry")
|
||||
.and_then(|e| e.get("start_url"))
|
||||
.and_then(|u| u.as_str()),
|
||||
Some("https://x.com/editor")
|
||||
);
|
||||
assert_eq!(v.get("version").and_then(|v| v.as_u64()), Some(3));
|
||||
assert!(v.get("pages").is_none());
|
||||
assert_eq!(v["recorder"]["vom"], 1);
|
||||
assert_eq!(v["states"].as_array().unwrap().len(), 2);
|
||||
let round: Trace = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, trace);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn discriminated_union_click_ignores_extra_key_field() {
|
||||
let bad = json!({
|
||||
"op": "click",
|
||||
"id": 1,
|
||||
"page": "p1",
|
||||
"target": { "tag": "button", "name": "OK" },
|
||||
"key": "Enter"
|
||||
});
|
||||
let step: Step = serde_json::from_value(bad).unwrap();
|
||||
assert!(matches!(step, Step::Click { .. }));
|
||||
fn default_fields_are_omitted() {
|
||||
let step = Step::Click {
|
||||
common: sample_common(1, "s1", "s1"),
|
||||
target: TargetDescriptor {
|
||||
element_ref: Some("e1".into()),
|
||||
role: Some("button".into()),
|
||||
name: Some("OK".into()),
|
||||
ctx: None,
|
||||
unmatched: false,
|
||||
},
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert!(v.get("key").is_none());
|
||||
assert!(v.get("unmatched").is_none());
|
||||
assert!(v["target"].get("ctx").is_none());
|
||||
assert!(v["target"].get("unmatched").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unmatched_target_serializes_flag() {
|
||||
let step = Step::Click {
|
||||
common: sample_common(1, "s1", "s2"),
|
||||
target: TargetDescriptor {
|
||||
element_ref: None,
|
||||
role: Some("button".into()),
|
||||
name: Some("发布".into()),
|
||||
ctx: None,
|
||||
unmatched: true,
|
||||
},
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert_eq!(v["target"]["unmatched"], true);
|
||||
assert!(v["target"].get("ref").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recorded_trace_classifies_v2_and_v3() {
|
||||
let v2 = json!({
|
||||
"recorded_at": "2026-07-21T08:00:00Z",
|
||||
"entry": { "start_url": "https://example.com/" },
|
||||
"pages": [{ "id": "p1", "url": "https://example.com/" }],
|
||||
"steps": []
|
||||
});
|
||||
match RecordedTrace::classify_value(&v2).unwrap() {
|
||||
RecordedTrace::V2(_) => {}
|
||||
other => panic!("expected v2, got {other:?}"),
|
||||
}
|
||||
|
||||
let v3 = json!({
|
||||
"version": 3,
|
||||
"recorded_at": "2026-07-21T08:00:00Z",
|
||||
"stopped_by": "user_finish",
|
||||
"entry": { "start_url": "https://example.com/" },
|
||||
"recorder": { "bsk": "0.1.10", "vom": 1 },
|
||||
"states": [],
|
||||
"steps": []
|
||||
});
|
||||
match RecordedTrace::classify_value(&v3).unwrap() {
|
||||
RecordedTrace::V3(t) => assert_eq!(t.version, 3),
|
||||
other => panic!("expected v3, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn recorded_trace_schema_includes_v2_and_v3() {
|
||||
let schema = serde_json::to_value(schemars::schema_for!(RecordStopResult)).unwrap();
|
||||
let variants = schema["definitions"]["RecordedTrace"]["oneOf"]
|
||||
.as_array()
|
||||
.expect("RecordedTrace schema should use oneOf");
|
||||
|
||||
assert_eq!(variants.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extension_trace_deserializes() {
|
||||
let v = json!({
|
||||
"version": 3,
|
||||
"recorded_at": "2026-07-21T08:00:00Z",
|
||||
"started_at": "2026-07-21T07:59:00Z",
|
||||
"purpose": "demo",
|
||||
"stopped_by": "user_finish",
|
||||
"entry": { "start_url": "https://example.com/editor" },
|
||||
"pages": [
|
||||
{ "id": "p1", "url": "https://example.com/editor" },
|
||||
{ "id": "p2", "url": "https://example.com/p/99" }
|
||||
"recorder": { "bsk": "0.1.10", "vom": 1 },
|
||||
"states": [
|
||||
{ "id": "s1", "url": "https://example.com/editor", "page": "s1.vom.txt" },
|
||||
{ "id": "s2", "url": "https://example.com/p/99", "page": "s2.vom.txt" }
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"op": "fill",
|
||||
"id": 1,
|
||||
"page": "p1",
|
||||
"target": { "tag": "input", "role": "textbox", "name": "标题" },
|
||||
"value": "hello"
|
||||
"state": "s1",
|
||||
"result": { "state": "s1" },
|
||||
"target": { "ref": "e1", "role": "textbox", "name": "标题" },
|
||||
"value": "hello",
|
||||
"commit": "blur"
|
||||
},
|
||||
{
|
||||
"op": "click",
|
||||
"id": 2,
|
||||
"page": "p1",
|
||||
"target": { "tag": "button", "role": "button", "name": "发布" },
|
||||
"effect": { "navigated_to": "p2" }
|
||||
"state": "s1",
|
||||
"result": { "state": "s2" },
|
||||
"target": { "ref": "e2", "role": "button", "name": "发布" }
|
||||
}
|
||||
]
|
||||
});
|
||||
let trace: Trace = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(trace.entry.start_url, "https://example.com/editor");
|
||||
assert_eq!(trace.pages.len(), 2);
|
||||
assert_eq!(trace.version, 3);
|
||||
assert_eq!(trace.states.len(), 2);
|
||||
assert_eq!(trace.steps.len(), 2);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,215 @@
|
||||
//! Trace v2 — record-only user-action log with `pages[]` and step `page` refs.
|
||||
//!
|
||||
//! Legacy wire format: no top-level `version` field. New peers omit
|
||||
//! `trace_version` on `record_start` to request this shape.
|
||||
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::interaction::KeyModifier;
|
||||
use super::record::TraceEntry;
|
||||
|
||||
/// Stable semantic handle for an interacted element (v2).
|
||||
///
|
||||
/// `name` and `nearby_label` are **untrusted page text**.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct TargetDescriptorV2 {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub role: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub name: Option<String>,
|
||||
pub tag: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub name_attr: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub placeholder: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub nearby_label: Option<String>,
|
||||
}
|
||||
|
||||
/// Page context dictionary entry — referenced by steps via `page` id.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct PageRef {
|
||||
pub id: String,
|
||||
pub url: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub title: Option<String>,
|
||||
}
|
||||
|
||||
/// One selected option (`select` op).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct SelectedOptionV2 {
|
||||
pub value: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub label: Option<String>,
|
||||
}
|
||||
|
||||
/// Observed navigation after a step (objective fact only).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct StepEffectV2 {
|
||||
/// Reference into `pages[]` for the destination page.
|
||||
pub navigated_to: String,
|
||||
}
|
||||
|
||||
/// Fields shared by every v2 step variant (flattened in JSON).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct StepCommonV2 {
|
||||
pub id: u32,
|
||||
/// Reference into `pages[]`.
|
||||
pub page: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub effect: Option<StepEffectV2>,
|
||||
}
|
||||
|
||||
/// One recorded user action — discriminated union by `op` (v2).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(tag = "op", rename_all = "snake_case")]
|
||||
pub enum StepV2 {
|
||||
Navigate {
|
||||
#[serde(flatten)]
|
||||
common: StepCommonV2,
|
||||
to: String,
|
||||
},
|
||||
Click {
|
||||
#[serde(flatten)]
|
||||
common: StepCommonV2,
|
||||
target: TargetDescriptorV2,
|
||||
},
|
||||
Fill {
|
||||
#[serde(flatten)]
|
||||
common: StepCommonV2,
|
||||
target: TargetDescriptorV2,
|
||||
value: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
redacted: Option<bool>,
|
||||
},
|
||||
Select {
|
||||
#[serde(flatten)]
|
||||
common: StepCommonV2,
|
||||
target: TargetDescriptorV2,
|
||||
selection: Vec<SelectedOptionV2>,
|
||||
},
|
||||
Press {
|
||||
#[serde(flatten)]
|
||||
common: StepCommonV2,
|
||||
key: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
modifiers: Option<Vec<KeyModifier>>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
target: Option<TargetDescriptorV2>,
|
||||
},
|
||||
}
|
||||
|
||||
/// Persisted user-action trace exported by legacy `tool.record_stop` / `await`.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct TraceV2 {
|
||||
/// RFC 3339 timestamp when recording stopped.
|
||||
pub recorded_at: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub started_at: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub purpose: Option<String>,
|
||||
pub entry: TraceEntry,
|
||||
pub pages: Vec<PageRef>,
|
||||
pub steps: Vec<StepV2>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
fn sample_common(id: u32) -> StepCommonV2 {
|
||||
StepCommonV2 {
|
||||
id,
|
||||
page: "p1".into(),
|
||||
effect: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_target() -> TargetDescriptorV2 {
|
||||
TargetDescriptorV2 {
|
||||
role: Some("button".into()),
|
||||
name: Some("发布".into()),
|
||||
tag: "button".into(),
|
||||
name_attr: None,
|
||||
placeholder: None,
|
||||
nearby_label: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_v2_has_no_version_field() {
|
||||
let trace = TraceV2 {
|
||||
recorded_at: "2026-07-17T09:01:10Z".into(),
|
||||
started_at: None,
|
||||
purpose: None,
|
||||
entry: TraceEntry {
|
||||
start_url: "https://example.com/".into(),
|
||||
},
|
||||
pages: vec![PageRef {
|
||||
id: "p1".into(),
|
||||
url: "https://example.com/".into(),
|
||||
title: None,
|
||||
}],
|
||||
steps: vec![StepV2::Click {
|
||||
common: sample_common(1),
|
||||
target: sample_target(),
|
||||
}],
|
||||
};
|
||||
let v = serde_json::to_value(&trace).unwrap();
|
||||
assert!(v.get("version").is_none());
|
||||
assert!(v.get("pages").is_some());
|
||||
assert!(v.get("states").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extension_v2_trace_deserializes() {
|
||||
let v = json!({
|
||||
"recorded_at": "2026-07-21T08:00:00Z",
|
||||
"started_at": "2026-07-21T07:59:00Z",
|
||||
"purpose": "demo",
|
||||
"entry": { "start_url": "https://example.com/editor" },
|
||||
"pages": [
|
||||
{ "id": "p1", "url": "https://example.com/editor" },
|
||||
{ "id": "p2", "url": "https://example.com/p/99" }
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"op": "fill",
|
||||
"id": 1,
|
||||
"page": "p1",
|
||||
"target": { "tag": "input", "role": "textbox", "name": "标题" },
|
||||
"value": "hello"
|
||||
},
|
||||
{
|
||||
"op": "click",
|
||||
"id": 2,
|
||||
"page": "p1",
|
||||
"target": { "tag": "button", "role": "button", "name": "发布" },
|
||||
"effect": { "navigated_to": "p2" }
|
||||
}
|
||||
]
|
||||
});
|
||||
let trace: TraceV2 = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(trace.pages.len(), 2);
|
||||
assert_eq!(trace.steps.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn historical_v2_rejects_hover_steps() {
|
||||
let value = json!({
|
||||
"recorded_at": "2026-07-21T08:00:00Z",
|
||||
"entry": { "start_url": "https://example.com/" },
|
||||
"pages": [{ "id": "p1", "url": "https://example.com/" }],
|
||||
"steps": [{
|
||||
"op": "hover",
|
||||
"id": 1,
|
||||
"page": "p1",
|
||||
"target": { "tag": "button", "name": "结束" }
|
||||
}]
|
||||
});
|
||||
|
||||
assert!(serde_json::from_value::<TraceV2>(value).is_err());
|
||||
}
|
||||
}
|
||||
@@ -6,20 +6,20 @@
|
||||
"disconnected": "Disconnected",
|
||||
"connecting": "Connecting…",
|
||||
"connected": "Connected",
|
||||
"version_skew": "Protocol mismatch",
|
||||
"version_skew": "Version mismatch",
|
||||
"disabled": "Connection off"
|
||||
},
|
||||
"stateDetail": {
|
||||
"disconnected": "Open BrowserSkill to connect.",
|
||||
"connecting": "Connecting to BrowserSkill…",
|
||||
"connected": "Ready for automation tasks.",
|
||||
"version_skew": "Some features may be unavailable."
|
||||
"version_skew": "bsk and the browser extension differ in version. You can continue using them, but updating is recommended."
|
||||
},
|
||||
"stateBadge": {
|
||||
"disconnected": "Standby",
|
||||
"connecting": "Igniting",
|
||||
"connected": "Ready",
|
||||
"version_skew": "Action needed",
|
||||
"version_skew": "Update recommended",
|
||||
"disabled": "Off"
|
||||
},
|
||||
"connectionToggleTitle": "BrowserSkill connection",
|
||||
@@ -27,6 +27,7 @@
|
||||
"controlHintsToggleHint": "Show the status pill and orange glow while the Agent controls a page.",
|
||||
"controlHintsInfoLabel": "About control hints",
|
||||
"versionSkewWarning": "Extension protocol v{{extensionProtocol}}, daemon v{{daemonProtocol}}.",
|
||||
"versionIncompatibleError": "bsk and the BrowserSkill extension are incompatible. Update both, reload the extension, then run bsk daemon restart.",
|
||||
"upgradeAvailable": "Upgradable",
|
||||
"launcher": {
|
||||
"title": "Quick actions"
|
||||
@@ -48,7 +49,7 @@
|
||||
"record": {
|
||||
"sectionTitle": "Action recording",
|
||||
"cardDesc": "Record your actions for Agent reference",
|
||||
"sectionHint": "Generates a recording instruction — send it to your Agent to start recording your actions; the Agent can then use the recording to automate similar tasks.",
|
||||
"sectionHint": "Generates a recording instruction — send it to your Agent to start recording your actions; the Agent can then use the resulting trace bundle to automate similar tasks.",
|
||||
"topicLabel": "Topic (optional)",
|
||||
"topicPlaceholder": "e.g. Publish an article",
|
||||
"startUrlLabel": "Starting URL (optional)",
|
||||
@@ -56,7 +57,7 @@
|
||||
"copyButton": "Copy recording instructions",
|
||||
"copied": "Copied",
|
||||
"hintDisconnected": "Available when connected",
|
||||
"promptTemplate": "Use BrowserSkill's `bsk` CLI to record the browser actions I'm about to demonstrate.\n\nSteps:\n1. Run: {{command}}\n2. Once you confirm it's started, I'll operate in the opened window myself.\n3. After I click Finish, read the returned trace.json and summarize the actions (pages + steps); don't re-run or \"verify\"."
|
||||
"promptTemplate": "Use BrowserSkill's `bsk` CLI to record the browser actions I'm about to demonstrate.\n\nSteps:\n1. Run: {{command}}\n2. Once you confirm it's started, I'll operate in the opened window myself.\n3. After I click Finish, read the written `./trace` bundle (`trace.json` + `pages/`) and summarize what I did."
|
||||
}
|
||||
},
|
||||
"controlOverlay": {
|
||||
|
||||
@@ -6,20 +6,20 @@
|
||||
"disconnected": "未连接",
|
||||
"connecting": "连接中…",
|
||||
"connected": "已连接",
|
||||
"version_skew": "协议不一致",
|
||||
"version_skew": "版本不一致",
|
||||
"disabled": "连接已关闭"
|
||||
},
|
||||
"stateDetail": {
|
||||
"disconnected": "请先打开 BrowserSkill。",
|
||||
"connecting": "正在连接 BrowserSkill…",
|
||||
"connected": "可接收自动化任务。",
|
||||
"version_skew": "部分能力可能不可用。"
|
||||
"version_skew": "bsk 与浏览器扩展版本不同,当前仍可使用,建议更新至最新版本。"
|
||||
},
|
||||
"stateBadge": {
|
||||
"disconnected": "Standby",
|
||||
"connecting": "Igniting",
|
||||
"connected": "Ready",
|
||||
"version_skew": "Action needed",
|
||||
"version_skew": "建议更新",
|
||||
"disabled": "Off"
|
||||
},
|
||||
"connectionToggleTitle": "BrowserSkill 连接",
|
||||
@@ -27,6 +27,7 @@
|
||||
"controlHintsToggleHint": "Agent 控制页面时显示提示条和橙色闪光。",
|
||||
"controlHintsInfoLabel": "控制提示说明",
|
||||
"versionSkewWarning": "扩展协议 v{{extensionProtocol}},daemon 协议 v{{daemonProtocol}}。",
|
||||
"versionIncompatibleError": "bsk 与 BrowserSkill 扩展版本不兼容。请将两者更新至最新版本,重新加载扩展后,再运行 bsk daemon restart。",
|
||||
"upgradeAvailable": "可升级",
|
||||
"launcher": {
|
||||
"title": "快捷功能"
|
||||
@@ -48,7 +49,7 @@
|
||||
"record": {
|
||||
"sectionTitle": "操作录制",
|
||||
"cardDesc": "录制你的操作,供 Agent 参考",
|
||||
"sectionHint": "生成一段录制指令,发给 Agent 后可以启动插件录制用户操作,之后 Agent 可以参考录制结果自动完成同类任务。",
|
||||
"sectionHint": "生成一段录制指令,发给 Agent 后可以启动插件录制用户操作,之后 Agent 可以参考写出的 trace 目录包自动完成同类任务。",
|
||||
"topicLabel": "操作主题(可选)",
|
||||
"topicPlaceholder": "例如:发布一篇文章",
|
||||
"startUrlLabel": "起始网址(可选)",
|
||||
@@ -56,7 +57,7 @@
|
||||
"copyButton": "复制录制指令",
|
||||
"copied": "已复制",
|
||||
"hintDisconnected": "连接后可用",
|
||||
"promptTemplate": "用 BrowserSkill 的 `bsk` CLI 录制我接下来的浏览器操作。\n\n步骤:\n1. 执行:{{command}}\n2. 你确认已开始后,我会在打开的窗口里自己操作。\n3. 我点“结束”后,读取返回的 trace.json 并总结操作(pages + steps),不要重跑或“验证”。"
|
||||
"promptTemplate": "用 BrowserSkill 的 `bsk` CLI 录制我接下来的浏览器操作。\n\n步骤:\n1. 执行:{{command}}\n2. 你确认已开始后,我会在打开的窗口里自己操作。\n3. 我点“结束”后,读取写出的 `./trace` 目录包(`trace.json` + `pages/`),总结我做了什么。"
|
||||
}
|
||||
},
|
||||
"controlOverlay": {
|
||||
|
||||
@@ -24,6 +24,11 @@ function scene(nodes: VomNode[]): VomScene {
|
||||
};
|
||||
}
|
||||
|
||||
/** Compare refs without recording-metadata fields (role/name/ctx/line). */
|
||||
function coreRefs(refs: ReturnType<typeof renderVom>["refs"]) {
|
||||
return refs.map(({ ref: id, backendNodeId }) => ({ ref: id, backendNodeId }));
|
||||
}
|
||||
|
||||
describe("renderVom single-layer page", () => {
|
||||
it("always emits a complete @vom single-layer document when there is no blocker", () => {
|
||||
const out = renderVom(
|
||||
@@ -51,7 +56,7 @@ describe("renderVom single-layer page", () => {
|
||||
' @e2 button "加入购物车"',
|
||||
].join("\n"),
|
||||
);
|
||||
expect(out.refs).toEqual([
|
||||
expect(coreRefs(out.refs)).toEqual([
|
||||
{ ref: "e1", backendNodeId: 3 },
|
||||
{ ref: "e2", backendNodeId: 6 },
|
||||
]);
|
||||
@@ -72,7 +77,7 @@ describe("renderVom single-layer page", () => {
|
||||
expect(out.text).toContain(' heading "Title"');
|
||||
expect(out.text).toContain(' img "Logo"');
|
||||
expect(out.text).toContain(' @e1 button "Continue"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||
});
|
||||
|
||||
it("assigns refs to listbox controls", () => {
|
||||
@@ -84,7 +89,7 @@ describe("renderVom single-layer page", () => {
|
||||
);
|
||||
|
||||
expect(out.text).toContain(' @e1 listbox "Choices"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
it("annotates external links regardless of role casing", () => {
|
||||
@@ -121,7 +126,7 @@ describe("renderVom single-layer page", () => {
|
||||
expect(out.text).toContain(" alertdialog");
|
||||
expect(out.text).toContain(" section");
|
||||
expect(out.text).toContain(' @e1 button "Close"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 5 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 5 }]);
|
||||
});
|
||||
|
||||
it("skips generic nodes without adding an extra indentation level", () => {
|
||||
@@ -192,7 +197,7 @@ describe("renderVom single-layer page", () => {
|
||||
expect(out.text).toContain(" section");
|
||||
expect(out.text).not.toContain("Too deep");
|
||||
expect(out.text).toContain(' @e1 button "Later"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||
});
|
||||
|
||||
it("truncates when maxTokens is exceeded while keeping refs in sync with rendered text", () => {
|
||||
@@ -227,7 +232,7 @@ describe("renderVom single-layer page", () => {
|
||||
);
|
||||
|
||||
expect(out.text).toContain('@e1 button "Open settings"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
it("keeps the deepest custom control when wrapper and child both look clickable", () => {
|
||||
@@ -257,7 +262,7 @@ describe("renderVom single-layer page", () => {
|
||||
|
||||
expect(out.text).not.toContain('@e1 button "Card"');
|
||||
expect(out.text).toContain('@e1 button "Details"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
||||
});
|
||||
|
||||
it("does not recover clickable wrappers around native controls", () => {
|
||||
@@ -280,7 +285,7 @@ describe("renderVom single-layer page", () => {
|
||||
|
||||
expect(out.text).not.toContain('@e1 button "Checkout"');
|
||||
expect(out.text).toContain('@e1 button "Pay"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
||||
});
|
||||
|
||||
it("adds context to duplicate weak action labels", () => {
|
||||
@@ -343,7 +348,7 @@ describe("renderVom single-layer page", () => {
|
||||
|
||||
expect(out.text).toContain("@layers 1 focus=L1");
|
||||
expect(out.text).toContain('@e1 button "Background"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
it("can filter refs blocked by active positioned regions without changing layer output", () => {
|
||||
@@ -386,7 +391,7 @@ describe("renderVom single-layer page", () => {
|
||||
expect(out.text).not.toContain("active-region");
|
||||
expect(out.text).not.toContain("Background");
|
||||
expect(out.text).toContain('@e1 button "Foreground"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||
});
|
||||
|
||||
it("renders conditional surface items inline on the trigger", () => {
|
||||
@@ -405,7 +410,7 @@ describe("renderVom single-layer page", () => {
|
||||
});
|
||||
|
||||
expect(out.text).toContain('@e1 button "Products" [hover first: Shoes | Bags | Accessories]');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
it("injects active scope blocks immediately after their trigger without refs", () => {
|
||||
@@ -431,7 +436,7 @@ describe("renderVom single-layer page", () => {
|
||||
" Bob - great sound",
|
||||
].join("\n"),
|
||||
);
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
it("does not render redundant children inside named ref controls", () => {
|
||||
@@ -445,7 +450,7 @@ describe("renderVom single-layer page", () => {
|
||||
|
||||
expect(out.text).toContain('@e1 button "Save"');
|
||||
expect(out.text).not.toContain("disk icon");
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
it("does not render redundant children inside native text inputs", () => {
|
||||
@@ -612,7 +617,7 @@ describe("renderVom double-layer page", () => {
|
||||
].join("\n"),
|
||||
);
|
||||
expect(out.text).not.toContain("底层按钮");
|
||||
expect(out.refs).toEqual([
|
||||
expect(coreRefs(out.refs)).toEqual([
|
||||
{ ref: "e1", backendNodeId: 4 },
|
||||
{ ref: "e2", backendNodeId: 5 },
|
||||
]);
|
||||
@@ -664,4 +669,24 @@ describe("renderVom double-layer page", () => {
|
||||
expect(out.text).toContain(' @e2 textbox "密码" [filled] ="•••"');
|
||||
expect(out.refs.map((r) => r.backendNodeId)).toEqual([101, 102]);
|
||||
});
|
||||
|
||||
it("redactValues omits literal form values", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Form" }),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
role: "textbox",
|
||||
name: "邮箱",
|
||||
value: "user@example.com",
|
||||
inputState: "filled",
|
||||
tag: "input",
|
||||
}),
|
||||
]),
|
||||
{ redactValues: true },
|
||||
);
|
||||
expect(out.text).toContain("[filled]");
|
||||
expect(out.text).not.toContain("user@example.com");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5,6 +5,7 @@ export type {
|
||||
CondSurface,
|
||||
LayerKind,
|
||||
Rect,
|
||||
RenderedRef,
|
||||
Viewport,
|
||||
VomNode,
|
||||
VomOptions,
|
||||
|
||||
@@ -4,6 +4,7 @@ import type {
|
||||
BlockingLayer,
|
||||
CondSurface,
|
||||
Rect,
|
||||
RenderedRef,
|
||||
VomNode,
|
||||
VomOptions,
|
||||
VomResult,
|
||||
@@ -606,6 +607,7 @@ function renderNodeLine(
|
||||
ref: string | undefined,
|
||||
context: string[] = [],
|
||||
surface: CondSurface | undefined = undefined,
|
||||
redactValues = false,
|
||||
): string {
|
||||
let line = `${" ".repeat(depth)}${ref ? `@${ref} ` : ""}${node.role}`;
|
||||
|
||||
@@ -627,11 +629,12 @@ function renderNodeLine(
|
||||
}
|
||||
const placeholder = cleaned(node.placeholder);
|
||||
if (placeholder) line += ` placeholder=${JSON.stringify(placeholder)}`;
|
||||
const value = node.sensitive
|
||||
? rawValue !== undefined
|
||||
? SENSITIVE_MASK
|
||||
: undefined
|
||||
: rawValue?.slice(0, MAX_VALUE_LENGTH);
|
||||
const value =
|
||||
node.sensitive || redactValues
|
||||
? rawValue !== undefined
|
||||
? SENSITIVE_MASK
|
||||
: undefined
|
||||
: rawValue?.slice(0, MAX_VALUE_LENGTH);
|
||||
if (value !== undefined) line += ` =${JSON.stringify(value)}`;
|
||||
|
||||
if (surface && surface.subItems.length > 0) {
|
||||
@@ -684,11 +687,12 @@ function shouldSkipRedundantChildren(node: VomNode, state: RenderState): boolean
|
||||
|
||||
interface RenderState {
|
||||
lines: string[];
|
||||
refs: Array<{ ref: string; backendNodeId: number }>;
|
||||
refs: RenderedRef[];
|
||||
nextRef: number;
|
||||
tokens: number;
|
||||
maxDepth: number;
|
||||
maxTokens: number;
|
||||
redactValues: boolean;
|
||||
truncated: boolean;
|
||||
stopped: boolean;
|
||||
children: Map<number | null, VomNode[]>;
|
||||
@@ -753,7 +757,7 @@ function renderTree(
|
||||
const ref = isVomReferenceNode(node) ? `e${state.nextRef}` : undefined;
|
||||
const context = ref ? handleContext(node, state) : [];
|
||||
const surface = state.surfaceMap.get(node.id);
|
||||
const line = renderNodeLine(node, depth, ref, context, surface);
|
||||
const line = renderNodeLine(node, depth, ref, context, surface, state.redactValues);
|
||||
const nextTokens = state.tokens + estimateTokens(line);
|
||||
if (nextTokens > state.maxTokens) {
|
||||
state.truncated = true;
|
||||
@@ -764,7 +768,16 @@ function renderTree(
|
||||
state.lines.push(line);
|
||||
state.tokens = nextTokens;
|
||||
if (ref) {
|
||||
state.refs.push({ ref, backendNodeId: node.id });
|
||||
const lineIndex = state.lines.length - 1;
|
||||
const refName = cleaned(node.name);
|
||||
state.refs.push({
|
||||
ref,
|
||||
backendNodeId: node.id,
|
||||
role: node.role,
|
||||
...(refName ? { name: refName } : {}),
|
||||
...(context.length > 0 ? { ctx: context.join(" > ") } : {}),
|
||||
line: lineIndex,
|
||||
});
|
||||
state.nextRef += 1;
|
||||
}
|
||||
|
||||
@@ -793,6 +806,7 @@ function renderNodes(
|
||||
tokens: estimateTokens(initialLines.join("\n")),
|
||||
maxDepth: options.maxDepth ?? DEFAULT_MAX_DEPTH,
|
||||
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
|
||||
redactValues: options.redactValues ?? false,
|
||||
truncated: false,
|
||||
stopped: false,
|
||||
children,
|
||||
|
||||
@@ -59,6 +59,10 @@ export interface VomScene {
|
||||
export interface VomOptions {
|
||||
maxDepth?: number;
|
||||
maxTokens?: number;
|
||||
/**
|
||||
* When true, all form values render as [filled]/[empty] without literal values.
|
||||
*/
|
||||
redactValues?: boolean;
|
||||
/**
|
||||
* Experimental: filter refs that are geometrically blocked by foreground
|
||||
* fixed/absolute/sticky regions. Disabled by default and does not alter the
|
||||
@@ -79,9 +83,19 @@ export interface ActiveScopeBlock {
|
||||
lines: string[];
|
||||
}
|
||||
|
||||
export interface RenderedRef {
|
||||
ref: string;
|
||||
backendNodeId: number;
|
||||
role?: string;
|
||||
name?: string;
|
||||
ctx?: string;
|
||||
/** Zero-based line index in `VomResult.text`. */
|
||||
line: number;
|
||||
}
|
||||
|
||||
export interface VomResult {
|
||||
text: string;
|
||||
refs: Array<{ ref: string; backendNodeId: number }>;
|
||||
refs: RenderedRef[];
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
|
||||
+14
-6
@@ -265,19 +265,27 @@ retry — complete the task autonomously or stop gracefully.
|
||||
|
||||
### Recording — `bsk record`
|
||||
|
||||
Capture the user's own actions in the Agent Window to a `trace.json`, for later LLM-driven automation:
|
||||
Capture the user's own actions in the Agent Window to a **trace bundle**, for later LLM-driven automation. New CLI builds request **trace v3** (page observations + action chain); older extensions may still return **trace v2** (actions only), which the CLI exports as a single `trace.json` without `pages/`.
|
||||
|
||||
```bash
|
||||
bsk record start --browser <instance-id-or-label> [--url https://…] [--purpose "publish a wiki doc"] [--output trace.json]
|
||||
bsk record start --browser <instance-id-or-label> \
|
||||
[--url https://…] [--purpose "publish a wiki doc"] \
|
||||
[--max-page-tokens 3000] [--redact-values] \
|
||||
[--output trace]
|
||||
# `--url` is optional; default https://example.com/ when omitted (must be http(s)).
|
||||
# Blocks until the user clicks Finish in the recording panel, then writes ./trace.json and closes the window.
|
||||
# Blocks until the user clicks Finish in the recording panel, then writes:
|
||||
# trace/trace.json — action chain (+ state index when v3)
|
||||
# trace/pages/ — v3 only: one `.vom.txt` observe snapshot per settled page state
|
||||
|
||||
bsk record stop [--output trace.json] # terminal fallback if the browser panel is unavailable
|
||||
bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable
|
||||
```
|
||||
|
||||
- The trace is a **record-only action log** (a `pages[]` dictionary + `navigate`/`click`/`fill`/`select`/`press` steps with `target` descriptors). It records *what the user did*; deciding which inputs are variable is left to the executing agent.
|
||||
- **v3 bundle (preferred):** `--output` is a directory (default `./trace`) containing `trace.json` and `pages/`. `trace.json` lists `states[]` (page observation ids) and `steps[]` (each step binds `state` = observe snapshot *before* the action and `result.state` = snapshot *after* settle). Page bodies live in `pages/sN.vom.txt` using the same VOM format as `bsk observe`.
|
||||
- **v2 fallback:** when the connected extension is older, export may contain only legacy `trace.json` with `pages[]` action context and no `pages/` observation files. Update the extension for full v3 bundles.
|
||||
- Each `states[]` entry is one **settled page observation** (captured after recording start, navigation landing, or action settle — not on a timer).
|
||||
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
|
||||
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
||||
- There is **no** `bsk replay` — to redo a flow, read the trace and reuse the existing `session` / `snapshot` / `@eN` / `click` / `fill` tools. Follow **Stop when the goal is met**.
|
||||
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
|
||||
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
||||
|
||||
## Error handling
|
||||
|
||||
Reference in New Issue
Block a user