Compare commits

...
Author SHA1 Message Date
“haonan”andCursor f7e08f461b feat(record): capture interactions in embedded frames
Track recording per document so injectable child frames are captured while partial frame failures remain visible in exported traces.

Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-17 14:41:02 +08:00
“haonan” 61a0e0d218 fix(record): keep committed empty fill steps in exported traces 2026-08-17 11:02:10 +08:00
“haonan” 67b7227abf fix(popup): point record prompt at the ./trace directory bundle 2026-08-13 19:55:34 +08:00
“haonan” 02d1d69379 docs(ux): clarify version mismatches and update record skill docs
Surface actionable update guidance in the popup and connection errors, document
the trace v3 directory bundle in SKILL.md.
2026-08-13 19:40:54 +08:00
“haonan” 025751ef03 feat(cli): write record output as a trace directory bundle
Persist `trace.json` plus per-state page VOM files under `--output`, tolerate
raced session teardown on stop, and clarify version-mismatch recovery hints.
2026-08-13 19:38:05 +08:00
“haonan” 13a61b2688 feat(extension): record user flows as trace v3 page-state observations
Capture settled page observations into a states/steps trace, match targets by
geometry with capture fallbacks, and reuse VOM rendering with richer refs and
optional value redaction for record bundles.
2026-08-13 19:34:37 +08:00
“haonan” 1508839bf9 feat(protocol): bump to 1.1 and redefine record traces as state-linked v3
Replace page-indexed record traces with states/steps + FillCommit/hover/scroll,
and raise daemon/extension protocol floors together so old 1.0 peers are rejected.
2026-08-13 19:31:35 +08:00
71 changed files with 7333 additions and 1417 deletions
+4
View File
@@ -17,3 +17,7 @@ dist/
# Environment
.env.local
.env.*.local
.pnpm-store/
# Local recording output
/trace/
+63 -38
View File
@@ -1,4 +1,5 @@
import { useTranslation } from "@browser-skill/i18n/react";
import type { FrameCaptureFailure, FrameCaptureStatus } from "@/transport/types";
import {
type PointerEvent as ReactPointerEvent,
useCallback,
@@ -21,6 +22,8 @@ export interface RecordRequestData {
type Props = {
request: RecordRequestData | null;
captureStatus?: FrameCaptureStatus;
captureFailures?: FrameCaptureFailure[];
};
const VIEWPORT_MARGIN = 16;
@@ -52,7 +55,11 @@ function clampDragPos(
};
}
export function RecordOverlay({ request }: Props) {
export function RecordOverlay({
request,
captureStatus = "complete",
captureFailures = [],
}: Props) {
const { t } = useTranslation("extension");
const [show, setShow] = useState(false);
const [elapsedSeconds, setElapsedSeconds] = useState(0);
@@ -154,6 +161,7 @@ export function RecordOverlay({ request }: Props) {
const introVisible = elapsedSeconds * 1_000 < INTRO_DURATION_MS;
const positioned = dragPos !== null;
const partialCapture = captureStatus === "partial" || captureFailures.length > 0;
const defaultTransform = show
? "translateX(-50%) translateY(0)"
: "translateX(-50%) translateY(8px)";
@@ -208,8 +216,9 @@ export function RecordOverlay({ request }: Props) {
data-slot="record-overlay-status"
style={{
display: "flex",
alignItems: "center",
gap: 8,
flexDirection: "column",
alignItems: "flex-start",
gap: 2,
minWidth: 0,
flex: 1,
borderRadius: 9999,
@@ -218,41 +227,57 @@ export function RecordOverlay({ request }: Props) {
paddingRight: 4,
}}
>
<span
data-slot="record-overlay-indicator"
style={{
width: 9,
height: 9,
borderRadius: "50%",
backgroundColor: "#ef4444",
flexShrink: 0,
animation: "bsk-rec-pulse 1.4s ease-in-out infinite",
}}
/>
<span
data-slot="record-overlay-label"
style={{
display: "block",
flex: "0 1 auto",
fontSize: 16,
fontWeight: 500,
color: "#333",
whiteSpace: "nowrap",
userSelect: "none",
overflow: "hidden",
textOverflow: "ellipsis",
width: introVisible ? "auto" : 48,
maxWidth: introVisible ? 280 : 48,
fontVariantNumeric: introVisible ? undefined : "tabular-nums",
textAlign: introVisible ? "left" : "center",
margin: 0,
padding: 0,
minWidth: 0,
transition: "max-width 180ms ease-out",
}}
>
{introVisible ? t("recordOverlay.recording") : formatElapsed(elapsedSeconds)}
</span>
<div style={{ display: "flex", alignItems: "center", gap: 8, minWidth: 0, width: "100%" }}>
<span
data-slot="record-overlay-indicator"
style={{
width: 9,
height: 9,
borderRadius: "50%",
backgroundColor: "#ef4444",
flexShrink: 0,
animation: "bsk-rec-pulse 1.4s ease-in-out infinite",
}}
/>
<span
data-slot="record-overlay-label"
style={{
display: "block",
flex: "0 1 auto",
fontSize: 16,
fontWeight: 500,
color: "#333",
whiteSpace: "nowrap",
userSelect: "none",
overflow: "hidden",
textOverflow: "ellipsis",
width: introVisible ? "auto" : 48,
maxWidth: introVisible ? 280 : 48,
fontVariantNumeric: introVisible ? undefined : "tabular-nums",
textAlign: introVisible ? "left" : "center",
margin: 0,
padding: 0,
minWidth: 0,
transition: "max-width 180ms ease-out",
}}
>
{introVisible ? t("recordOverlay.recording") : formatElapsed(elapsedSeconds)}
</span>
</div>
{partialCapture ? (
<span
data-slot="record-overlay-partial-warning"
style={{
fontSize: 12,
lineHeight: "16px",
color: "#b45309",
whiteSpace: "normal",
maxWidth: 280,
}}
>
{t("recordOverlay.partialCapture")}
</span>
) : null}
</div>
<button
type="button"
@@ -178,4 +178,30 @@ describe("RecordOverlay", () => {
expect(pill.style.bottom).toBe("auto");
expect(pill.style.transform).toBe("none");
});
it("shows a persistent partial-capture warning when frame capture is incomplete", () => {
const { container } = render(
createElement(
I18nextProvider,
{ i18n },
createElement(RecordOverlay, {
request: { id: "rec-1", onFinish: vi.fn() },
captureStatus: "partial",
captureFailures: [
{
reason: "not_injectable",
frame_id: 2,
url: "chrome://settings",
},
],
}),
),
);
const warning = container.querySelector("[data-slot='record-overlay-partial-warning']");
expect(warning).toBeTruthy();
expect(warning?.textContent).toBe(
i18n.t("recordOverlay.partialCapture", { ns: "extension" }),
);
});
});
@@ -1,5 +1,5 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
import { RECORD_START, RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
import { handleRecordContentMessage, startRecordCapture } from "../record-capture";
vi.stubGlobal("chrome", {
@@ -96,6 +96,65 @@ describe("handleRecordContentMessage stop/cancel", () => {
expect(dispose).not.toHaveBeenCalled();
expect(onStop).not.toHaveBeenCalled();
});
it("keeps a failed STOP retryable and redelivers its recorded step", async () => {
document.body.innerHTML = `<label for="retry">Draft</label><input id="retry" />`;
const sendMessage = vi.mocked(chrome.runtime.sendMessage);
sendMessage.mockReset();
sendMessage.mockRejectedValueOnce(new Error("service worker unavailable"));
sendMessage.mockResolvedValueOnce(undefined);
let activeRequestId: string | null = null;
let capture: ReturnType<typeof startRecordCapture> | null = null;
const onStop = vi.fn();
const dispatch = (
message:
| { type: typeof RECORD_START; requestId: string; startedAtMs?: number }
| { type: typeof RECORD_STOP; requestId: string },
sendResponse?: (response: unknown) => void,
) =>
handleRecordContentMessage(
message,
{
activeRequestId,
capture,
setActiveRequestId: (id) => {
activeRequestId = id;
},
setCapture: (next) => {
capture = next;
},
onStart: vi.fn(),
onStop,
},
sendResponse,
);
dispatch({ type: RECORD_START, requestId: "rec-retry" });
const input = document.querySelector("input")!;
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
input.value = "dirty final value";
input.dispatchEvent(new Event("input", { bubbles: true }));
const firstResponse = vi.fn();
expect(dispatch({ type: RECORD_STOP, requestId: "rec-retry" }, firstResponse)).toBe(true);
await vi.waitFor(() =>
expect(firstResponse).toHaveBeenCalledWith({
ok: false,
error: "failed to deliver one or more recorded steps",
}),
);
expect(activeRequestId).toBe("rec-retry");
expect(onStop).not.toHaveBeenCalled();
const retryResponse = vi.fn();
expect(dispatch({ type: RECORD_STOP, requestId: "rec-retry" }, retryResponse)).toBe(true);
await vi.waitFor(() => expect(retryResponse).toHaveBeenCalledWith({ ok: true }));
expect(sendMessage).toHaveBeenCalledTimes(2);
expect(activeRequestId).toBeNull();
expect(onStop).toHaveBeenCalledTimes(1);
});
});
describe("record-capture semantic", () => {
@@ -228,6 +287,7 @@ describe("record-capture semantic", () => {
expect(steps[0]).toMatchObject({
op: "hover",
target: { role: "button", name: "Open user navigation menu" },
geometry: { tag: "button", rect: { x: 900, y: 8, w: 32, h: 32 } },
});
expect(steps[1]).toMatchObject({
op: "click",
@@ -1554,3 +1614,79 @@ describe("record-capture semantic", () => {
expect(steps.map((s) => s.op)).toEqual(["click"]);
});
});
describe("record-capture child frame", () => {
let steps: RecordStepPayload[];
beforeEach(() => {
steps = [];
document.body.innerHTML = `
<label for="child-q">Child query</label>
<input id="child-q" name="child-q" />
<button type="button" aria-label="Child search">Search</button>
<select aria-label="Child status">
<option value="on">On</option>
<option value="off">Off</option>
</select>
`;
});
it("stamps a document token on start and clears it on dispose", () => {
const capture = startRecordCapture("rec-child-token", (step) => steps.push(step), {
frameMode: "child",
});
const token = document.documentElement.getAttribute("data-bsk-record-document");
expect(token).toBeTruthy();
click(document.querySelector("button")!);
expect(steps[0]?.documentToken).toBe(token);
expect(steps[0]?.frame_url).toBeTruthy();
capture.dispose();
expect(document.documentElement.hasAttribute("data-bsk-record-document")).toBe(false);
});
it("records click, fill, select, and press in child frame mode", () => {
const capture = startRecordCapture("rec-child-ops", (step) => steps.push(step), {
frameMode: "child",
});
const input = document.querySelector("input")!;
const select = document.querySelector("select")!;
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
input.value = "child value";
input.dispatchEvent(new Event("input", { bubbles: true }));
input.dispatchEvent(new FocusEvent("focusout", { bubbles: true }));
select.value = "off";
select.dispatchEvent(new Event("change", { bubbles: true }));
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
input.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", bubbles: true }));
click(document.querySelector("button")!);
capture.dispose();
expect(steps.map((s) => s.op)).toEqual(
expect.arrayContaining(["fill", "select", "press", "click"]),
);
expect(steps.every((s) => s.documentToken)).toBe(true);
});
it("commits the last fill when dispose runs in child frame mode", () => {
const capture = startRecordCapture("rec-child-fill-stop", (step) => steps.push(step), {
frameMode: "child",
});
const input = document.querySelector("input")!;
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
input.value = "pending child fill";
input.dispatchEvent(new Event("input", { bubbles: true }));
capture.dispose();
expect(steps.some((s) => s.op === "fill" && s.value === "pending child fill")).toBe(true);
});
it("does not emit navigate steps for child frame navigations", () => {
const capture = startRecordCapture("rec-child-nav", (step) => steps.push(step), {
frameMode: "child",
});
history.pushState({}, "", "/child-frame-path");
capture.dispose();
expect(steps.filter((s) => s.op === "navigate")).toHaveLength(0);
});
});
+141 -35
View File
@@ -1,9 +1,9 @@
import {
type CaptureTargetDescriptor,
describeEventTarget,
describeTarget,
resolveClickableElement,
resolveHoverElement,
type TargetDescriptor,
} from "@/lib/describe-target";
import {
evaluateHoverTrigger,
@@ -21,12 +21,18 @@ import {
RECORD_STEP,
RECORD_STOP,
type RecordCancelMessage,
type RecordFrameMode,
type RecordStartAck,
type RecordStartMessage,
type RecordStepPayload,
type RecordStopAck,
type RecordStopMessage,
} from "@/lib/record-bridge";
import {
clearRecordDocumentToken,
createRecordDocumentToken,
stampRecordDocumentToken,
} from "@/lib/record-document-token";
import { shouldRecordPress } from "@/lib/trace-reducer";
import {
closestHoverSurfaceCandidate,
@@ -39,17 +45,26 @@ import {
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
const failedStepDeliveries = new Set<string>();
const failedStepPayloads = new Map<string, RecordStepPayload[]>();
const pendingStopFlushes = new Map<string, Promise<RecordStopAck>>();
const knownRecordRequests = new Set<string>();
function sendRecordStep(requestId: string, step: RecordStepPayload): void {
function deliverRecordStep(requestId: string, step: RecordStepPayload): Promise<boolean> {
const payload = { type: RECORD_STEP, requestId, step };
const pending = Promise.resolve(chrome.runtime.sendMessage(payload)).then(
return Promise.resolve(chrome.runtime.sendMessage(payload)).then(
() => true,
() => {
failedStepDeliveries.add(requestId);
const failed = failedStepPayloads.get(requestId) ?? [];
failed.push(step);
failedStepPayloads.set(requestId, failed);
return false;
},
);
}
function sendRecordStep(requestId: string, step: RecordStepPayload): void {
const pending = deliverRecordStep(requestId, step);
const sends = pendingStepSends.get(requestId) ?? new Set<Promise<boolean>>();
sends.add(pending);
pendingStepSends.set(requestId, sends);
@@ -65,14 +80,15 @@ export interface RecordCaptureController {
interface FillSession {
element: FillableElement;
target: TargetDescriptor;
target: CaptureTargetDescriptor;
baselineValue: string;
lastValue: string;
pendingCommit?: "enter" | "suggestion" | "blur";
}
interface HoverCandidate {
element: Element;
target: TargetDescriptor;
target: CaptureTargetDescriptor;
recordedAt: number;
score: number;
eligible: boolean;
@@ -157,6 +173,28 @@ function nearbyFillableFromSearchChrome(target: Element): FillableElement | null
return null;
}
function captureGeometry(el: Element): RecordStepPayload["geometry"] {
const rect = el.getBoundingClientRect();
const style = window.getComputedStyle(el);
return {
rect: { x: rect.x, y: rect.y, w: rect.width, h: rect.height },
scrollX: window.scrollX,
scrollY: window.scrollY,
position: style.position,
tag: el.tagName.toLowerCase(),
};
}
function geometryForEventTarget(
target: EventTarget | null,
): RecordStepPayload["geometry"] | undefined {
if (!(target instanceof Element)) return undefined;
const clickable = target.closest(
'a,button,input,select,textarea,[role="button"],[role="link"],[role="menuitem"],[contenteditable="true"]',
);
return captureGeometry(clickable ?? target);
}
function fillableValue(el: FillableElement): string {
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
return el.value;
@@ -232,7 +270,7 @@ function collectHoverTriggerLabelText(root: Element): string {
return normalizeLabelText(text);
}
function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDescriptor {
function compactHoverTargetName(el: Element, desc: CaptureTargetDescriptor): CaptureTargetDescriptor {
if (!desc.name) return desc;
const fullText = normalizeLabelText(el.textContent ?? "");
const compactName = collectHoverTriggerLabelText(el);
@@ -250,7 +288,7 @@ function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDesc
return desc;
}
function isWeakHoverTarget(target: TargetDescriptor): boolean {
function isWeakHoverTarget(target: CaptureTargetDescriptor): boolean {
return !target.role && !target.name && target.tag === "div";
}
@@ -261,9 +299,9 @@ function looksLikeAvatarElement(el: Element): boolean {
function normalizeHoverTarget(
el: Element,
desc: TargetDescriptor,
desc: CaptureTargetDescriptor,
decision: HoverTriggerDecision,
): TargetDescriptor {
): CaptureTargetDescriptor {
if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
return { ...desc, name: "image" };
}
@@ -277,7 +315,7 @@ function normalizeHoverTarget(
return desc;
}
function hoverTriggerSignals(el: Element, desc: TargetDescriptor) {
function hoverTriggerSignals(el: Element, desc: CaptureTargetDescriptor) {
if (!(el instanceof HTMLElement)) return null;
const style = hoverTriggerStyle(el);
return {
@@ -370,12 +408,37 @@ function scheduleInputCompletionCommit(
setTimeout(syncAndCommit, 0);
}
export interface RecordCaptureOptions {
frameMode?: RecordFrameMode;
}
export function startRecordCapture(
_requestId: string,
sendStep: (step: RecordStepPayload) => void,
options: RecordCaptureOptions = {},
): RecordCaptureController {
const frameMode = options.frameMode ?? "top";
const emitNavigate = frameMode === "top";
const documentToken = createRecordDocumentToken();
stampRecordDocumentToken(documentToken);
const capturedAtMs = () => Date.now();
const resolveTopPageUrl = (): string => {
try {
return window.top?.location.href ?? location.href;
} catch {
return location.href;
}
};
const emitStep = (step: RecordStepPayload) => {
sendStep({ page_url: location.href, ...step });
sendStep({
page_url: resolveTopPageUrl(),
frame_url: location.href,
documentToken,
capturedAtMs: capturedAtMs(),
...step,
});
};
const hoverSurfaceStateMap = (states: HoverSurfaceState[]): Map<Element, string> =>
new Map(states.map((state) => [state.element, state.signature]));
@@ -411,14 +474,17 @@ export function startRecordCapture(
emitStep({
op: "fill",
target: session.target,
geometry: captureGeometry(session.element),
value,
commit: session.pendingCommit ?? "blur",
...(isPassword ? { redacted: true } : {}),
});
};
const commitFillSession = () => {
const commitFillSession = (commit: "enter" | "suggestion" | "blur" = "blur") => {
if (!fillSession || composing) return;
const session = fillSession;
session.pendingCommit = commit;
fillSession = null;
emitFill(session);
committedValues.set(session.element, session.lastValue);
@@ -443,6 +509,7 @@ export function startRecordCapture(
};
const emitNavigateIfChanged = (causedByAction?: boolean) => {
if (!emitNavigate) return;
if (location.href === lastUrl) return;
commitFillSession();
lastUrl = location.href;
@@ -624,6 +691,7 @@ export function startRecordCapture(
emitStep({
op: "hover",
target: hover.target,
geometry: captureGeometry(hover.element),
});
emittedHoverElements.add(hover.element);
};
@@ -678,6 +746,7 @@ export function startRecordCapture(
emitStep({
op: "click",
target,
geometry: geometryForEventTarget(eventTarget(event)),
expects_navigation: true,
});
};
@@ -779,7 +848,7 @@ export function startRecordCapture(
scheduleInputCompletionCommit(
sessionElement,
syncFillSessionValue,
commitFillSession,
() => commitFillSession("suggestion"),
(el) => fillSession?.element === el,
);
return;
@@ -839,6 +908,7 @@ export function startRecordCapture(
emitStep({
op: "select",
target: desc,
geometry: captureGeometry(target),
values,
labels,
expects_navigation: true,
@@ -879,7 +949,7 @@ export function startRecordCapture(
};
}
if (fillable) {
commitFillSession();
commitFillSession(event.key === "Enter" ? "enter" : "blur");
}
const desc = describeEventTarget(target);
if (!desc && !event.key) return;
@@ -887,7 +957,7 @@ export function startRecordCapture(
emitStep({
op: "press",
key: event.key,
...(desc ? { target: desc } : {}),
...(desc ? { target: desc, geometry: geometryForEventTarget(target) } : {}),
...(modifiers.length ? { modifiers } : {}),
expects_navigation: event.key === "Enter",
});
@@ -906,9 +976,12 @@ export function startRecordCapture(
const urlObserver = new MutationObserver(() => emitNavigateIfChanged());
urlObserver.observe(document, { subtree: true, childList: true });
const onUrlEvent = () => emitNavigateIfChanged();
// Wrap: passing commitFillSession directly would forward the DOM event as
// the `commit` argument and stamp it onto the recorded fill step.
const onPageHide = () => commitFillSession();
window.addEventListener("hashchange", onUrlEvent);
window.addEventListener("popstate", onUrlEvent);
window.addEventListener("pagehide", commitFillSession);
window.addEventListener("pagehide", onPageHide);
const originalPushState = history.pushState;
const originalReplaceState = history.replaceState;
@@ -924,6 +997,7 @@ export function startRecordCapture(
return {
dispose() {
commitFillSession();
clearRecordDocumentToken();
document.removeEventListener("click", onClick, true);
document.removeEventListener("mouseover", onMouseOver, true);
document.removeEventListener("focusin", onFocusIn, true);
@@ -936,7 +1010,7 @@ export function startRecordCapture(
urlObserver.disconnect();
window.removeEventListener("hashchange", onUrlEvent);
window.removeEventListener("popstate", onUrlEvent);
window.removeEventListener("pagehide", commitFillSession);
window.removeEventListener("pagehide", onPageHide);
history.pushState = originalPushState;
history.replaceState = originalReplaceState;
},
@@ -956,8 +1030,8 @@ export function handleRecordContentMessage(
capture: RecordCaptureController | null;
setActiveRequestId(id: string | null): void;
setCapture(capture: RecordCaptureController | null): void;
onStart(requestId: string, startedAtMs?: number): void;
onStop(): void;
onStart?(requestId: string, startedAtMs?: number): void;
onStop?(): void;
},
sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
): boolean {
@@ -965,15 +1039,23 @@ export function handleRecordContentMessage(
if (!knownRecordRequests.has(message.requestId)) {
knownRecordRequests.add(message.requestId);
failedStepDeliveries.delete(message.requestId);
failedStepPayloads.delete(message.requestId);
}
state.capture?.dispose();
const frameMode = message.frameMode ?? "top";
state.setCapture(
startRecordCapture(message.requestId, (step) => {
sendRecordStep(message.requestId, step);
}),
startRecordCapture(
message.requestId,
(step) => {
sendRecordStep(message.requestId, step);
},
{ frameMode },
),
);
state.setActiveRequestId(message.requestId);
state.onStart(message.requestId, message.startedAtMs);
if (message.showOverlay !== false && frameMode === "top") {
state.onStart?.(message.requestId, message.startedAtMs);
}
sendResponse?.({ ok: true });
return sendResponse !== undefined;
}
@@ -988,31 +1070,55 @@ export function handleRecordContentMessage(
state.capture?.dispose();
state.setCapture(null);
const finishStop = () => {
state.onStop();
state.onStop?.();
state.setActiveRequestId(null);
};
if (isRecordStopMessage(message) && sendResponse) {
const existingFlush = pendingStopFlushes.get(message.requestId);
if (existingFlush) {
void existingFlush.then(sendResponse);
return true;
}
const failedBeforeStop = [...(failedStepPayloads.get(message.requestId) ?? [])];
if (failedBeforeStop.length > 0) {
failedStepPayloads.delete(message.requestId);
failedStepDeliveries.delete(message.requestId);
}
const pending = [...(pendingStepSends.get(message.requestId) ?? [])];
void Promise.all(pending).then((delivered) => {
finishStop();
const succeeded = delivered.every(Boolean) && !failedStepDeliveries.has(message.requestId);
const flush = Promise.all(pending).then(async (delivered): Promise<RecordStopAck> => {
const retried = await Promise.all(
failedBeforeStop.map((step) => deliverRecordStep(message.requestId, step)),
);
const succeeded =
delivered.every(Boolean) &&
retried.every(Boolean) &&
!failedStepDeliveries.has(message.requestId);
if (succeeded) {
failedStepDeliveries.delete(message.requestId);
failedStepPayloads.delete(message.requestId);
knownRecordRequests.delete(message.requestId);
finishStop();
}
return succeeded
? { ok: true }
: {
ok: false,
error: "failed to deliver one or more recorded steps",
};
});
pendingStopFlushes.set(message.requestId, flush);
void flush.then(sendResponse).finally(() => {
if (pendingStopFlushes.get(message.requestId) === flush) {
pendingStopFlushes.delete(message.requestId);
}
sendResponse(
succeeded
? { ok: true }
: {
ok: false,
error: "failed to deliver one or more recorded steps",
},
);
});
return true;
}
if (isRecordCancelMessage(message)) {
failedStepDeliveries.delete(message.requestId);
failedStepPayloads.delete(message.requestId);
pendingStopFlushes.delete(message.requestId);
knownRecordRequests.delete(message.requestId);
}
finishStop();
+23 -1
View File
@@ -33,6 +33,9 @@ import {
defaultBorrowChromeWindows,
requestBorrowConfirmation,
} from "@/tools/borrow-confirmation";
import {
type RecordCaptureStatusMessage,
} from "@/lib/record-bridge";
import { ToolDispatcher } from "@/tools/dispatcher";
import {
attachRecordFinishListener,
@@ -172,8 +175,27 @@ export default defineBackground(() => {
dispatcher.start();
const recordDeps = {
tabsApi: chrome.tabs,
cdp,
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
chrome.tabs.sendMessage(tabId, msg),
sendToDocument: (
tabId: number,
documentId: string,
msg: Parameters<typeof chrome.tabs.sendMessage>[1],
) => chrome.tabs.sendMessage(tabId, msg, { documentId }),
getAllFrames: async (tabId: number) => {
if (!chrome.webNavigation?.getAllFrames) return [];
const frames = await chrome.webNavigation.getAllFrames({ tabId });
return (frames ?? []).map((frame) => ({
frameId: frame.frameId,
documentId: frame.documentId,
url: frame.url,
parentFrameId: frame.parentFrameId,
}));
},
broadcastCaptureStatus: async (tabId: number, msg: RecordCaptureStatusMessage) => {
await chrome.tabs.sendMessage(tabId, msg, { frameId: 0 });
},
bypassOverlay: async (tabId: number, enabled: boolean) => {
try {
await chrome.tabs.sendMessage(tabId, {
@@ -188,7 +210,7 @@ export default defineBackground(() => {
// Message listeners stay up (cheap; fire only for record message types).
// Tab / webNavigation observation attaches lazily while a recording is
// active — see ensureBrowserObservationListeners in tools/record.ts.
attachRecordStepListener();
attachRecordStepListener(recordDeps);
attachRecordFinishListener(recordDeps);
attachRecordQueryListener(recordDeps);
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
+23 -2
View File
@@ -44,12 +44,15 @@ import {
} from "@/lib/overlay-bridge";
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
import {
isRecordCaptureStatusMessage,
RECORD_FINISH,
RECORD_QUERY,
type RecordCaptureStatusMessage,
type RecordQueryResponse,
type RecordStartAck,
type RecordStopAck,
} from "@/lib/record-bridge";
import type { FrameCaptureFailure, FrameCaptureStatus } from "@/transport/types";
import type {
BorrowCancelMessage,
BorrowRequestMessage,
@@ -70,6 +73,8 @@ export default defineContentScript({
const overlays = new OverlayController();
let recordCapture: RecordCaptureController | null = null;
let activeRecordRequestId: string | null = null;
let recordCaptureStatus: FrameCaptureStatus = "complete";
let recordCaptureFailures: FrameCaptureFailure[] = [];
let reactRoot: ReactDOM.Root | null = null;
let overlayHost: HTMLElement | null = null;
let overlayContainer: HTMLElement | null = null;
@@ -188,7 +193,11 @@ export default defineContentScript({
requests: overlayState.borrowRequests,
}),
React.createElement(HelpRequestOverlay, { request: overlayState.activeHelp }),
React.createElement(RecordOverlay, { request: overlayState.activeRecord }),
React.createElement(RecordOverlay, {
request: overlayState.activeRecord,
captureStatus: recordCaptureStatus,
captureFailures: recordCaptureFailures,
}),
React.createElement(ControlOverlay, {
visible: controlOverlayVisible,
interrupting: overlayState.interrupting,
@@ -260,7 +269,8 @@ export default defineContentScript({
| CaptureSuppressMessage
| OverlayAgentOverlayResetMessage
| OverlayAgentStateMessage
| OverlayAutomationBypassMessage,
| OverlayAutomationBypassMessage
| RecordCaptureStatusMessage,
_sender: chrome.runtime.MessageSender,
sendResponse: (response: BorrowResponseMessage | HelpAckMessage | CaptureSuppressAck) => void,
) => {
@@ -268,6 +278,13 @@ export default defineContentScript({
return captureSuppress.handleMessage(message, sendResponse);
}
if (isRecordCaptureStatusMessage(message)) {
recordCaptureStatus = message.status;
recordCaptureFailures = message.failures ?? [];
renderAll();
return false;
}
if (isRecordContentMessage(message)) {
const needsAsync = handleRecordContentMessage(
message,
@@ -295,6 +312,8 @@ export default defineContentScript({
},
onStop: () => {
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
recordCaptureStatus = "complete";
recordCaptureFailures = [];
renderAll();
},
},
@@ -449,6 +468,8 @@ export default defineContentScript({
) {
const requestId = recordQuery.requestId;
const startedAtMs = recordQuery.startedAtMs;
recordCaptureStatus = recordQuery.captureStatus ?? "complete";
recordCaptureFailures = recordQuery.captureFailures ?? [];
overlays.setAgentRecordRequest({
id: requestId,
...(typeof startedAtMs === "number" ? { startedAtMs } : {}),
@@ -100,6 +100,24 @@ describe("App", () => {
expect(screen.queryByText("录制你的操作,供 Agent 参考")).toBeNull();
});
it("renders localized copy for a version failure instead of protocol details", () => {
mockUseConnectionState.mockReturnValue({
snapshot: {
...baseSnapshot,
lastError:
"[handshake] daemon rejected handshake: version_too_old — peer protocol 1.0 is below local min_compatible_protocol 1.1",
},
statusState: "disconnected",
setLabel,
setConnectionEnabled,
});
render(<App />);
const error = screen.getByText(/bsk 与 BrowserSkill 扩展版本不兼容/);
expect(error.textContent).not.toContain("min_compatible_protocol");
});
it("shows single-line compact metadata and copies the instance id", async () => {
mockUseConnectionState.mockReturnValue({
snapshot: {
@@ -200,6 +218,13 @@ describe("App", () => {
expect(copied).toContain("--browser 03c3e47f");
expect(copied).toContain('--purpose "发布 wiki 文档"');
expect(copied).not.toMatch(/bsk record start[^\n]*--url/);
expect(copied).toContain("./trace");
expect(copied).toContain("trace.json");
expect(copied).toContain("pages/");
expect(copied).toContain("总结我做了什么");
expect(copied).not.toContain("不要重跑");
expect(copied).not.toContain("pages + steps");
expect(copied).not.toContain("@eN");
// Button label stays static; a transient toast confirms the copy.
expect(copyButton.textContent).toContain("复制录制指令");
await waitFor(() => expect(screen.getByRole("status").textContent).toContain("已复制"));
+5 -2
View File
@@ -83,6 +83,9 @@ export function App() {
const daemonProtocol = snapshot.handshake?.protocol_version ?? "—";
const extensionVersion = snapshot.extensionVersion || "—";
const instanceId = snapshot.instanceId || "—";
const errorMessage = snapshot.lastError?.includes("version_too_old")
? t("popup.versionIncompatibleError")
: snapshot.lastError;
const copyInstanceId = async () => {
if (!snapshot.instanceId || !navigator.clipboard?.writeText) return;
@@ -252,12 +255,12 @@ export function App() {
</div>
</section>
{snapshot.lastError && (
{errorMessage && (
<div
className="rounded-lg border border-destructive/25 bg-destructive/10 px-3 py-2 text-xs leading-snug text-destructive"
data-slot="popup-error"
>
{snapshot.lastError}
{errorMessage}
</div>
)}
@@ -0,0 +1,77 @@
import {
handleRecordContentMessage,
isRecordContentMessage,
type RecordCaptureController,
} from "@/content/record-capture";
import {
RECORD_QUERY,
type RecordQueryResponse,
} from "@/lib/record-bridge";
/** Capture-only content script for child frames (no React overlay). */
export default defineContentScript({
matches: ["<all_urls>"],
runAt: "document_end",
allFrames: true,
matchAboutBlank: true,
matchOriginAsFallback: true,
main(ctx) {
if (window.top === window) return;
let recordCapture: RecordCaptureController | null = null;
let activeRecordRequestId: string | null = null;
const onMessage = (
message: unknown,
_sender: chrome.runtime.MessageSender,
sendResponse: (response: unknown) => void,
) => {
if (isRecordContentMessage(message)) {
const needsAsync = handleRecordContentMessage(
message,
{
activeRequestId: activeRecordRequestId,
capture: recordCapture,
setActiveRequestId: (id) => {
activeRecordRequestId = id;
},
setCapture: (capture) => {
recordCapture = capture;
},
},
sendResponse as never,
);
return needsAsync;
}
return false;
};
async function queryActiveRecord(): Promise<void> {
try {
const recordQuery = (await chrome.runtime.sendMessage({
type: RECORD_QUERY,
})) as RecordQueryResponse | undefined;
if (
recordQuery?.active &&
typeof recordQuery.requestId === "string" &&
activeRecordRequestId === null
) {
// Background rearm sends RECORD_START; nothing to mount locally.
}
} catch (err) {
console.debug("[bsk record-frame] record query failed", err);
}
}
chrome.runtime.onMessage.addListener(onMessage);
void queryActiveRecord();
ctx.onInvalidated(() => {
chrome.runtime.onMessage.removeListener(onMessage);
recordCapture?.dispose();
recordCapture = null;
activeRecordRequestId = null;
});
},
});
@@ -27,19 +27,19 @@ function handshake(
describe("computeConnectedState (protocol-based compat)", () => {
it("returns connected when protocol strings match", () => {
expect(computeConnectedState(handshake("1.0", "1.0"), MIN_COMPATIBLE_PROTOCOL)).toEqual({
expect(computeConnectedState(handshake("1.1", "1.1"), MIN_COMPATIBLE_PROTOCOL)).toEqual({
kind: "connected",
});
});
it("returns version_skew when daemon protocol minor is newer", () => {
expect(computeConnectedState(handshake("1.1", "1.0"))).toEqual({
expect(computeConnectedState(handshake("1.2", "1.1"))).toEqual({
kind: "version_skew",
});
});
it("returns version_skew when daemon protocol string differs but floor is satisfied", () => {
expect(computeConnectedState(handshake("1", "1.0"))).toEqual({
expect(computeConnectedState(handshake("1.1.0", "1.1"))).toEqual({
kind: "version_skew",
});
});
@@ -48,16 +48,19 @@ describe("computeConnectedState (protocol-based compat)", () => {
const result = computeConnectedState(handshake("2.0", "1.0"));
expect(result.kind).toBe("rejected");
if (result.kind === "rejected") {
expect(result.reason).toContain("protocol-major mismatch");
expect(result.reason).toContain("bsk CLI");
expect(result.reason).toContain("BrowserSkill extension");
expect(result.reason).not.toContain("protocol");
}
});
it("rejects when extension is below daemon min_compatible_protocol", () => {
const result = computeConnectedState(handshake("1.0", "1.5"));
const result = computeConnectedState(handshake("1.1", "1.5"));
expect(result.kind).toBe("rejected");
if (result.kind === "rejected") {
expect(result.reason).toContain("min_compatible_protocol");
expect(result.reason).toContain("1.5");
expect(result.reason).toContain("BrowserSkill extension is out of date");
expect(result.reason).toContain("Update and reload the extension");
expect(result.reason).not.toContain("min_compatible_protocol");
}
});
@@ -65,7 +68,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
const result = computeConnectedState({
server: "browser-skill-daemon",
version: "0.1.0",
protocol_version: "1.0",
protocol_version: "1.1",
min_compatible_peer: "0.1.0",
});
expect(result).toEqual({ kind: "connected" });
@@ -75,16 +78,23 @@ describe("computeConnectedState (protocol-based compat)", () => {
const result = computeConnectedState(handshake("1.0", "1.0"), "1.1");
expect(result.kind).toBe("rejected");
if (result.kind === "rejected") {
expect(result.reason).toContain("below extension min_compatible_protocol");
expect(result.reason).toContain("bsk CLI is out of date");
expect(result.reason).toContain("bsk daemon restart");
expect(result.reason).not.toContain("min_compatible_protocol");
}
});
it("rejects a pre-Trace-v3 daemon using the configured extension floor", () => {
const result = computeConnectedState(handshake("1.0", "1.0"));
expect(result.kind).toBe("rejected");
});
it("rejects malformed daemon min_compatible_protocol with a daemon-floor reason", () => {
const result = computeConnectedState(handshake("1.0", "not-a-protocol"));
const result = computeConnectedState(handshake("1.1", "not-a-protocol"));
expect(result.kind).toBe("rejected");
if (result.kind === "rejected") {
expect(result.reason).toContain("daemon min_compatible_protocol");
expect(result.reason).toContain("not-a-protocol");
expect(result.reason).toContain("could not verify version compatibility");
expect(result.reason).not.toContain("min_compatible_protocol");
}
});
@@ -242,11 +252,11 @@ describe("ConnectionController connectionEnabled", () => {
const second = transport.send.mock.calls[1]?.[0] as { id: string };
expect(second.id).not.toBe(first.id);
transport.emitMessage({ id: first.id, result: handshake("1.0", "1.0") });
transport.emitMessage({ id: first.id, result: handshake("1.1", "1.1") });
await Promise.resolve();
expect(controller.snapshot().state).not.toBe("connected");
transport.emitMessage({ id: second.id, result: handshake("1.0", "1.0") });
transport.emitMessage({ id: second.id, result: handshake("1.1", "1.1") });
await vi.waitFor(() => expect(controller.snapshot().state).toBe("connected"));
});
});
@@ -0,0 +1,172 @@
import type { RenderedRef } from "@browser-skill/vom";
import { describe, expect, it } from "vitest";
import type { CapturedNode } from "@/tools/vom/capture";
import { matchTarget } from "../match-target";
function node(backendNodeId: number, overrides: Partial<CapturedNode> = {}): CapturedNode {
return {
backendNodeId,
parentBackendNodeId: null,
tag: "button",
attrs: {},
rect: { x: 10, y: 20, w: 100, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
...overrides,
};
}
describe("matchTarget", () => {
const refs: RenderedRef[] = [
{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 3 },
];
it("matches a unique node by viewport coordinates", () => {
const target = matchTarget({
geometry: {
rect: { x: 10, y: 20, w: 100, h: 30 },
scrollX: 0,
scrollY: 0,
position: "static",
tag: "button",
},
captured: [node(42)],
refs,
fallback: { tag: "button", name: "发布" },
});
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
});
it("matches a non-fixed node after the top frame scrolls", () => {
const target = matchTarget({
geometry: {
rect: { x: 10, y: 20, w: 100, h: 30 },
scrollX: 40,
scrollY: 300,
position: "static",
tag: "button",
},
captured: [
node(42, {
documentRect: { x: 50, y: 320, w: 100, h: 30 },
localRect: { x: 10, y: 20, w: 100, h: 30 },
rect: { x: 10, y: 20, w: 100, h: 30 },
}),
],
refs,
fallback: { tag: "button", name: "发布" },
});
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
});
it("matches a static node by document coordinates when scroll changed after observation", () => {
const target = matchTarget({
geometry: {
rect: { x: 10, y: 50, w: 100, h: 30 },
scrollX: 0,
scrollY: 250,
position: "static",
tag: "button",
},
captured: [
node(42, {
documentRect: { x: 10, y: 300, w: 100, h: 30 },
localRect: { x: 10, y: 200, w: 100, h: 30 },
rect: { x: 10, y: 200, w: 100, h: 30 },
}),
],
refs,
fallback: { tag: "button", name: "发布" },
});
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
});
it("returns unmatched when multiple candidates match", () => {
const target = matchTarget({
geometry: {
rect: { x: 10, y: 20, w: 100, h: 30 },
scrollX: 0,
scrollY: 0,
position: "static",
tag: "button",
},
captured: [node(42), node(43)],
refs,
});
expect(target.unmatched).toBe(true);
});
it("uses viewport coordinates for fixed elements", () => {
const target = matchTarget({
geometry: {
rect: { x: 5, y: 5, w: 80, h: 24 },
scrollX: 100,
scrollY: 200,
position: "fixed",
tag: "button",
},
captured: [node(42, { position: "fixed", rect: { x: 5, y: 5, w: 80, h: 24 } })],
refs,
});
expect(target.ref).toBe("e1");
});
it("isolates geometry matching to the owner iframe backend node", () => {
const sharedRect = { x: 10, y: 20, w: 100, h: 30 };
const iframeRefs: RenderedRef[] = [
{ ref: "e1", backendNodeId: 42, role: "button", name: "Top", line: 1 },
{ ref: "e2", backendNodeId: 99, role: "button", name: "Iframe", line: 2 },
];
const target = matchTarget({
geometry: {
rect: sharedRect,
scrollX: 0,
scrollY: 0,
position: "static",
tag: "button",
ownerFrameBackendNodeId: 13,
},
captured: [
node(42, { rect: sharedRect, documentRect: sharedRect }),
node(99, {
rect: sharedRect,
documentRect: sharedRect,
ownerFrameBackendNodeId: 13,
}),
],
refs: iframeRefs,
fallback: { tag: "button", name: "Iframe" },
});
expect(target).toEqual({ ref: "e2", role: "button", name: "Iframe" });
});
it("matches iframe static elements using frame scroll and document coordinates", () => {
const iframeRefs: RenderedRef[] = [
{ ref: "e2", backendNodeId: 99, role: "button", name: "Iframe", line: 2 },
];
const target = matchTarget({
geometry: {
rect: { x: 10, y: 50, w: 100, h: 30 },
scrollX: 0,
scrollY: 120,
position: "static",
tag: "button",
ownerFrameBackendNodeId: 13,
},
captured: [
node(99, {
ownerFrameBackendNodeId: 13,
documentRect: { x: 10, y: 170, w: 100, h: 30 },
localRect: { x: 10, y: 50, w: 100, h: 30 },
rect: { x: 10, y: 50, w: 100, h: 30 },
}),
],
refs: iframeRefs,
fallback: { tag: "button", name: "Iframe" },
});
expect(target).toEqual({ ref: "e2", role: "button", name: "Iframe" });
});
});
@@ -0,0 +1,116 @@
import { beforeEach, describe, expect, it } from "vitest";
import type { CapturedNode } from "@/tools/vom/capture";
import type { DraftTraceStep } from "@/transport/types";
import { resetStateIdCounterForTests } from "../record-constants";
import {
applyTargetMatching,
buildTraceV3,
createObservationState,
rememberStepOnPage,
} from "../record-observation";
import { registerObservation } from "../trace-reducer";
const URL = "https://example.com/login";
function capturedInput(): CapturedNode {
return {
backendNodeId: 42,
parentBackendNodeId: null,
tag: "input",
attrs: {},
rect: { x: 20, y: 40, w: 200, h: 30 },
localRect: { x: 20, y: 40, w: 200, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
};
}
function finalizedFillBody(value: string, redactValues: boolean): string {
const obs = createObservationState({ redactValues });
const rawVomText = '@vom 1\ntextbox "Password" value="••••••" [ref=e1]';
const stateId = registerObservation(obs.stateRegistry, { url: URL, rawVomText });
obs.lastSettled = {
stateId,
captured: [capturedInput()],
matchNodes: [capturedInput()],
documentTokenOwners: new Map(),
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
url: URL,
vomText: rawVomText,
};
const draft: DraftTraceStep = {
op: "fill",
target: { unmatched: true },
value,
geometry: {
rect: { x: 20, y: 40, w: 200, h: 30 },
scrollX: 0,
scrollY: 0,
position: "static",
tag: "input",
},
};
applyTargetMatching(obs, draft, 1);
draft.postStateId = stateId;
rememberStepOnPage(obs, draft, 1);
return (
buildTraceV3({
obs,
steps: [draft],
startedAt: "2026-08-12T00:00:00.000Z",
stoppedBy: "user_finish",
bskVersion: "test",
}).states[0]?.body ?? ""
);
}
describe("record observation annotations", () => {
beforeEach(() => {
resetStateIdCounterForTests();
});
it("omits a fill literal from finalized state bodies when values are redacted", () => {
const secret = "hunter2-private";
const body = finalizedFillBody(secret, true);
expect(body).toContain("step 1: fill");
expect(body).not.toContain(secret);
});
it("keeps fill details in ordinary recording annotations", () => {
const value = "ordinary text";
const body = finalizedFillBody(value, false);
expect(body).toContain(`step 1: fill: ${JSON.stringify(value)}`);
});
});
describe("applyTargetMatching without a settled observation", () => {
it("keeps the capture role/name instead of a bare unmatched target", () => {
const obs = createObservationState();
const draft: DraftTraceStep = {
op: "hover",
target: { unmatched: true },
captureTarget: { tag: "button", role: "button", name: "新建" },
geometry: {
rect: { x: 900, y: 8, w: 60, h: 32 },
scrollX: 0,
scrollY: 0,
position: "static",
tag: "button",
},
};
applyTargetMatching(obs, draft, 1);
expect(draft.preStateId).toBeUndefined();
expect(draft.target).toEqual({
role: "button",
name: "新建",
unmatched: true,
});
});
});
@@ -12,12 +12,42 @@ describe("recording-step-buffer", () => {
expect(buffer.steps).toEqual([
{
op: "click",
target: { tag: "button", role: "button", name: "发布" },
target: { unmatched: true },
captureTarget: { tag: "button", role: "button", name: "发布" },
},
]);
expect(buffer.pendingNavigation).toBe(true);
});
it("keeps the hovered element description as a capture fallback", () => {
const buffer = { steps: [], pendingNavigation: false };
appendRecordedPayload(buffer, {
op: "hover",
target: { tag: "button", role: "button", name: "新建" },
geometry: {
rect: { x: 900, y: 8, w: 60, h: 32 },
scrollX: 0,
scrollY: 0,
position: "static",
tag: "button",
},
});
expect(buffer.steps).toEqual([
{
op: "hover",
target: { unmatched: true },
captureTarget: { tag: "button", role: "button", name: "新建" },
geometry: {
rect: { x: 900, y: 8, w: 60, h: 32 },
scrollX: 0,
scrollY: 0,
position: "static",
tag: "button",
},
},
]);
});
it("annotates navigated_to on action-caused navigation instead of wait_for_navigation", () => {
const buffer = {
steps: [
@@ -71,10 +101,41 @@ describe("recording-step-buffer", () => {
currentUrl: "https://example.com/a",
pendingNavigation: false,
};
observeRecordedNavigation(buffer, "https://example.com/b", false);
const result = observeRecordedNavigation(buffer, "https://example.com/b", false);
expect(result).toEqual({ kind: "appended", index: 0 });
expect(buffer.steps[0]).toMatchObject({
op: "navigate",
url: "https://example.com/b",
});
});
it("asks the recorder to coalesce redirect hops instead of emitting each one", () => {
const buffer = {
steps: [],
currentUrl: "https://passport.example/login",
pendingNavigation: false,
};
const hop1 = observeRecordedNavigation(
buffer,
"https://passport.example/callback",
false,
"link",
["server_redirect"],
);
expect(hop1).toEqual({
kind: "coalesce_redirect",
url: "https://passport.example/callback",
});
expect(buffer.steps).toEqual([]);
const hop2 = observeRecordedNavigation(buffer, "https://app.example/dashboard", false, "link", [
"client_redirect",
]);
expect(hop2).toEqual({
kind: "coalesce_redirect",
url: "https://app.example/dashboard",
});
expect(buffer.steps).toEqual([]);
expect(buffer.currentUrl).toBe("https://app.example/dashboard");
});
});
@@ -1,6 +1,11 @@
import { describe, expect, it } from "vitest";
import type { DraftTraceStep } from "@/transport/types";
import { reduceTraceSteps, resolveTraceStartUrl, shouldRecordPress } from "../trace-reducer";
import { resetStateIdCounterForTests } from "@/lib/record-constants";
import {
reduceTraceSteps,
registerObservation,
resolveTraceStartUrl,
shouldRecordPress,
} from "../trace-reducer";
describe("shouldRecordPress", () => {
it("keeps Enter and Escape", () => {
@@ -17,86 +22,85 @@ describe("shouldRecordPress", () => {
});
});
describe("reduceTraceSteps", () => {
it("builds steps with pages dictionary and page id references", () => {
const drafts: DraftTraceStep[] = [
{
op: "navigate",
url: "https://example.com/search?q=hello&utm_source=x",
page_url: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "fill",
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
value: "browser skill",
page_url: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "press",
key: "Enter",
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
navigated_to: "https://example.com/results/42",
page_url: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "click",
target: { tag: "button", role: "button", name: "发布" },
navigated_to: "https://example.com/p/99",
page_url: "https://example.com/results/42",
},
{
op: "press",
key: "a",
page_url: "https://example.com/p/99",
},
];
const { pages, steps } = reduceTraceSteps(
drafts,
"https://example.com/search?q=hello&utm_source=x",
);
expect(JSON.stringify(steps)).not.toContain("parameters");
expect(JSON.stringify(steps)).not.toContain("intent");
expect(JSON.stringify(steps)).not.toContain("summary");
expect(steps.map((s) => s.op)).toEqual(["navigate", "fill", "press", "click"]);
expect(pages.map((p) => p.url)).toEqual([
"https://example.com/search?q=hello&utm_source=x",
"https://example.com/results/42",
"https://example.com/p/99",
]);
expect(steps[0]).toMatchObject({
id: 1,
op: "navigate",
page: "p1",
to: "https://example.com/search?q=hello&utm_source=x",
describe("reduceTraceSteps v3", () => {
it("builds steps with state/result references and deduped states", () => {
resetStateIdCounterForTests();
const registry = new Map();
const body = "@vom 1\nL1 page";
const s1 = registerObservation(registry, {
url: "https://example.com/search",
rawVomText: body,
});
const s2 = registerObservation(registry, {
url: "https://example.com/results/42",
rawVomText: "@vom 1\nL1 results",
});
const { states, steps } = reduceTraceSteps(
[
{
op: "navigate",
url: "https://example.com/search",
preStateId: s1,
postStateId: s1,
},
{
op: "fill",
target: { ref: "e1", role: "textbox", name: "搜索" },
value: "browser skill",
commit: "enter",
preStateId: s1,
postStateId: s1,
},
{
op: "press",
key: "Enter",
target: { ref: "e1", role: "textbox", name: "搜索" },
preStateId: s1,
postStateId: s2,
navigated_to: "https://example.com/results/42",
},
{
op: "click",
target: { ref: "e2", role: "button", name: "发布" },
preStateId: s2,
postStateId: s2,
},
],
registry,
);
expect(states).toHaveLength(2);
expect(steps.map((s) => s.op)).toEqual(["navigate", "fill", "press", "click"]);
expect(steps[1]).toMatchObject({
op: "fill",
page: "p1",
value: "browser skill",
state: s1,
result: { state: s1 },
commit: "enter",
});
expect(steps[2]).toMatchObject({
op: "press",
key: "Enter",
page: "p1",
effect: { navigated_to: "p2" },
});
expect(steps[3]).toMatchObject({
op: "click",
page: "p2",
effect: { navigated_to: "p3" },
state: s1,
result: { state: s2 },
});
expect(JSON.stringify(steps)).not.toContain("page");
expect(JSON.stringify(steps)).not.toContain("effect");
});
it("collapses consecutive navigations", () => {
const { steps } = reduceTraceSteps([
{ op: "navigate", url: "https://a.example/redirect1" },
{ op: "navigate", url: "https://a.example/final" },
]);
resetStateIdCounterForTests();
const registry = new Map();
const s1 = registerObservation(registry, {
url: "https://a.example/redirect1",
rawVomText: "a",
});
const { steps } = reduceTraceSteps(
[
{ op: "navigate", url: "https://a.example/redirect1", preStateId: s1, postStateId: s1 },
{ op: "navigate", url: "https://a.example/final", preStateId: s1, postStateId: s1 },
],
registry,
);
expect(steps).toHaveLength(1);
expect(steps[0]).toMatchObject({
op: "navigate",
@@ -104,52 +108,124 @@ describe("reduceTraceSteps", () => {
});
});
it("maps select navigated_to onto effect.navigated_to (page id)", () => {
const { pages, steps } = reduceTraceSteps(
it("keeps the origin state and cause of the hop that started a redirect chain", () => {
resetStateIdCounterForTests();
const registry = new Map();
const start = registerObservation(registry, {
url: "https://example.com/",
rawVomText: "start",
});
const hop = registerObservation(registry, {
url: "https://iwiki.woa.com/",
rawVomText: "hop",
});
const landing = registerObservation(registry, {
url: "https://iwiki.woa.com/dashboard",
rawVomText: "landing",
});
const { steps, stepIdByDraftId } = reduceTraceSteps(
[
{
op: "select",
target: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
labels: ["技术"],
navigated_to: "https://example.com/list?cat=tech",
page_url: "https://example.com/list",
op: "navigate",
url: "https://iwiki.woa.com/",
preStateId: start,
postStateId: hop,
transitionType: "typed",
},
{
op: "navigate",
url: "https://iwiki.woa.com/dashboard",
preStateId: hop,
postStateId: landing,
transitionType: "link",
},
],
"https://example.com/list",
registry,
);
expect(pages.map((p) => p.url)).toEqual([
"https://example.com/list",
"https://example.com/list?cat=tech",
]);
expect(steps).toHaveLength(1);
expect(steps[0]).toMatchObject({
op: "select",
page: "p1",
selection: [{ value: "tech", label: "技术" }],
effect: { navigated_to: "p2" },
op: "navigate",
id: 1,
state: start,
result: { state: landing },
to: "https://iwiki.woa.com/dashboard",
cause: "user_typed",
});
// Both drafts fold into step 1, so anything tagged with either draft id
// must resolve to it.
expect(stepIdByDraftId.get(1)).toBe(1);
expect(stepIdByDraftId.get(2)).toBe(1);
});
it("maps draft ids to published step ids across dropped drafts", () => {
resetStateIdCounterForTests();
const registry = new Map();
const s1 = registerObservation(registry, { url: "https://a.example/", rawVomText: "a" });
const { steps, stepIdByDraftId } = reduceTraceSteps(
[
// Dropped: bare typing press.
{ op: "press", key: "a", preStateId: s1, postStateId: s1 },
{ op: "click", target: { ref: "e2" }, preStateId: s1, postStateId: s1 },
],
registry,
);
expect(steps.map((step) => step.op)).toEqual(["click"]);
expect(stepIdByDraftId.get(1)).toBeUndefined();
expect(stepIdByDraftId.get(2)).toBe(1);
});
it("keeps committed empty fills", () => {
resetStateIdCounterForTests();
const registry = new Map();
const s1 = registerObservation(registry, { url: "https://a.example/", rawVomText: "a" });
const { steps } = reduceTraceSteps(
[
{ op: "fill", target: { ref: "e1", role: "textbox", name: "Search" }, value: "", preStateId: s1, postStateId: s1 },
{ op: "click", target: { ref: "e2", role: "button", name: "Apply" }, preStateId: s1, postStateId: s1 },
],
registry,
);
expect(steps.map((step) => step.op)).toEqual(["fill", "click"]);
expect(steps[0]).toMatchObject({ op: "fill", value: "" });
});
it("keeps hover steps before menu clicks", () => {
resetStateIdCounterForTests();
const registry = new Map();
const state = registerObservation(registry, {
url: "https://example.com/app",
rawVomText: '@e1 button "Account"\n@e2 link "Profile"',
});
const { steps } = reduceTraceSteps(
[
{
op: "hover",
target: { tag: "span", role: "button", name: "Account" },
target: { ref: "e1", role: "button", name: "Account" },
page_url: "https://example.com/app",
preStateId: state,
postStateId: state,
},
{
op: "click",
target: { tag: "a", role: "link", name: "Profile" },
target: { ref: "e2", role: "link", name: "Profile" },
page_url: "https://example.com/app",
preStateId: state,
postStateId: state,
},
],
"https://example.com/app",
registry,
);
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
expect(steps[0]).toMatchObject({
op: "hover",
page: "p1",
state,
result: { state },
target: { name: "Account" },
});
});
@@ -157,7 +233,7 @@ describe("reduceTraceSteps", () => {
it("resolveTraceStartUrl prefers explicit start URL", () => {
expect(
resolveTraceStartUrl(
[{ op: "navigate", url: "https://example.com/other" }],
[{ op: "navigate", url: "https://example.com/other", preStateId: "s1", postStateId: "s1" }],
"https://example.com/start",
),
).toBe("https://example.com/start");
@@ -313,6 +313,15 @@ export type HandshakeVerdict =
| { kind: "version_skew" }
| { kind: "rejected"; reason: string };
const UPDATE_BOTH_VERSION_MESSAGE =
"The installed bsk CLI and BrowserSkill extension are incompatible. Update both, reload the extension, then run `bsk daemon restart`.";
const UPDATE_BSK_VERSION_MESSAGE =
"The installed bsk CLI is out of date for this BrowserSkill extension. Update bsk, then run `bsk daemon restart`.";
const UPDATE_EXTENSION_VERSION_MESSAGE =
"This BrowserSkill extension is out of date for the installed bsk CLI. Update and reload the extension, then try again.";
const INVALID_VERSION_MESSAGE =
"BrowserSkill could not verify version compatibility. Update bsk and the BrowserSkill extension, reload the extension, then run `bsk daemon restart`.";
/**
* Symmetric extension-side **protocol** compat check.
*
@@ -339,19 +348,19 @@ export function computeConnectedState(
if (daemonMajor === null || ourMajor === null || daemonMajor !== ourMajor) {
return {
kind: "rejected",
reason: `protocol-major mismatch (daemon protocol v${handshake.protocol_version}, extension protocol v${PROTOCOL_VERSION})`,
reason: UPDATE_BOTH_VERSION_MESSAGE,
};
}
if (compareProtocol(handshake.protocol_version, localMinCompatibleProtocol) === null) {
return {
kind: "rejected",
reason: `daemon protocol v${handshake.protocol_version} is unparseable`,
reason: INVALID_VERSION_MESSAGE,
};
}
if (compareProtocol(handshake.protocol_version, localMinCompatibleProtocol)! < 0) {
return {
kind: "rejected",
reason: `daemon protocol v${handshake.protocol_version} below extension min_compatible_protocol ${localMinCompatibleProtocol}`,
reason: UPDATE_BSK_VERSION_MESSAGE,
};
}
const peerFloor = handshake.min_compatible_protocol;
@@ -359,13 +368,13 @@ export function computeConnectedState(
if (compareProtocol(PROTOCOL_VERSION, peerFloor) === null) {
return {
kind: "rejected",
reason: `daemon min_compatible_protocol ${peerFloor} is unparseable`,
reason: INVALID_VERSION_MESSAGE,
};
}
if (compareProtocol(PROTOCOL_VERSION, peerFloor)! < 0) {
return {
kind: "rejected",
reason: `extension protocol v${PROTOCOL_VERSION} below daemon min_compatible_protocol ${peerFloor}`,
reason: UPDATE_EXTENSION_VERSION_MESSAGE,
};
}
}
+5 -4
View File
@@ -6,7 +6,8 @@
* `{ "tag": "div" }` / “点击div” is useless and must not be recorded.
*/
export interface TargetDescriptor {
/** Content-script capture descriptor before VOM geometric matching. */
export interface CaptureTargetDescriptor {
role?: string;
name?: string;
tag: string;
@@ -247,7 +248,7 @@ export function resolveHoverElement(target: Element): Element | null {
* (visible name), or at least a form `name_attr` for checkbox/radio.
* Recording “点击div” with no name fails this bar.
*/
export function isMeaningfulClickTarget(target: TargetDescriptor): boolean {
export function isMeaningfulClickTarget(target: CaptureTargetDescriptor): boolean {
const name = target.name?.trim();
if (name && isActionableLabel(name)) return true;
if (
@@ -323,7 +324,7 @@ function nearbyLabelText(el: Element): string | undefined {
return undefined;
}
export function describeTarget(el: Element): TargetDescriptor {
export function describeTarget(el: Element): CaptureTargetDescriptor {
const tag = el.tagName.toLowerCase();
const role = inferRole(el);
const name = accessibleName(el);
@@ -350,7 +351,7 @@ export function describeTarget(el: Element): TargetDescriptor {
};
}
export function describeEventTarget(target: EventTarget | null): TargetDescriptor | null {
export function describeEventTarget(target: EventTarget | null): CaptureTargetDescriptor | null {
if (!(target instanceof Element)) return null;
const clickable = resolveClickableElement(target);
if (!clickable) return null;
@@ -0,0 +1,62 @@
import type { Step } from "@/transport/types";
import { OBSERVATION_FILE_VERSION } from "./record-constants";
export interface ObservationAnnotation {
stepId: number;
op: Step["op"];
line: number;
stateId: string;
detail?: string;
}
function formatAnnotation({ stepId, op, detail }: ObservationAnnotation): string {
const suffix = detail ? `: ${detail}` : "";
return ` ⟵ step ${stepId}: ${op}${suffix}`;
}
/** Serialize a page observation file (front matter + VOM body + step annotations). */
export function formatObservationFile(input: {
stateId: string;
url: string;
title?: string;
stepsHere: number[];
body: string;
annotations?: ObservationAnnotation[];
}): string {
const lines: string[] = [
`# bsk-observation ${OBSERVATION_FILE_VERSION}`,
`state: ${input.stateId}`,
`url: ${input.url}`,
];
if (input.title) lines.push(`title: ${input.title}`);
if (input.stepsHere.length > 0) {
lines.push(`steps_here: [${input.stepsHere.join(", ")}]`);
}
lines.push("---");
const bodyLines = input.body.split("\n");
const annotationMap = new Map<number, ObservationAnnotation[]>();
for (const ann of input.annotations ?? []) {
const bucket = annotationMap.get(ann.line) ?? [];
bucket.push(ann);
annotationMap.set(ann.line, bucket);
}
for (let i = 0; i < bodyLines.length; i += 1) {
let line = bodyLines[i] ?? "";
const anns = annotationMap.get(i);
if (anns) {
for (const ann of anns) {
line += formatAnnotation(ann);
}
}
lines.push(line);
}
return `${lines.join("\n")}\n`;
}
/** Hash input: VOM body **before** annotations are inserted. */
export function observationBodyForHash(vomText: string): string {
return vomText;
}
+102
View File
@@ -0,0 +1,102 @@
import type { RenderedRef } from "@browser-skill/vom";
import type { CapturedNode } from "@/tools/vom/capture";
import type { CaptureGeometry, TargetDescriptor } from "@/transport/types";
import type { CaptureTargetDescriptor } from "./describe-target";
import { GEOM_MATCH_TOLERANCE_PX } from "./record-constants";
export interface MatchTargetInput {
geometry: CaptureGeometry;
captured: CapturedNode[];
refs: RenderedRef[];
fallback?: CaptureTargetDescriptor;
}
function withinTolerance(a: number, b: number): boolean {
return Math.abs(a - b) <= GEOM_MATCH_TOLERANCE_PX;
}
function rectsMatch(
a: { x: number; y: number; w: number; h: number },
b: { x: number; y: number; w: number; h: number },
): boolean {
return (
withinTolerance(a.x, b.x) &&
withinTolerance(a.y, b.y) &&
withinTolerance(a.w, b.w) &&
withinTolerance(a.h, b.h)
);
}
function nodeViewportRect(
node: CapturedNode,
): { x: number; y: number; w: number; h: number } | null {
return node.localRect ?? node.rect;
}
function matchingRects(
geometry: CaptureGeometry,
node: CapturedNode,
): {
target: { x: number; y: number; w: number; h: number };
node: { x: number; y: number; w: number; h: number } | null;
} {
const viewportPosition = geometry.position === "fixed" || geometry.position === "sticky";
if (viewportPosition || !node.documentRect) {
return { target: geometry.rect, node: nodeViewportRect(node) };
}
return {
target: {
x: geometry.rect.x + geometry.scrollX,
y: geometry.rect.y + geometry.scrollY,
w: geometry.rect.w,
h: geometry.rect.h,
},
node: node.documentRect,
};
}
function tagMatches(geometryTag: string, nodeTag: string): boolean {
return geometryTag.toLowerCase() === nodeTag.toLowerCase();
}
/** Best description available when the element cannot be located in the VOM. */
export function fallbackDescriptor(fallback?: CaptureTargetDescriptor): TargetDescriptor {
if (!fallback) {
return { unmatched: true };
}
return {
...(fallback.role ? { role: fallback.role } : {}),
...(fallback.name ? { name: fallback.name } : {}),
unmatched: true,
};
}
/** Locate the interacted element in the last settled observation by geometry. */
export function matchTarget(input: MatchTargetInput): TargetDescriptor {
const ownerFrame = input.geometry.ownerFrameBackendNodeId ?? null;
const candidates = input.captured.filter((node) => {
if (ownerFrame !== (node.ownerFrameBackendNodeId ?? null)) return false;
if (!tagMatches(input.geometry.tag, node.tag)) return false;
const rects = matchingRects(input.geometry, node);
if (!rects.node) return false;
return rectsMatch(rects.target, rects.node);
});
if (candidates.length !== 1) {
return fallbackDescriptor(input.fallback);
}
const backendNodeId = candidates[0]!.backendNodeId;
const refEntry = input.refs.find((r) => r.backendNodeId === backendNodeId);
if (!refEntry) {
return fallbackDescriptor(input.fallback);
}
return {
ref: refEntry.ref,
...(refEntry.role ? { role: refEntry.role } : {}),
...(refEntry.name ? { name: refEntry.name } : {}),
...(refEntry.ctx ? { ctx: refEntry.ctx } : {}),
};
}
+92
View File
@@ -0,0 +1,92 @@
// Deciding *when* a page is worth observing.
//
// A fixed delay cannot serve both a modal that toggles in one frame and a
// route change that renders for a second. Instead of guessing, ask the page:
// a document that has stopped mutating and is no longer loading is done
// reacting to whatever the user just did.
import type { CdpRunner } from "@/tools/shared";
import { SETTLE_MAX_MS, SETTLE_MIN_MS, SETTLE_POLL_MS, SETTLE_QUIET_MS } from "./record-constants";
/**
* Installed once per document. Records only the timestamp of the last DOM
* change so each poll stays O(1) no matter how large the page is.
*/
const QUIET_PROBE = `(() => {
const scope = window;
let probe = scope.__bskRecordQuiet;
if (!probe) {
probe = { changedAt: Date.now() };
const observer = new MutationObserver(() => {
probe.changedAt = Date.now();
});
observer.observe(document.documentElement, {
subtree: true,
childList: true,
attributes: true,
characterData: true,
});
scope.__bskRecordQuiet = probe;
}
return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState };
})()`;
export type SettleOutcome =
/** The page stopped changing on its own. */
| "quiet"
/** Still changing when the budget ran out; observe it as it is. */
| "timeout"
/** A newer action took over; this observation is no longer wanted. */
| "cancelled";
interface QuietProbe {
idleMs: number;
readyState: string;
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function readQuietProbe(cdp: CdpRunner, tabId: number): Promise<QuietProbe | null> {
try {
const reply = await cdp.send<{ result?: { value?: unknown } }>(tabId, "Runtime.evaluate", {
expression: QUIET_PROBE,
returnByValue: true,
});
const value = reply.result?.value;
if (!value || typeof value !== "object") return null;
const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown };
if (typeof idleMs !== "number" || typeof readyState !== "string") return null;
return { idleMs, readyState };
} catch {
// An unreadable page is a page mid-swap; that is a reason to keep waiting,
// not a reason to give up.
return null;
}
}
/** Wait until the page has finished reacting, or until the budget runs out. */
export async function waitForPageSettled(
cdp: CdpRunner,
tabId: number,
options: { cancelled?: () => boolean } = {},
): Promise<SettleOutcome> {
const startedAt = Date.now();
const floor = startedAt + SETTLE_MIN_MS;
const deadline = startedAt + SETTLE_MAX_MS;
for (;;) {
if (options.cancelled?.()) return "cancelled";
await sleep(SETTLE_POLL_MS);
if (options.cancelled?.()) return "cancelled";
const probe = await readQuietProbe(cdp, tabId);
const now = Date.now();
if (now < floor) continue;
if (now >= deadline) return "timeout";
if (!probe) continue;
if (probe.readyState === "loading") continue;
if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet";
}
}
+49 -4
View File
@@ -3,7 +3,8 @@
* service worker and a tab's content script.
*/
import type { TargetDescriptor } from "./describe-target";
import type { CaptureTargetDescriptor } from "./describe-target";
import type { FrameCaptureFailure, FrameCaptureStatus } from "@/transport/types";
export const RECORD_START = "bsk-record-start";
export const RECORD_STEP = "bsk-record-step";
@@ -11,6 +12,7 @@ export const RECORD_STOP = "bsk-record-stop";
export const RECORD_CANCEL = "bsk-record-cancel";
export const RECORD_FINISH = "bsk-record-finish";
export const RECORD_QUERY = "bsk-record-query";
export const RECORD_CAPTURE_STATUS = "bsk-record-capture-status";
export interface RecordStartAck {
ok: true;
@@ -27,8 +29,12 @@ export interface RecordQueryResponse {
requestId?: string;
/** Epoch ms when the recording began; see `RecordStartMessage.startedAtMs`. */
startedAtMs?: number;
captureStatus?: FrameCaptureStatus;
captureFailures?: FrameCaptureFailure[];
}
export type RecordFrameMode = "top" | "child";
export interface RecordStartMessage {
type: typeof RECORD_START;
requestId: string;
@@ -38,11 +44,15 @@ export interface RecordStartMessage {
* content-script remounts instead of restarting per page.
*/
startedAtMs?: number;
/** When true, the top-frame overlay should mount. Child frames omit this. */
showOverlay?: boolean;
/** Whether this document runs top-frame or child-frame capture semantics. */
frameMode?: RecordFrameMode;
}
export interface RecordStepPayload {
op: "click" | "hover" | "fill" | "press" | "select" | "navigate";
target?: TargetDescriptor;
op: "click" | "hover" | "fill" | "press" | "select" | "navigate" | "scroll";
target?: CaptureTargetDescriptor;
value?: string;
key?: string;
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">;
@@ -50,12 +60,31 @@ export interface RecordStepPayload {
labels?: string[];
url?: string;
redacted?: boolean;
/** Page URL when the step was captured. */
commit?: "enter" | "suggestion" | "blur";
/** Top-level tab URL when the step was captured. */
page_url?: string;
/** URL of the frame document where the interaction occurred. */
frame_url?: string;
/** Ephemeral token stamped on the active document root; not persisted in trace steps. */
documentToken?: string;
/** Epoch ms when the step was captured in the content script. */
capturedAtMs?: number;
/** Document geometry at capture time for background geometric matching. */
geometry?: {
rect: { x: number; y: number; w: number; h: number };
scrollX: number;
scrollY: number;
position: string;
tag: string;
ownerFrameBackendNodeId?: number | null;
};
/** Capture-only hint; never persisted unless converted to navigated_to. */
expects_navigation?: boolean;
/** Whether an observed URL change was synchronously caused by the action. */
navigation_caused_by_action?: boolean;
/** Raw webNavigation transition metadata for cause mapping. */
transitionType?: string;
transitionQualifiers?: string[];
}
export interface RecordStepMessage {
@@ -79,6 +108,12 @@ export interface RecordFinishMessage {
requestId: string;
}
export interface RecordCaptureStatusMessage {
type: typeof RECORD_CAPTURE_STATUS;
status: FrameCaptureStatus;
failures?: FrameCaptureFailure[];
}
export function isRecordStartMessage(msg: unknown): msg is RecordStartMessage {
if (typeof msg !== "object" || msg === null) return false;
const m = msg as Record<string, unknown>;
@@ -109,6 +144,16 @@ export function isRecordQueryMessage(msg: unknown): msg is RecordQueryMessage {
return m.type === RECORD_QUERY;
}
export function isRecordCaptureStatusMessage(msg: unknown): msg is RecordCaptureStatusMessage {
if (typeof msg !== "object" || msg === null) return false;
const m = msg as Record<string, unknown>;
return (
m.type === RECORD_CAPTURE_STATUS &&
(m.status === "complete" || m.status === "partial") &&
(m.failures === undefined || Array.isArray(m.failures))
);
}
export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage {
if (typeof msg !== "object" || msg === null) return false;
const m = msg as Record<string, unknown>;
@@ -0,0 +1,97 @@
import type { FillCommit, NavigationCause } from "@/transport/types";
/**
* Floor before a post-action observation may be taken. A page usually has not
* started reacting in the first frames after an action, and capturing then
* would record the page as it was, not as the action left it.
*/
export const SETTLE_MIN_MS = 150;
/** How long the DOM must stop changing before a page counts as settled. */
export const SETTLE_QUIET_MS = 250;
/** Upper bound on waiting for a page to settle; animations never stop. */
export const SETTLE_MAX_MS = 2_000;
/** How often to ask the page whether it has gone quiet. */
export const SETTLE_POLL_MS = 60;
/** Minimum interval between consecutive observations on the same tab. */
export const OBSERVATION_MIN_INTERVAL_MS = 200;
/** Delay before retrying a capture that lost its execution context. */
export const CAPTURE_RETRY_DELAY_MS = 250;
/** Document-coordinate matching tolerance in CSS pixels. */
export const GEOM_MATCH_TOLERANCE_PX = 2;
/** Default max tokens per page observation when CLI omits `--max-page-tokens`. */
export const DEFAULT_MAX_PAGE_TOKENS = 3000;
/** VOM observation file format version (front matter header). */
export const OBSERVATION_FILE_VERSION = 1;
/** FNV-1a 64-bit offset basis. */
const FNV_OFFSET = 0xcbf29ce484222325n;
/** FNV-1a 64-bit prime. */
const FNV_PRIME = 0x100000001b3n;
/** Content-hash for state deduplication (sync, non-cryptographic). */
export function fnv1a64(text: string): string {
let hash = FNV_OFFSET;
for (let i = 0; i < text.length; i += 1) {
hash ^= BigInt(text.charCodeAt(i));
hash = (hash * FNV_PRIME) & 0xffffffffffffffffn;
}
return hash.toString(16).padStart(16, "0");
}
let nextStateSerial = 0;
/** Monotonic state id generator (`s1`, `s2`, …). */
export function nextStateId(): string {
nextStateSerial += 1;
return `s${nextStateSerial}`;
}
/** Test seam: reset the state id counter. */
export function resetStateIdCounterForTests(): void {
nextStateSerial = 0;
}
const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
const TRANSITION_TO_CAUSE: Record<string, NavigationCause> = {
typed: "user_typed",
generated: "user_typed",
keyword: "user_typed",
keyword_generated: "user_typed",
link: "link",
form_submit: "form_submit",
reload: "reload",
auto_bookmark: "browser",
start_page: "browser",
};
export interface NavigationTransitionMeta {
transitionType?: string;
transitionQualifiers?: string[];
navigationActionPending?: boolean;
}
/** Map webNavigation metadata to protocol `NavigationCause`. Returns null for redirects. */
export function mapNavigationCause(meta: NavigationTransitionMeta): NavigationCause | null {
const qualifiers = meta.transitionQualifiers ?? [];
if (qualifiers.includes("forward_back")) return "history";
if (qualifiers.some((q) => REDIRECT_QUALIFIERS.has(q))) return null;
if (qualifiers.includes("from_address_bar")) return "user_typed";
const type = meta.transitionType ?? "";
const mapped = TRANSITION_TO_CAUSE[type];
if (mapped) return mapped;
if (!type && meta.navigationActionPending === false) return "script";
return "browser";
}
export const DEFAULT_FILL_COMMIT: FillCommit = "blur";
@@ -0,0 +1,24 @@
/** Attribute written on `<html>` while a document is actively recording. */
export const RECORD_DOCUMENT_TOKEN_ATTR = "data-bsk-record-document";
export function createRecordDocumentToken(): string {
if (typeof crypto !== "undefined" && typeof crypto.randomUUID === "function") {
return crypto.randomUUID();
}
return `doc-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
}
export function stampRecordDocumentToken(token: string): void {
document.documentElement.setAttribute(RECORD_DOCUMENT_TOKEN_ATTR, token);
}
export function clearRecordDocumentToken(): void {
document.documentElement.removeAttribute(RECORD_DOCUMENT_TOKEN_ATTR);
}
export function readRecordDocumentTokenFromAttrs(
attrs: Record<string, string> | undefined,
): string | undefined {
const token = attrs?.[RECORD_DOCUMENT_TOKEN_ATTR];
return token?.trim() ? token : undefined;
}
@@ -0,0 +1,250 @@
import {
RECORD_CANCEL,
RECORD_START,
RECORD_STOP,
type RecordCancelMessage,
type RecordStartAck,
type RecordStartMessage,
type RecordStopAck,
type RecordStopMessage,
} from "@/lib/record-bridge";
import type { FrameCaptureFailure, FrameCaptureFailureReason } from "@/transport/types";
export interface WebNavigationFrameInfo {
frameId: number;
documentId?: string;
url?: string;
parentFrameId: number;
}
export interface ArmedDocument {
tabId: number;
frameId: number;
documentId: string;
url?: string;
parentFrameId: number;
}
export interface RecordFrameSendDeps {
sendToDocument(
tabId: number,
documentId: string,
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
): Promise<unknown>;
getAllFrames(tabId: number): Promise<WebNavigationFrameInfo[]>;
}
const RECORD_START_RETRIES = 3;
const RECORD_START_RETRY_DELAY_MS = 500;
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
export function isFrameUrlRestricted(url: string | undefined): boolean {
if (!url) return false;
const lower = url.toLowerCase();
return (
lower.startsWith("chrome://") ||
lower.startsWith("chrome-extension://") ||
lower.startsWith("edge://") ||
lower.startsWith("devtools://") ||
lower.startsWith("devtools:") ||
lower.startsWith("https://chrome.google.com/webstore")
);
}
function isRecordStartAck(response: unknown): response is RecordStartAck {
return (
typeof response === "object" &&
response !== null &&
"ok" in response &&
(response as RecordStartAck).ok === true
);
}
function isRecordStopAckOk(response: unknown): boolean {
return (
typeof response === "object" &&
response !== null &&
"ok" in response &&
(response as RecordStopAck).ok === true
);
}
export function makeFrameCaptureFailure(
reason: FrameCaptureFailureReason,
input: { frameId: number; documentId?: string; url?: string; detail?: string },
): FrameCaptureFailure {
return {
reason,
frame_id: input.frameId,
...(input.documentId ? { document_id: input.documentId } : {}),
...(input.url ? { url: input.url } : {}),
...(input.detail ? { detail: input.detail } : {}),
};
}
export async function listInjectableFrames(
tabId: number,
deps: RecordFrameSendDeps,
): Promise<{ injectable: WebNavigationFrameInfo[]; failures: FrameCaptureFailure[] }> {
const failures: FrameCaptureFailure[] = [];
let frames: WebNavigationFrameInfo[] = [];
try {
frames = await deps.getAllFrames(tabId);
} catch (err) {
failures.push(
makeFrameCaptureFailure("arm_failed", {
frameId: 0,
detail: err instanceof Error ? err.message : "webNavigation.getAllFrames failed",
}),
);
return { injectable: [], failures };
}
const injectable = frames.filter((frame) => {
if (!frame.documentId) {
failures.push(
makeFrameCaptureFailure("not_injectable", {
frameId: frame.frameId,
url: frame.url,
detail: "missing documentId",
}),
);
return false;
}
if (isFrameUrlRestricted(frame.url)) {
failures.push(
makeFrameCaptureFailure("not_injectable", {
frameId: frame.frameId,
documentId: frame.documentId,
url: frame.url,
}),
);
return false;
}
return true;
});
return { injectable, failures };
}
export async function sendRecordStartWithAckToDocument(
tabId: number,
documentId: string,
msg: RecordStartMessage,
deps: RecordFrameSendDeps,
): Promise<void> {
let lastError: unknown;
for (let attempt = 0; attempt < RECORD_START_RETRIES; attempt += 1) {
try {
const response = await deps.sendToDocument(tabId, documentId, msg);
if (isRecordStartAck(response)) return;
lastError = new Error("content script did not ack RECORD_START");
} catch (err) {
lastError = err;
}
if (attempt + 1 < RECORD_START_RETRIES) {
await sleep(RECORD_START_RETRY_DELAY_MS);
}
}
throw lastError ?? new Error("failed to start recording in content script");
}
export async function armDocumentCapture(
tabId: number,
frame: WebNavigationFrameInfo,
startMsg: RecordStartMessage,
deps: RecordFrameSendDeps,
): Promise<{ armed?: ArmedDocument; failure?: FrameCaptureFailure }> {
if (!frame.documentId) {
return {
failure: makeFrameCaptureFailure("not_injectable", {
frameId: frame.frameId,
url: frame.url,
detail: "missing documentId",
}),
};
}
if (isFrameUrlRestricted(frame.url)) {
return {
failure: makeFrameCaptureFailure("not_injectable", {
frameId: frame.frameId,
documentId: frame.documentId,
url: frame.url,
}),
};
}
try {
await sendRecordStartWithAckToDocument(tabId, frame.documentId, startMsg, deps);
return {
armed: {
tabId,
frameId: frame.frameId,
documentId: frame.documentId,
url: frame.url,
parentFrameId: frame.parentFrameId,
},
};
} catch (err) {
return {
failure: makeFrameCaptureFailure("arm_failed", {
frameId: frame.frameId,
documentId: frame.documentId,
url: frame.url,
detail: err instanceof Error ? err.message : "RECORD_START failed",
}),
};
}
}
export async function armAllDocumentsForTab(
tabId: number,
startMsg: RecordStartMessage,
deps: RecordFrameSendDeps,
): Promise<{ armed: ArmedDocument[]; failures: FrameCaptureFailure[] }> {
const { injectable, failures } = await listInjectableFrames(tabId, deps);
const armed: ArmedDocument[] = [];
for (const frame of injectable) {
const result = await armDocumentCapture(tabId, frame, startMsg, deps);
if (result.armed) armed.push(result.armed);
if (result.failure) failures.push(result.failure);
}
return { armed, failures };
}
export async function stopDocumentCapture(
armed: ArmedDocument,
stopMsg: RecordStopMessage,
deps: RecordFrameSendDeps,
): Promise<{ ok: true } | FrameCaptureFailure> {
try {
const response = await deps.sendToDocument(armed.tabId, armed.documentId, stopMsg);
if (isRecordStopAckOk(response)) return { ok: true };
return makeFrameCaptureFailure("flush_failed", {
frameId: armed.frameId,
documentId: armed.documentId,
url: armed.url,
detail: "content script did not confirm recorded steps",
});
} catch (err) {
return makeFrameCaptureFailure("flush_failed", {
frameId: armed.frameId,
documentId: armed.documentId,
url: armed.url,
detail: err instanceof Error ? err.message : "RECORD_STOP failed",
});
}
}
export async function cancelDocumentCapture(
armed: ArmedDocument,
cancelMsg: RecordCancelMessage,
deps: RecordFrameSendDeps,
): Promise<void> {
try {
await deps.sendToDocument(armed.tabId, armed.documentId, cancelMsg);
} catch {
// Best-effort cleanup.
}
}
@@ -0,0 +1,673 @@
import type { RenderedRef } from "@browser-skill/vom";
import { formatObservationFile, type ObservationAnnotation } from "@/lib/format-observation-file";
import { fallbackDescriptor, matchTarget } from "@/lib/match-target";
import { waitForPageSettled } from "@/lib/page-settled";
import {
CAPTURE_RETRY_DELAY_MS,
DEFAULT_MAX_PAGE_TOKENS,
OBSERVATION_MIN_INTERVAL_MS,
} from "@/lib/record-constants";
import { registerObservation, type StateRegistryEntry } from "@/lib/trace-reducer";
import { captureVomObservation } from "@/tools/capture-vom-observation";
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import type { CapturedNode } from "@/tools/vom/capture";
import type { DraftTraceStep, Step, StopReason, Trace, TraceState } from "@/transport/types";
import { reduceTraceSteps, resolveTraceStartUrl } from "./trace-reducer";
export interface LastSettledObservation {
stateId: string;
captured: CapturedNode[];
matchNodes: CapturedNode[];
documentTokenOwners: Map<string, number | null>;
refs: RenderedRef[];
url: string;
title?: string;
vomText: string;
}
/** A post-action observation that has been queued but not yet written down. */
interface PendingSettle {
draftIndex: number;
/** Set when a newer action has already decided where this step landed. */
cancelled: boolean;
}
/** Latest URL in an in-flight redirect chain awaiting coalesce + settle. */
export interface PendingRedirectLanding {
url: string;
/** Bumped on every hop so an in-flight `waitForPageSettled` can cancel. */
generation: number;
}
export interface RecordingObservationState {
stateRegistry: Map<string, StateRegistryEntry>;
lastSettled: LastSettledObservation | null;
/** Latest token→owner map from the most recent observation capture. */
documentTokenOwners: Map<string, number | null>;
maxPageTokens: number;
redactValues: boolean;
lastCaptureAtMs: number;
/**
* Tail of the settle chain. Observations run one at a time so that a slow
* capture can never land after a faster one and report the pages out of the
* order the user visited them.
*/
settleQueue: Promise<void>;
/** Queued and in-flight settles, by draft index, so newer actions can supersede them. */
settles: Map<number, PendingSettle>;
stepAnnotations: Map<number, ObservationAnnotation[]>;
/**
* OAuth / server redirects are coalesced here: intermediate hops only update
* `url`+`generation`, and one `navigate` is emitted after the page settles.
*/
pendingRedirect: PendingRedirectLanding | null;
redirectFlushQueue: Promise<void>;
}
export function createObservationState(options?: {
maxPageTokens?: number;
redactValues?: boolean;
}): RecordingObservationState {
return {
stateRegistry: new Map(),
lastSettled: null,
documentTokenOwners: new Map(),
maxPageTokens: options?.maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS,
redactValues: options?.redactValues ?? false,
lastCaptureAtMs: 0,
settleQueue: Promise.resolve(),
settles: new Map(),
stepAnnotations: new Map(),
pendingRedirect: null,
redirectFlushQueue: Promise.resolve(),
};
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
async function readTabMeta(
tabsApi: ChromeTabsApi,
tabId: number,
): Promise<{ url: string; title?: string }> {
try {
const tab = await tabsApi.get(tabId);
return { url: tab.url ?? "about:blank", title: tab.title };
} catch {
return { url: "about:blank" };
}
}
export async function captureAndRegisterObservation(
obs: RecordingObservationState,
cdp: CdpRunner,
tabId: number,
tabsApi: ChromeTabsApi,
urlOverride?: string,
): Promise<LastSettledObservation> {
const now = Date.now();
const waitMs = Math.max(0, OBSERVATION_MIN_INTERVAL_MS - (now - obs.lastCaptureAtMs));
if (waitMs > 0) await sleep(waitMs);
const { url, title } = urlOverride
? { url: urlOverride, title: undefined }
: await readTabMeta(tabsApi, tabId);
const meta = urlOverride ? await readTabMeta(tabsApi, tabId) : { url, title };
const resolvedTitle = title ?? meta.title;
const rendered = await captureVomObservation(cdp, tabId, url, {
maxTokens: obs.maxPageTokens,
redactValues: obs.redactValues,
conditionalSurfaceProbe: false,
});
const stateId = registerObservation(obs.stateRegistry, {
url,
title: resolvedTitle,
rawVomText: rendered.text,
truncated: rendered.truncated,
});
obs.lastCaptureAtMs = Date.now();
obs.documentTokenOwners = rendered.documentTokenOwners;
const settled: LastSettledObservation = {
stateId,
captured: rendered.captured,
matchNodes: rendered.matchNodes,
documentTokenOwners: rendered.documentTokenOwners,
refs: rendered.refs,
url,
title: resolvedTitle,
vomText: rendered.text,
};
obs.lastSettled = settled;
return settled;
}
/**
* Bind a draft to the page it was performed on: origin state, geometric target
* match, and the inline annotation. All three read the *current* observation,
* which is only the page the user acted on while the draft is still fresh —
* once the action settles, `lastSettled` is the destination page instead.
*/
export function applyTargetMatching(
obs: RecordingObservationState,
draft: DraftTraceStep,
stepId: number,
): void {
if (!obs.lastSettled) {
// Initial observation can fail or race the first action. Keep whatever
// the content script captured so the exported step is still teachable.
if ("target" in draft && "captureTarget" in draft) {
draft.target = fallbackDescriptor(draft.captureTarget);
}
rememberStepAnnotation(obs, stepId, draft);
return;
}
draft.preStateId = obs.lastSettled.stateId;
if ("target" in draft) {
draft.target =
"geometry" in draft && draft.geometry
? matchTarget({
geometry: draft.geometry,
captured: obs.lastSettled.matchNodes,
refs: obs.lastSettled.refs,
fallback: draft.captureTarget,
})
: // No geometry to match on, but the capture still knew what the user
// touched — keep that instead of an anonymous unmatched target.
fallbackDescriptor(draft.captureTarget);
}
rememberStepAnnotation(obs, stepId, draft);
}
function fillDetailForDraft(
obs: RecordingObservationState,
draft: DraftTraceStep,
): string | undefined {
if (obs.redactValues) return undefined;
if (draft.op === "fill") return JSON.stringify(draft.value);
return undefined;
}
function refLineForDraft(
obs: RecordingObservationState,
draft: DraftTraceStep,
): number | undefined {
if (!("target" in draft) || !obs.lastSettled) return undefined;
const targetRef = draft.target?.ref;
if (!targetRef) return undefined;
return obs.lastSettled.refs.find((r) => r.ref === targetRef)?.line;
}
function rememberStepAnnotation(
obs: RecordingObservationState,
stepId: number,
draft: DraftTraceStep,
): void {
const line = refLineForDraft(obs, draft);
if (line === undefined || draft.op === "navigate" || draft.op === "scroll") return;
if (!draft.preStateId) return;
const ann: ObservationAnnotation = {
stepId,
op: draft.op,
line,
stateId: draft.preStateId,
detail: fillDetailForDraft(obs, draft),
};
const bucket = obs.stepAnnotations.get(line) ?? [];
bucket.push(ann);
obs.stepAnnotations.set(line, bucket);
}
/** Record the step under the page it was performed on, not the one it led to. */
export function rememberStepOnPage(
obs: RecordingObservationState,
draft: DraftTraceStep,
stepId: number,
): void {
const stateId = draft.preStateId;
if (!stateId) return;
const entry = obs.stateRegistry.get(stateId);
if (entry && !entry.stepsHere.includes(stepId)) {
entry.stepsHere.push(stepId);
}
}
/** Drop an in-flight redirect coalesce (e.g. a real navigate superseded it). */
export function clearPendingRedirectLanding(obs: RecordingObservationState): void {
obs.pendingRedirect = null;
}
function enqueueRedirectFlush(obs: RecordingObservationState, task: () => Promise<void>): void {
obs.redirectFlushQueue = obs.redirectFlushQueue.then(task, task).catch(() => {});
}
/**
* Remember a redirect hop and schedule a settle-then-emit of a single navigate
* to the final URL. Later hops only bump `generation` / replace `url`.
*/
export function scheduleRedirectLandingFlush(
obs: RecordingObservationState,
steps: DraftTraceStep[],
cdp: CdpRunner,
tabId: number,
tabsApi: ChromeTabsApi,
url: string,
options: { cancelled?: () => boolean } = {},
): void {
const generation = (obs.pendingRedirect?.generation ?? 0) + 1;
obs.pendingRedirect = { url, generation };
enqueueRedirectFlush(obs, () =>
coalesceRedirectLanding(obs, steps, cdp, tabId, tabsApi, options),
);
}
/**
* Drain any coalesced redirect so the next action matches against the real
* landing page. Safe to call when nothing is pending.
*/
export async function flushPendingRedirectLanding(
obs: RecordingObservationState,
steps: DraftTraceStep[],
cdp: CdpRunner | undefined,
tabId: number,
tabsApi: ChromeTabsApi,
options: { cancelled?: () => boolean } = {},
): Promise<void> {
if (obs.pendingRedirect && cdp) {
enqueueRedirectFlush(obs, () =>
coalesceRedirectLanding(obs, steps, cdp, tabId, tabsApi, options),
);
}
await obs.redirectFlushQueue;
}
/**
* Wait until the redirect chain stops changing the document, then emit one
* `navigate` (cause `browser`) to the tab's final URL and settle its observation.
*/
async function coalesceRedirectLanding(
obs: RecordingObservationState,
steps: DraftTraceStep[],
cdp: CdpRunner,
tabId: number,
tabsApi: ChromeTabsApi,
options: { cancelled?: () => boolean },
): Promise<void> {
while (obs.pendingRedirect) {
if (options.cancelled?.()) {
obs.pendingRedirect = null;
return;
}
const snap = obs.pendingRedirect;
const outcome = await waitForPageSettled(cdp, tabId, {
cancelled: () =>
!!options.cancelled?.() ||
obs.pendingRedirect === null ||
obs.pendingRedirect.generation !== snap.generation,
});
if (options.cancelled?.()) {
obs.pendingRedirect = null;
return;
}
// A newer hop cancelled this wait — loop and settle against the latest URL.
if (outcome === "cancelled") continue;
if (!obs.pendingRedirect || obs.pendingRedirect.generation !== snap.generation) {
continue;
}
const finalUrl = (await readTabMeta(tabsApi, tabId)).url || snap.url;
obs.pendingRedirect = null;
if (!finalUrl || finalUrl === "about:blank") return;
if (obs.lastSettled?.url === finalUrl) return;
const last = steps[steps.length - 1];
if (last?.op === "navigate" && last.url === finalUrl) {
if (!last.postStateId) {
const draftIndex = steps.length - 1;
scheduleDraftSettle(obs, draftIndex, draftIndex + 1, cdp, tabId, tabsApi, steps);
await obs.settleQueue;
}
return;
}
if (outcome === "timeout") {
console.debug(
`[bsk record] redirect landing still changing after settle budget; ` +
`recording navigate to ${finalUrl}`,
);
}
const draft: DraftTraceStep = {
op: "navigate",
url: finalUrl,
page_url: finalUrl,
cause: "browser",
preStateId: obs.lastSettled?.stateId,
};
steps.push(draft);
const draftIndex = steps.length - 1;
const stepId = draftIndex + 1;
rememberStepOnPage(obs, draft, stepId);
scheduleDraftSettle(obs, draftIndex, stepId, cdp, tabId, tabsApi, steps);
// Callers that flush before the next action need `lastSettled` to already
// be the landing page so target matching does not use the pre-redirect view.
await obs.settleQueue;
return;
}
}
/**
* A capture that lands mid-navigation fails on a destroyed execution context.
* That is transient, so give it one more chance before the step has to fall
* back to a stale state.
*/
async function captureWithRetry(
obs: RecordingObservationState,
cdp: CdpRunner,
tabId: number,
tabsApi: ChromeTabsApi,
urlOverride?: string,
): Promise<LastSettledObservation> {
try {
return await captureAndRegisterObservation(obs, cdp, tabId, tabsApi, urlOverride);
} catch (err) {
console.debug("[bsk record] observation failed, retrying once", err);
await sleep(CAPTURE_RETRY_DELAY_MS);
return captureAndRegisterObservation(obs, cdp, tabId, tabsApi, urlOverride);
}
}
export async function settleDraftObservation(
obs: RecordingObservationState,
cdp: CdpRunner,
tabId: number,
tabsApi: ChromeTabsApi,
steps: DraftTraceStep[],
draftIndex: number,
pending?: PendingSettle,
): Promise<void> {
const cancelled = () => pending?.cancelled === true;
const outcome = await waitForPageSettled(cdp, tabId, { cancelled });
if (outcome === "cancelled") return;
if (outcome === "timeout") {
console.debug(
`[bsk record] step ${draftIndex + 1} was still changing the page after the settle ` +
"budget; observing it as it stands",
);
}
const started = steps[draftIndex];
if (!started) return;
// The URL attached to the draft may only be an intermediate redirect hop.
// Once the page is settled, register the capture against live tab metadata.
const settled = await captureWithRetry(obs, cdp, tabId, tabsApi);
// The observation still counts as the latest view of the page even when this
// step no longer wants it — the next action will start from it.
if (cancelled()) return;
// Re-read the slot: a navigation observed while the capture ran may have
// rewritten this draft, and the observation belongs to whatever occupies the
// slot now — writing to the object we started with would strand it.
const draft = steps[draftIndex] ?? started;
draft.postStateId = settled.stateId;
}
/**
* Recover landing states from the shape of the recording itself: wherever the
* next action was performed is, by definition, where the previous one landed.
* Only the immediately following draft counts — a later one would claim a page
* that several unobserved actions away, which is worse than admitting we saw
* no change.
*/
export function inferMissingPostStates(steps: DraftTraceStep[]): void {
for (let i = 0; i < steps.length - 1; i += 1) {
const draft = steps[i];
const next = steps[i + 1];
if (!draft || !next || draft.postStateId || !next.preStateId) continue;
draft.postStateId = next.preStateId;
console.debug(
`[bsk record] step ${i + 1} (${draft.op}) had no post-action observation; ` +
`using where step ${i + 2} started (${next.preStateId})`,
);
}
}
/**
* Last chance for the closing actions of a recording. Stopping right after the
* final action is normal, and so is that action navigating away, so take one
* final look at the page. Only trailing drafts may claim it: an earlier step
* did not land on whatever the user happens to be looking at when they stop.
*/
export async function settleUnsettledDrafts(
obs: RecordingObservationState,
cdp: CdpRunner,
tabId: number,
tabsApi: ChromeTabsApi,
steps: DraftTraceStep[],
): Promise<void> {
const trailing: DraftTraceStep[] = [];
for (let i = steps.length - 1; i >= 0; i -= 1) {
const draft = steps[i];
if (!draft || draft.postStateId) break;
trailing.push(draft);
}
if (trailing.length === 0) return;
let settled: LastSettledObservation;
try {
settled = await captureWithRetry(obs, cdp, tabId, tabsApi);
} catch (err) {
console.warn("[bsk record] final observation at stop failed", err);
return;
}
for (const draft of trailing) draft.postStateId = settled.stateId;
console.debug(
`[bsk record] settled ${trailing.length} trailing step(s) against the page at stop ` +
`(${settled.stateId} ${settled.url})`,
);
}
/**
* Performing a new action ends the previous one: the page the user reached for
* is, by definition, where the earlier step left them. Taking the landing from
* the newer step also keeps the trace monotonic — a capture that finished late
* would otherwise credit an earlier step with a page that only exists because
* of the newer action.
*/
function supersedeEarlierSettles(
obs: RecordingObservationState,
draftIndex: number,
steps: DraftTraceStep[],
): void {
const landing = steps[draftIndex]?.preStateId;
for (const [index, pending] of obs.settles) {
if (index >= draftIndex) continue;
pending.cancelled = true;
obs.settles.delete(index);
const draft = steps[index];
if (!draft || draft.postStateId) continue;
if (!landing) continue;
draft.postStateId = landing;
console.debug(
`[bsk record] step ${index + 1} (${draft.op}) was still settling when step ` +
`${draftIndex + 1} started; landing it on ${landing}`,
);
}
}
export function scheduleDraftSettle(
obs: RecordingObservationState,
draftIndex: number,
stepId: number,
cdp: CdpRunner,
tabId: number,
tabsApi: ChromeTabsApi,
steps: DraftTraceStep[],
): void {
supersedeEarlierSettles(obs, draftIndex, steps);
// Rescheduling the same step (a navigation showed up after the action)
// replaces the pending observation rather than racing it.
const superseded = obs.settles.get(draftIndex);
if (superseded) superseded.cancelled = true;
const pending: PendingSettle = { draftIndex, cancelled: false };
obs.settles.set(draftIndex, pending);
enqueueSettle(obs, async () => {
if (pending.cancelled) return;
try {
await settleDraftObservation(obs, cdp, tabId, tabsApi, steps, draftIndex, pending);
} catch (err) {
// Recoverable: stop-time repair still gives the step a landing page.
const op = steps[draftIndex]?.op ?? "?";
console.warn(`[bsk record] post-action observation failed for step ${stepId} (${op})`, err);
} finally {
if (obs.settles.get(draftIndex) === pending) obs.settles.delete(draftIndex);
}
});
}
function enqueueSettle(obs: RecordingObservationState, task: () => Promise<void>): void {
obs.settleQueue = obs.settleQueue.then(task, task).catch(() => {});
}
export function cancelPendingSettles(obs: RecordingObservationState): void {
for (const pending of obs.settles.values()) pending.cancelled = true;
obs.settles.clear();
}
/** Bound the drain loop so a self-rescheduling settle cannot block stop. */
const MAX_SETTLE_FLUSH_ROUNDS = 10;
/**
* Drain settle work before exporting, including work queued while draining —
* a navigation observed during the last capture schedules another one, and
* the trace would drop it if stop did not wait.
*/
export async function flushPendingSettles(obs: RecordingObservationState): Promise<void> {
for (let round = 0; round < MAX_SETTLE_FLUSH_ROUNDS; round += 1) {
const drained = obs.settleQueue;
await drained;
if (obs.settleQueue === drained) return;
}
console.warn("[bsk record] settle queue kept growing at stop; exporting what has been observed");
}
function finalizeStateBodies(
entries: StateRegistryEntry[],
annotationsByState: Map<string, ObservationAnnotation[]>,
stepIdByDraftId: Map<number, number>,
idByOldId: Map<string, string>,
): TraceState[] {
return entries.map((entry) => {
const id = idByOldId.get(entry.id) ?? entry.id;
const body = formatObservationFile({
stateId: id,
url: entry.url,
title: entry.title,
stepsHere: remapStepIds(entry.stepsHere, stepIdByDraftId),
body: entry.rawVomText,
annotations: annotationsByState.get(entry.id) ?? [],
});
return {
id,
url: entry.url,
...(entry.title ? { title: entry.title } : {}),
body,
...(entry.truncated ? { truncated: true } : {}),
};
});
}
/** Draft ids only become step ids after collapsing and filtering. */
function remapStepIds(draftIds: number[], stepIdByDraftId: Map<number, number>): number[] {
const mapped = new Set<number>();
for (const draftId of draftIds) {
const stepId = stepIdByDraftId.get(draftId);
if (stepId !== undefined) mapped.add(stepId);
}
return [...mapped].sort((a, b) => a - b);
}
function collectAnnotationsByState(
obs: RecordingObservationState,
stepIdByDraftId: Map<number, number>,
): Map<string, ObservationAnnotation[]> {
const byState = new Map<string, ObservationAnnotation[]>();
for (const anns of obs.stepAnnotations.values()) {
for (const ann of anns) {
const stepId = stepIdByDraftId.get(ann.stepId);
if (stepId === undefined) continue;
const bucket = byState.get(ann.stateId) ?? [];
bucket.push({ ...ann, stepId });
byState.set(ann.stateId, bucket);
}
}
return byState;
}
/**
* Keep only the pages the published steps point at. Redirect hops and
* mid-load captures land in the registry too, and shipping them would invite
* a reader to treat a page the flow merely passed through as a real stop.
* With no steps at all the first observation is the whole artifact, so it
* stays.
*/
function selectPublishedStates(
registry: Map<string, StateRegistryEntry>,
steps: Step[],
): StateRegistryEntry[] {
const entries = [...registry.values()];
if (steps.length === 0) return entries.slice(0, 1);
const referenced = new Set<string>();
for (const step of steps) {
referenced.add(step.state);
referenced.add(step.result.state);
}
return entries.filter((entry) => referenced.has(entry.id));
}
export function buildTraceV3(input: {
obs: RecordingObservationState;
steps: DraftTraceStep[];
startedAt: string;
purpose?: string;
startUrl?: string;
stoppedBy: StopReason;
bskVersion: string;
frameCapture?: Trace["frame_capture"];
}): Trace {
const { steps, stepIdByDraftId } = reduceTraceSteps(input.steps, input.obs.stateRegistry);
const published = selectPublishedStates(input.obs.stateRegistry, steps);
// Renumber so the shipped dictionary reads s1..sN without holes where the
// dropped captures used to be.
const idByOldId = new Map(published.map((entry, index) => [entry.id, `s${index + 1}`]));
for (const step of steps) {
step.state = idByOldId.get(step.state) ?? step.state;
step.result.state = idByOldId.get(step.result.state) ?? step.result.state;
}
const annotationsByState = collectAnnotationsByState(input.obs, stepIdByDraftId);
const states = finalizeStateBodies(published, annotationsByState, stepIdByDraftId, idByOldId);
const startUrl = resolveTraceStartUrl(input.steps, input.startUrl, states);
return {
version: 3,
...(input.purpose ? { purpose: input.purpose } : {}),
recorded_at: new Date().toISOString(),
started_at: input.startedAt,
stopped_by: input.stoppedBy,
entry: { start_url: startUrl },
recorder: { bsk: input.bskVersion, vom: 1 },
...(input.frameCapture ? { frame_capture: input.frameCapture } : {}),
states,
steps,
};
}
+102 -26
View File
@@ -1,23 +1,29 @@
import type { DraftTraceStep } from "@/transport/types";
import type { DraftTraceStep, NavigationCause } from "@/transport/types";
import type { RecordStepPayload } from "./record-bridge";
import { mapNavigationCause } from "./record-constants";
export interface RecordingStepBuffer {
steps: DraftTraceStep[];
currentUrl?: string;
pendingNavigation: boolean;
pendingNavigationDeadline?: number;
lastTransitionType?: string;
lastTransitionQualifiers?: string[];
}
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
const pageUrl = payload.page_url;
const geometry = payload.geometry;
switch (payload.op) {
case "click":
return payload.target
? {
op: "click",
target: payload.target,
target: { unmatched: true },
captureTarget: payload.target,
...(geometry ? { geometry } : {}),
...(pageUrl ? { page_url: pageUrl } : {}),
}
: null;
@@ -25,7 +31,9 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
return payload.target
? {
op: "hover",
target: payload.target,
target: { unmatched: true },
captureTarget: payload.target,
...(geometry ? { geometry } : {}),
...(pageUrl ? { page_url: pageUrl } : {}),
}
: null;
@@ -33,8 +41,11 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
return payload.target
? {
op: "fill",
target: payload.target,
target: { unmatched: true },
captureTarget: payload.target,
...(geometry ? { geometry } : {}),
value: payload.value ?? "",
...(payload.commit ? { commit: payload.commit } : {}),
...(payload.redacted ? { redacted: true } : {}),
...(pageUrl ? { page_url: pageUrl } : {}),
}
@@ -44,7 +55,10 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
? {
op: "press",
key: payload.key,
...(payload.target ? { target: payload.target } : {}),
...(payload.target
? { target: { unmatched: true }, captureTarget: payload.target }
: {}),
...(geometry ? { geometry } : {}),
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
...(pageUrl ? { page_url: pageUrl } : {}),
}
@@ -53,37 +67,69 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
return payload.target && payload.values
? {
op: "select",
target: payload.target,
target: { unmatched: true },
captureTarget: payload.target,
...(geometry ? { geometry } : {}),
values: payload.values,
...(payload.labels?.length ? { labels: payload.labels } : {}),
...(pageUrl ? { page_url: pageUrl } : {}),
}
: null;
case "scroll":
return { op: "scroll", ...(pageUrl ? { page_url: pageUrl } : {}) };
case "navigate":
return null;
}
}
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): boolean {
/** Annotate the latest action with `navigated_to`. Returns its index, or -1. */
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): number {
for (let i = buffer.steps.length - 1; i >= 0; i -= 1) {
const step = buffer.steps[i];
if (!step) continue;
if (step.op === "click" || step.op === "press" || step.op === "select") {
buffer.steps[i] = { ...step, navigated_to: url };
return true;
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
// Mutate in place: an in-flight settle holds this object, and replacing
// it would strand that settle's observation on the discarded copy.
step.navigated_to = url;
return i;
}
break;
}
return false;
return -1;
}
function causeFromBuffer(buffer: RecordingStepBuffer): NavigationCause | null {
return mapNavigationCause({
transitionType: buffer.lastTransitionType,
transitionQualifiers: buffer.lastTransitionQualifiers,
});
}
/** Outcome of folding a main-frame URL change into the draft buffer. */
export type NavigationObserveResult =
| { kind: "noop" }
| { kind: "annotated"; index: number }
| { kind: "appended"; index: number }
/**
* Redirect hop: do not emit a step yet. The recorder coalesces the chain and
* emits one `navigate` after `waitForPageSettled` on the final URL.
*/
| { kind: "coalesce_redirect"; url: string };
export function observeRecordedNavigation(
buffer: RecordingStepBuffer,
url: string,
causedByAction?: boolean,
): void {
if (!url || url === buffer.currentUrl) return;
transitionType?: string,
transitionQualifiers?: string[],
): NavigationObserveResult {
if (!url || url === buffer.currentUrl) return { kind: "noop" };
buffer.currentUrl = url;
if (transitionType !== undefined) buffer.lastTransitionType = transitionType;
if (transitionQualifiers !== undefined) {
buffer.lastTransitionQualifiers = transitionQualifiers;
}
const pendingIsCurrent =
buffer.pendingNavigation &&
(buffer.pendingNavigationDeadline === undefined ||
@@ -92,40 +138,69 @@ export function observeRecordedNavigation(
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
buffer.pendingNavigation = false;
buffer.pendingNavigationDeadline = undefined;
if (!annotateLastStepNavigation(buffer, url)) {
buffer.steps.push({
op: "navigate",
url,
page_url: url,
});
const annotatedIndex = annotateLastStepNavigation(buffer, url);
if (annotatedIndex >= 0) {
return { kind: "annotated", index: annotatedIndex };
}
return;
const cause = causeFromBuffer(buffer);
if (cause === null) {
return { kind: "coalesce_redirect", url };
}
buffer.steps.push({
op: "navigate",
url,
page_url: url,
cause,
transitionType: buffer.lastTransitionType,
transitionQualifiers: buffer.lastTransitionQualifiers,
});
return { kind: "appended", index: buffer.steps.length - 1 };
}
buffer.pendingNavigation = false;
buffer.pendingNavigationDeadline = undefined;
const cause = causeFromBuffer(buffer);
if (cause === null) {
return { kind: "coalesce_redirect", url };
}
buffer.steps.push({
op: "navigate",
url,
page_url: url,
cause,
transitionType: buffer.lastTransitionType,
transitionQualifiers: buffer.lastTransitionQualifiers,
});
return { kind: "appended", index: buffer.steps.length - 1 };
}
/**
* Buffer one captured payload. Returns the index of the draft it appended, or
* `null` when the payload produced none (an unnamed click target, a navigation
* that only annotated the previous draft, …). Callers must not fall back to
* "the last draft" on `null`: that would re-match an already recorded step
* against a newer observation and overwrite its origin state.
*/
export function appendRecordedPayload(
buffer: RecordingStepBuffer,
payload: RecordStepPayload,
): void {
): number | null {
if (payload.op === "navigate") {
if (payload.url) {
observeRecordedNavigation(buffer, payload.url, payload.navigation_caused_by_action);
}
return;
if (!payload.url) return null;
const result = observeRecordedNavigation(
buffer,
payload.url,
payload.navigation_caused_by_action,
payload.transitionType,
payload.transitionQualifiers,
);
return result.kind === "appended" ? result.index : null;
}
const step = toDraftStep({
...payload,
page_url: payload.page_url ?? buffer.currentUrl,
});
if (!step) return;
if (!step) return null;
buffer.steps.push(step);
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
buffer.pendingNavigation = payload.expects_navigation === true;
@@ -133,4 +208,5 @@ export function appendRecordedPayload(
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
: undefined;
}
return buffer.steps.length - 1;
}
+151 -140
View File
@@ -1,4 +1,17 @@
import type { DraftTraceStep, PageRef, SelectedOption, Step } from "@/transport/types";
import type {
DraftTraceStep,
FillCommit,
NavigationCause,
Step,
TraceState,
} from "@/transport/types";
import {
DEFAULT_FILL_COMMIT,
fnv1a64,
mapNavigationCause,
nextStateId,
resetStateIdCounterForTests,
} from "./record-constants";
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
@@ -12,206 +25,200 @@ export function shouldRecordPress(
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
if (key === "Enter" || key === "Escape") return true;
// Drop bare character typing — FillSession already records the value.
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
return false;
}
function shouldIncludeDraft(step: DraftTraceStep): boolean {
if (step.op === "fill" && !(step.value ?? "").trim() && !step.redacted) return false;
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
return true;
}
/** Collapse consecutive navigations to the last hop. */
function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
const out: DraftTraceStep[] = [];
for (const step of steps) {
/** A reduced draft plus the capture-order ids of every draft folded into it. */
interface CollapsedDraft {
draft: DraftTraceStep;
draftIds: number[];
}
/**
* Collapse a redirect chain into a single hop.
*
* The surviving step keeps the *first* hop's origin state and cause — that is
* where the user was and why they left — and the *last* hop's destination, so
* the intermediate URL the browser only passed through never becomes the
* step's `state` or `to`.
*/
function collapseNavigations(steps: DraftTraceStep[]): CollapsedDraft[] {
const out: CollapsedDraft[] = [];
steps.forEach((step, index) => {
const draftId = index + 1;
const prev = out[out.length - 1];
if (step.op === "navigate" && prev?.op === "navigate") {
out[out.length - 1] = step;
continue;
if (step.op === "navigate" && prev?.draft.op === "navigate") {
prev.draft = {
...prev.draft,
url: step.url,
postStateId: step.postStateId ?? prev.draft.postStateId,
};
prev.draftIds.push(draftId);
return;
}
out.push(step);
}
out.push({ draft: step, draftIds: [draftId] });
});
return out;
}
function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
const urls: string[] = [];
if (startUrl) urls.push(startUrl);
for (const step of steps) {
if (step.op === "navigate") {
urls.push(step.url);
continue;
}
if ("page_url" in step && step.page_url) urls.push(step.page_url);
if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to);
}
const seen = new Set<string>();
const unique: string[] = [];
for (const url of urls) {
if (!url || seen.has(url)) continue;
seen.add(url);
unique.push(url);
}
return unique;
export interface StateRegistryEntry extends TraceState {
contentHash: string;
rawVomText: string;
stepsHere: number[];
}
function buildPageRegistry(
steps: DraftTraceStep[],
startUrl?: string,
): { pages: PageRef[]; urlToId: Map<string, string> } {
const urls = collectUrls(steps, startUrl);
const urlToId = new Map<string, string>();
const pages = urls.map((url, index) => {
const id = `p${index + 1}`;
urlToId.set(url, id);
return { id, url };
});
return { pages, urlToId };
export function stateDedupKey(body: string, url: string): string {
return `${fnv1a64(body)}:${url}`;
}
function pageIdFor(
url: string | undefined,
urlToId: Map<string, string>,
fallbackUrl?: string,
/** Register or reuse a page observation in the states dictionary. */
export function registerObservation(
registry: Map<string, StateRegistryEntry>,
input: {
url: string;
title?: string;
rawVomText: string;
truncated?: boolean;
},
): string {
if (url && urlToId.has(url)) return urlToId.get(url)!;
if (fallbackUrl && urlToId.has(fallbackUrl)) return urlToId.get(fallbackUrl)!;
return urlToId.values().next().value ?? "p1";
const key = stateDedupKey(input.rawVomText, input.url);
for (const entry of registry.values()) {
if (entry.contentHash === key) return entry.id;
}
const id = nextStateId();
registry.set(id, {
id,
url: input.url,
...(input.title ? { title: input.title } : {}),
truncated: input.truncated ?? false,
contentHash: key,
rawVomText: input.rawVomText,
stepsHere: [],
});
return id;
}
function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined {
if (step.op === "navigate") return step.page_url ?? step.url;
if ("page_url" in step && step.page_url) return step.page_url;
return fallbackUrl;
}
function effectForNavigation(
navigatedTo: string | undefined,
urlToId: Map<string, string>,
): Step["effect"] {
if (!navigatedTo) return undefined;
const pageId = urlToId.get(navigatedTo);
if (!pageId) return undefined;
return { navigated_to: pageId };
}
function withEffect(step: Step, effect: Step["effect"]): Step {
if (!effect) return step;
return { ...step, effect };
}
function toSelection(values: string[], labels?: string[]): SelectedOption[] {
function toSelection(
values: string[],
labels?: string[],
): Array<{ value: string; label?: string }> {
return values.map((value, index) => ({
value,
...(labels?.[index] ? { label: labels[index] } : {}),
}));
}
function toV2Step(
step: DraftTraceStep,
id: number,
urlToId: Map<string, string>,
fallbackUrl?: string,
): Step | null {
if (!shouldIncludeDraft(step)) return null;
function navigateCauseForDraft(
draft: Extract<DraftTraceStep, { op: "navigate" }>,
): NavigationCause {
if (draft.cause) return draft.cause;
const mapped = mapNavigationCause({
transitionType: draft.transitionType,
transitionQualifiers: draft.transitionQualifiers,
});
return mapped ?? "browser";
}
const pageUrl = pageUrlForDraft(step, fallbackUrl);
const page = pageIdFor(pageUrl, urlToId, fallbackUrl);
function toV3Step(draft: DraftTraceStep, id: number): Step | null {
if (!shouldIncludeDraft(draft)) return null;
// An action the user really performed must not disappear because one of its
// observations is missing — a capture racing a document swap is routine, and
// the last step of a recording is the most exposed to it. Fall back to the
// endpoint we do know; only a step with no observation at all has nothing to
// point at.
const preState = draft.preStateId ?? draft.postStateId;
const postState = draft.postStateId ?? draft.preStateId;
if (!preState || !postState) return null;
switch (step.op) {
const common = { id, state: preState, result: { state: postState } };
switch (draft.op) {
case "navigate":
return {
op: "navigate",
id,
page: pageIdFor(step.url, urlToId, fallbackUrl),
to: step.url,
...common,
to: draft.url,
cause: navigateCauseForDraft(draft),
};
case "click":
return withEffect(
{
op: "click",
id,
page,
target: step.target,
},
effectForNavigation(step.navigated_to, urlToId),
);
return { op: "click", ...common, target: draft.target };
case "hover":
return {
op: "hover",
id,
page,
target: step.target,
};
return { op: "hover", ...common, target: draft.target };
case "fill":
return {
op: "fill",
id,
page,
target: step.target,
value: step.value,
...(step.redacted ? { redacted: true } : {}),
...common,
target: draft.target,
value: draft.value,
commit: draft.commit ?? DEFAULT_FILL_COMMIT,
...(draft.redacted ? { redacted: true } : {}),
};
case "press":
return withEffect(
{
op: "press",
id,
page,
key: step.key,
...(step.target ? { target: step.target } : {}),
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
},
effectForNavigation(step.navigated_to, urlToId),
);
return {
op: "press",
...common,
key: draft.key,
...(draft.target ? { target: draft.target } : {}),
...(draft.modifiers?.length ? { modifiers: draft.modifiers } : {}),
};
case "select":
return withEffect(
{
op: "select",
id,
page,
target: step.target,
selection: toSelection(step.values, step.labels),
},
effectForNavigation(step.navigated_to, urlToId),
);
return {
op: "select",
...common,
target: draft.target,
selection: toSelection(draft.values, draft.labels),
};
case "scroll":
return { op: "scroll", ...common };
}
}
export interface ReducedTrace {
pages: PageRef[];
states: TraceState[];
steps: Step[];
/**
* Capture-order draft id (`draftIndex + 1`) → published step id. Collapsed
* and dropped drafts make the two diverge, so anything the recorder tagged
* with a draft id (`steps_here`, inline annotations) must be remapped.
*/
stepIdByDraftId: Map<number, number>;
}
/**
* Compile capture drafts into record-only trace v2 steps.
* Variable inputs are NOT classified here — executing agents infer that at run time.
* Compile capture drafts into record-only trace v3 steps.
* Drafts must already carry `preStateId` / `postStateId` from the recorder.
*/
export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): ReducedTrace {
export function reduceTraceSteps(
steps: DraftTraceStep[],
registry: Map<string, StateRegistryEntry>,
): ReducedTrace {
const collapsed = collapseNavigations(steps);
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
const out: Step[] = [];
const stepIdByDraftId = new Map<number, number>();
let id = 1;
let lastUrl = startUrl;
for (const draft of collapsed) {
if (draft.op === "navigate") lastUrl = draft.url;
else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to;
else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url;
const step = toV2Step(draft, id, urlToId, lastUrl);
for (const { draft, draftIds } of collapsed) {
const step = toV3Step(draft, id);
if (!step) continue;
out.push(step);
for (const draftId of draftIds) stepIdByDraftId.set(draftId, id);
id += 1;
}
return { pages, steps: out };
const states = [...registry.values()].map(
({ contentHash: _hash, rawVomText: _body, stepsHere: _steps, ...state }) => state,
);
return { states, steps: out, stepIdByDraftId };
}
export function resolveTraceStartUrl(
drafts: DraftTraceStep[],
startUrl?: string,
pages?: PageRef[],
states?: TraceState[],
): string {
if (startUrl) return startUrl;
const navigate = drafts.find((step): step is Extract<DraftTraceStep, { op: "navigate" }> => {
@@ -221,5 +228,9 @@ export function resolveTraceStartUrl(
for (const draft of drafts) {
if ("page_url" in draft && draft.page_url) return draft.page_url;
}
return pages?.[0]?.url ?? "about:blank";
return states?.[0]?.url ?? "about:blank";
}
export type { FillCommit, NavigationCause };
/** Test seam */
export { resetStateIdCounterForTests };
@@ -0,0 +1,86 @@
import { describe, expect, it } from "vitest";
import { RECORD_DOCUMENT_TOKEN_ATTR } from "@/lib/record-document-token";
import type { CapturedNode } from "@/tools/vom/capture";
import { collectDocumentTokenOwners } from "../capture-vom-observation";
function htmlNode(
backendNodeId: number,
token: string,
ownerFrameBackendNodeId: number | null = null,
): CapturedNode {
return {
backendNodeId,
parentBackendNodeId: null,
tag: "html",
attrs: { [RECORD_DOCUMENT_TOKEN_ATTR]: token },
rect: { x: 0, y: 0, w: 1280, h: 720 },
paintOrder: 0,
position: "static",
pointerEvents: "auto",
...(ownerFrameBackendNodeId === null ? {} : { ownerFrameBackendNodeId }),
};
}
describe("collectDocumentTokenOwners", () => {
it("maps document tokens to their immediate iframe owner backend node", () => {
const topToken = "top-doc-token";
const childToken = "child-doc-token";
const owners = collectDocumentTokenOwners({
viewport: { width: 1280, height: 720 },
nodes: [htmlNode(1, topToken, null)],
iframeNodes: new Map([
[
13,
[
htmlNode(101, childToken, 13),
{
backendNodeId: 102,
parentBackendNodeId: 101,
tag: "input",
attrs: {},
rect: { x: 10, y: 20, w: 200, h: 30 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
ownerFrameBackendNodeId: 13,
},
],
],
]),
excludedBackendNodeIds: new Set(),
});
expect(owners.get(topToken)).toBeNull();
expect(owners.get(childToken)).toBe(13);
});
it("maps nested iframe documents to their direct owner, not the outer iframe", () => {
const nestedToken = "nested-doc-token";
const owners = collectDocumentTokenOwners({
viewport: { width: 1280, height: 720 },
nodes: [htmlNode(1, "top-token", null)],
iframeNodes: new Map([
[
13,
[
{
backendNodeId: 23,
parentBackendNodeId: 101,
tag: "iframe",
attrs: {},
rect: { x: 0, y: 0, w: 400, h: 300 },
paintOrder: 1,
position: "static",
pointerEvents: "auto",
ownerFrameBackendNodeId: 13,
},
],
],
[23, [htmlNode(201, nestedToken, 23)]],
]),
excludedBackendNodeIds: new Set(),
});
expect(owners.get(nestedToken)).toBe(23);
});
});
@@ -1034,7 +1034,7 @@ describe("buildVomScene", () => {
expect(rendered).toContain(' @e2 textbox "验证码"');
});
it("does not mark non-password inputs sensitive from password-like labels", () => {
it("marks current-password autocomplete as sensitive even for text inputs", () => {
const axNodes: CdpAxNode[] = [
{
nodeId: "1",
@@ -1069,7 +1069,7 @@ describe("buildVomScene", () => {
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
expect.objectContaining({
id: 10,
sensitive: false,
sensitive: true,
}),
);
});
@@ -1364,7 +1364,9 @@ describe("buildVomScene", () => {
);
const rendered = renderVom(scene);
expect(rendered.text).toContain('@e1 button "close"');
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
{ ref: "e1", backendNodeId: 20 },
]);
});
it("does not promote a clickable container that wraps a real interactive control", () => {
@@ -1508,7 +1510,9 @@ describe("buildVomScene", () => {
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic");
const rendered = renderVom(scene);
expect(rendered.text).toContain('@e1 button "收藏"');
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
{ ref: "e1", backendNodeId: 20 },
]);
});
it("builds active scope blocks from active aria-controls relationships", () => {
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,124 @@
import { type RenderedRef, renderVom, type VomOptions } from "@browser-skill/vom";
import { readRecordDocumentTokenFromAttrs } from "@/lib/record-document-token";
import type { CdpAxNode } from "./observation";
import { buildVomScene } from "./observation";
import type { CdpRunner } from "./shared";
import {
type CapturedNode,
type CapturedSurfaceProbe,
type CapturedViewModel,
captureViewModel,
collectOverlayExcludedBackendIds,
} from "./vom/capture";
export interface CaptureVomObservationOptions extends VomOptions {
conditionalSurfaceProbe?: boolean;
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
}
export interface CaptureVomObservationResult {
text: string;
refs: RenderedRef[];
truncated: boolean;
captured: CapturedNode[];
/** All nodes including iframe sub-documents, for geometric target matching. */
matchNodes: CapturedNode[];
/** Maps ephemeral record document tokens to immediate iframe owner backend ids. */
documentTokenOwners: Map<string, number | null>;
surfaceProbes?: CapturedSurfaceProbe[];
}
export function flattenCapturedNodes(vm: CapturedViewModel): CapturedNode[] {
return [...vm.nodes, ...[...vm.iframeNodes.values()].flat()];
}
export function collectDocumentTokenOwners(vm: CapturedViewModel): Map<string, number | null> {
const owners = new Map<string, number | null>();
for (const node of flattenCapturedNodes(vm)) {
if (node.tag !== "html") continue;
const token = readRecordDocumentTokenFromAttrs(node.attrs);
if (!token) continue;
owners.set(token, node.ownerFrameBackendNodeId ?? null);
}
return owners;
}
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
}
interface LayoutMetricsViewportReply {
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
layoutViewport?: { clientWidth?: number; clientHeight?: number };
}
async function fallbackCapturedViewModel(
cdp: CdpRunner,
tabId: number,
): Promise<CapturedViewModel> {
let viewport = { width: 0, height: 0 };
try {
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
viewport = {
width: vpSrc.clientWidth ?? 0,
height: vpSrc.clientHeight ?? 0,
};
} catch {
// viewport stays zero-sized
}
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId);
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
}
async function captureForVom(
cdp: CdpRunner,
tabId: number,
conditionalSurfaceProbe: boolean,
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>,
): Promise<CapturedViewModel> {
try {
return await captureViewModel(cdp, tabId, {
conditionalSurfaceProbe,
hoverProbeBypassOverlay,
});
} catch {
return fallbackCapturedViewModel(cdp, tabId);
}
}
/** Pure VOM capture pipeline shared by snapshot, observe, and recording. */
export async function captureVomObservation(
cdp: CdpRunner,
tabId: number,
url: string | undefined,
opts: CaptureVomObservationOptions = {},
): Promise<CaptureVomObservationResult> {
await cdp.ensureAttachedToUrl?.(tabId, url);
await cdp.send<unknown>(tabId, "Accessibility.enable", {});
const result = await cdp.send<{ nodes: CdpAxNode[] }>(tabId, "Accessibility.getFullAXTree", {});
const axNodes = result.nodes ?? [];
const captured = await captureForVom(
cdp,
tabId,
opts.conditionalSurfaceProbe ?? false,
opts.hoverProbeBypassOverlay,
);
const scene = buildVomScene(axNodes, captured, { pageUrl: url });
const rendered = renderVom(scene, {
maxDepth: opts.maxDepth,
maxTokens: opts.maxTokens,
redactValues: opts.redactValues,
activeRegionPolicy: opts.activeRegionPolicy ?? true,
});
const matchNodes = flattenCapturedNodes(captured);
return {
text: rendered.text,
refs: rendered.refs,
truncated: rendered.truncated,
captured: captured.nodes,
matchNodes,
documentTokenOwners: collectDocumentTokenOwners(captured),
surfaceProbes: captured.surfaceProbes,
};
}
+25 -70
View File
@@ -8,7 +8,6 @@ import {
applyVomInteractionRecovery,
type CondSurface,
isVomReferenceNode,
renderVom,
type VomNode,
type VomScene,
} from "@browser-skill/vom";
@@ -29,6 +28,7 @@ import type {
SnapshotParams,
SnapshotResult,
} from "@/transport/types";
import { captureVomObservation } from "./capture-vom-observation";
import { attachDialogs, markDialogCursor } from "./dialogs";
import { nodeBoundingRect, scrollNodeIntoView } from "./element-geometry";
import { rpcError } from "./errors";
@@ -44,12 +44,7 @@ import {
type ToolEffect,
} from "./shared";
import { resolveSnapshotRef } from "./snapshot-ref";
import {
type CapturedNode,
type CapturedViewModel,
captureViewModel,
collectOverlayExcludedBackendIds,
} from "./vom/capture";
import { type CapturedNode, type CapturedViewModel } from "./vom/capture";
// ---------------------------------------------------------------------------
// Shared helpers (legacy aliases — observation.ts kept exporting these
@@ -350,7 +345,18 @@ function isModalSignal(
function isSensitive(capturedNode: CapturedNode | undefined): boolean {
const attrs = capturedNode?.attrs ?? {};
return (attrs.type ?? "").toLowerCase() === "password";
const type = (attrs.type ?? "").toLowerCase();
if (type === "password") return true;
const autocomplete = (attrs.autocomplete ?? "").toLowerCase();
if (autocomplete.startsWith("cc-")) return true;
if (
autocomplete === "one-time-code" ||
autocomplete === "current-password" ||
autocomplete === "new-password"
) {
return true;
}
return false;
}
interface AxNodeSignals {
@@ -400,7 +406,13 @@ const AX_TEXT_STOP_ROLES = new Set([
"columnheader",
"rowheader",
]);
const SENSITIVE_AX_INPUT_TYPES = new Set(["password", "credit-card"]);
const SENSITIVE_AX_INPUT_TYPES = new Set([
"password",
"credit-card",
"one-time-code",
"current-password",
"new-password",
]);
function axPropertyString(axNode: CdpAxNode, name: string): string | undefined {
const prop = axNode.properties?.find((item) => item.name === name);
@@ -1314,50 +1326,6 @@ export async function handleGetHtml(
}
}
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
}
interface LayoutMetricsViewportReply {
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
layoutViewport?: { clientWidth?: number; clientHeight?: number };
}
async function fallbackCapturedViewModel(
cdp: CdpRunner,
tabId: number,
): Promise<CapturedViewModel> {
let viewport = { width: 0, height: 0 };
try {
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
viewport = {
width: vpSrc.clientWidth ?? 0,
height: vpSrc.clientHeight ?? 0,
};
} catch {
// viewport stays zero-sized
}
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId);
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
}
async function captureForVom(
cdp: CdpRunner,
tabId: number,
conditionalSurfaceProbe: boolean,
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>,
): Promise<CapturedViewModel> {
try {
return await captureViewModel(cdp, tabId, {
conditionalSurfaceProbe,
hoverProbeBypassOverlay,
});
} catch {
return fallbackCapturedViewModel(cdp, tabId);
}
}
async function handleVomObservation(
manager: SessionManager,
params: SnapshotParams | ObserveParams,
@@ -1383,27 +1351,14 @@ async function handleVomObservation(
try {
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
await deps.cdp.send<unknown>(target.tabId, "Accessibility.enable", {});
const result = await deps.cdp.send<{ nodes: CdpAxNode[] }>(
target.tabId,
"Accessibility.getFullAXTree",
{},
);
const axNodes = result.nodes ?? [];
const effectiveConditionalSurfaceProbe =
deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe;
const captured = await captureForVom(
deps.cdp,
target.tabId,
effectiveConditionalSurfaceProbe,
deps.hoverProbeBypassOverlay,
);
const scene = buildVomScene(axNodes, captured, { pageUrl: target.url });
const rendered = renderVom(scene, {
const rendered = await captureVomObservation(deps.cdp, target.tabId, target.url, {
maxDepth: params.max_depth,
maxTokens: params.max_tokens,
activeRegionPolicy: true,
conditionalSurfaceProbe: effectiveConditionalSurfaceProbe,
hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay,
});
ctx.refStore.replace(
rendered.refs.map(
@@ -1418,7 +1373,7 @@ async function handleVomObservation(
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
? {
debug: {
surface_probes: (captured.surfaceProbes ?? []).map((probe) => ({
surface_probes: (rendered.surfaceProbes ?? []).map((probe) => ({
trigger_backend_node_id: probe.triggerBackendNodeId,
...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}),
trigger_action: probe.triggerAction,
+566 -96
View File
@@ -7,21 +7,53 @@ import {
isRecordQueryMessage,
isRecordStepMessage,
RECORD_CANCEL,
RECORD_CAPTURE_STATUS,
RECORD_START,
RECORD_STEP,
RECORD_STOP,
type RecordCancelMessage,
type RecordCaptureStatusMessage,
type RecordFinishMessage,
type RecordQueryResponse,
type RecordStartAck,
type RecordStartMessage,
type RecordStepPayload,
type RecordStopMessage,
} from "@/lib/record-bridge";
import {
armAllDocumentsForTab,
armDocumentCapture,
cancelDocumentCapture,
type ArmedDocument,
listInjectableFrames,
type RecordFrameSendDeps,
stopDocumentCapture,
} from "@/lib/record-frame-arming";
import { DEFAULT_MAX_PAGE_TOKENS } from "@/lib/record-constants";
import {
applyTargetMatching,
buildTraceV3,
cancelPendingSettles,
captureAndRegisterObservation,
clearPendingRedirectLanding,
createObservationState,
flushPendingRedirectLanding,
flushPendingSettles,
inferMissingPostStates,
type RecordingObservationState,
rememberStepOnPage,
scheduleDraftSettle,
scheduleRedirectLandingFlush,
settleUnsettledDrafts,
} from "@/lib/record-observation";
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording-step-buffer";
import { reduceTraceSteps, resolveTraceStartUrl } from "@/lib/trace-reducer";
import type { SessionManager } from "@/session-manager/manager";
import { EXTENSION_VERSION } from "@/transport/handshake";
import type {
DraftTraceStep,
FrameCaptureFailure,
FrameCaptureInfo,
FrameCaptureStatus,
RecordAwaitParams,
RecordAwaitResult,
RecordStartParams,
@@ -29,6 +61,7 @@ import type {
RecordStopParams,
RecordStopResult,
RpcError,
StopReason,
Trace,
} from "@/transport/types";
import { handleNavigate } from "./navigation";
@@ -58,6 +91,37 @@ interface ActiveRecording {
currentUrl?: string;
pendingNavigation: boolean;
pendingNavigationDeadline?: number;
observation: RecordingObservationState;
stoppedBy: StopReason;
maxPageTokens: number;
redactValues: boolean;
/** Documents whose capture acknowledged START and still need STOP. */
armedDocuments: Map<string, ArmedDocument>;
frameCaptureFailures: FrameCaptureFailure[];
frameCaptureStatus: FrameCaptureStatus;
/** When false, ignore subframe rearm requests during finish. */
acceptingNewFrames: boolean;
/** Rearms already sending START when finish begins. */
rearmCallbacks: Set<Promise<void>>;
/** Navigation callbacks tracked from event receipt through action enqueue. */
navigationCallbacks: Set<Promise<void>>;
/** Synchronous intake gate closed only after finish drains to stability. */
acceptingNavigation: boolean;
/**
* Serializes step appends with navigation observation so a click is always
* in `steps` before a same-turn `webNavigation` tries to annotate it.
*/
actionQueue: Promise<void>;
/**
* CDP runner captured at `record_start`. The message listeners are attached
* once at service-worker startup, where no runner exists yet, so observation
* capture must go through the recording instead of the listener deps.
*/
cdp?: CdpRunner;
}
function enqueueRecordingAction(recording: ActiveRecording, task: () => Promise<void>): void {
recording.actionQueue = recording.actionQueue.then(task, task).catch(() => {});
}
const recordings = new Map<string, ActiveRecording>();
@@ -78,6 +142,9 @@ function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
/** Recording producer version mirrored into trace.recorder.bsk. */
export const BSK_TRACE_VERSION = EXTENSION_VERSION;
/** Injectable http(s) landing page when `tool.record_start` omits `url`. */
export const RECORD_DEFAULT_START_URL = "https://example.com/";
@@ -155,16 +222,95 @@ async function sendRecordStartWithAck(
}
function buildTrace(recording: ActiveRecording): Trace {
const { pages, steps } = reduceTraceSteps(recording.steps, recording.startUrl);
const startUrl = resolveTraceStartUrl(recording.steps, recording.startUrl, pages);
return {
const frameCapture: FrameCaptureInfo | undefined =
recording.frameCaptureFailures.length > 0 || recording.frameCaptureStatus === "partial"
? {
status: recording.frameCaptureStatus,
...(recording.frameCaptureFailures.length > 0
? { failures: recording.frameCaptureFailures }
: {}),
}
: undefined;
return buildTraceV3({
obs: recording.observation,
steps: recording.steps,
startedAt: recording.startedAt,
...(recording.purpose ? { purpose: recording.purpose } : {}),
recorded_at: new Date().toISOString(),
started_at: recording.startedAt,
entry: { start_url: startUrl },
pages,
steps,
startUrl: recording.startUrl,
stoppedBy: recording.stoppedBy,
bskVersion: BSK_TRACE_VERSION,
...(frameCapture ? { frameCapture } : {}),
});
}
function recordFrameFailure(recording: ActiveRecording, failure: FrameCaptureFailure): void {
recording.frameCaptureFailures.push(failure);
recording.frameCaptureStatus = "partial";
}
function enrichRecordedStep(
recording: ActiveRecording,
step: RecordStepPayload,
): RecordStepPayload {
const enriched: RecordStepPayload = { ...step };
if (enriched.geometry && enriched.documentToken) {
const owner = recording.observation.documentTokenOwners.get(enriched.documentToken);
if (owner !== undefined) {
enriched.geometry = { ...enriched.geometry, ownerFrameBackendNodeId: owner };
}
}
if (!enriched.page_url && recording.currentUrl) {
enriched.page_url = recording.currentUrl;
}
return enriched;
}
async function publishCaptureStatus(
recording: ActiveRecording,
deps: RecordDeps,
): Promise<void> {
if (!deps.broadcastCaptureStatus) return;
const msg: RecordCaptureStatusMessage = {
type: RECORD_CAPTURE_STATUS,
status: recording.frameCaptureStatus,
...(recording.frameCaptureFailures.length > 0
? { failures: recording.frameCaptureFailures }
: {}),
};
try {
await deps.broadcastCaptureStatus(recording.tabId, msg);
} catch {
// Overlay is best-effort.
}
}
async function processRecordedStep(
recording: ActiveRecording,
deps: RecordDeps,
draftIndex: number,
): Promise<void> {
const cdp = recording.cdp ?? deps.cdp;
if (!cdp) return;
const draft = recording.steps[draftIndex];
if (!draft) return;
const stepId = draftIndex + 1;
if (draft.op === "navigate") {
draft.preStateId = recording.observation.lastSettled?.stateId;
} else {
applyTargetMatching(recording.observation, draft, stepId);
}
rememberStepOnPage(recording.observation, draft, stepId);
scheduleDraftSettle(
recording.observation,
draftIndex,
stepId,
cdp,
recording.tabId,
deps.tabsApi,
recording.steps,
);
}
export interface RecordDeps {
@@ -173,17 +319,67 @@ export interface RecordDeps {
tabId: number,
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
): Promise<unknown>;
sendToDocument?(
tabId: number,
documentId: string,
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
): Promise<unknown>;
getAllFrames?(tabId: number): Promise<
Array<{
frameId: number;
documentId?: string;
url?: string;
parentFrameId: number;
}>
>;
broadcastCaptureStatus?(
tabId: number,
msg: RecordCaptureStatusMessage,
): Promise<void>;
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
cdp?: CdpRunner;
signal?: AbortSignal;
}
function frameSendDeps(deps: RecordDeps): RecordFrameSendDeps {
return {
sendToDocument: (tabId, documentId, msg) => {
if (deps.sendToDocument) return deps.sendToDocument(tabId, documentId, msg);
return deps.sendToTab(tabId, msg);
},
getAllFrames: async (tabId) => {
if (deps.getAllFrames) return deps.getAllFrames(tabId);
if (!chrome.webNavigation?.getAllFrames) return [];
const frames = await chrome.webNavigation.getAllFrames({ tabId });
return (frames ?? []).map((frame) => ({
frameId: frame.frameId,
documentId: frame.documentId,
url: frame.url,
parentFrameId: frame.parentFrameId,
}));
},
};
}
let defaultDeps: RecordDeps | null = null;
function getDefaultDeps(): RecordDeps {
if (!defaultDeps) {
defaultDeps = {
tabsApi: chromeTabsApi,
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
sendToDocument: (tabId, documentId, msg) =>
chrome.tabs.sendMessage(tabId, msg, { documentId }),
getAllFrames: async (tabId) => {
if (!chrome.webNavigation?.getAllFrames) return [];
const frames = await chrome.webNavigation.getAllFrames({ tabId });
return (frames ?? []).map((frame) => ({
frameId: frame.frameId,
documentId: frame.documentId,
url: frame.url,
parentFrameId: frame.parentFrameId,
}));
},
broadcastCaptureStatus: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg, { frameId: 0 }),
};
}
return defaultDeps;
@@ -241,16 +437,29 @@ export function releaseBrowserObservationListenersIfIdle(): void {
detachBrowserObservation = null;
}
export function attachRecordStepListener(): () => void {
export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): () => void {
const listener = (
message: unknown,
_sender: chrome.runtime.MessageSender,
sender: chrome.runtime.MessageSender,
_sendResponse: () => void,
) => {
if (!isRecordStepMessage(message)) return false;
for (const recording of recordings.values()) {
if (recording.requestId !== message.requestId) continue;
appendRecordedPayload(recording, message.step);
enqueueRecordingAction(recording, async () => {
const cdp = recording.cdp ?? deps.cdp;
await flushPendingRedirectLanding(
recording.observation,
recording.steps,
cdp,
recording.tabId,
deps.tabsApi,
{ cancelled: () => recording.settled },
);
const enriched = enrichRecordedStep(recording, message.step);
const draftIndex = appendRecordedPayload(recording, enriched);
if (draftIndex !== null) await processRecordedStep(recording, deps, draftIndex);
});
return false;
}
return false;
@@ -277,7 +486,11 @@ export function attachRecordFinishListener(deps: RecordDeps = getDefaultDeps()):
function findRecordingByTabId(tabId: number): ActiveRecording | null {
for (const recording of recordings.values()) {
if (recording.tabId === tabId && !recording.settled) return recording;
if (recording.settled) continue;
if (recording.tabId === tabId) return recording;
for (const armed of recording.armedDocuments.values()) {
if (armed.tabId === tabId) return recording;
}
}
return null;
}
@@ -325,71 +538,135 @@ async function clearRearmTimersForRecording(
}
}
async function stopRecordingOnAllAgentTabs(
async function stopRecordingOnAllDocuments(
recording: ActiveRecording,
deps: RecordDeps,
): Promise<void> {
recording.acceptingNewFrames = false;
const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
let tabIds = [recording.tabId];
try {
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
tabIds = [
...new Set([
recording.tabId,
...tabs.flatMap((tab) => (typeof tab.id === "number" ? [tab.id] : [])),
]),
];
} catch {
// Fall back to the current recording tab.
}
for (const tabId of tabIds) {
try {
const response = await deps.sendToTab(tabId, stopMsg);
if (tabId === recording.tabId && !isRecordStartAck(response)) {
throw new Error("content script did not confirm recorded steps");
}
} catch {
if (tabId === recording.tabId) {
throw new Error("failed to flush recorded steps");
}
const frameDeps = frameSendDeps(deps);
const armed = [...recording.armedDocuments.values()];
for (const doc of armed) {
const result = await stopDocumentCapture(doc, stopMsg, frameDeps);
if ("ok" in result) {
recording.armedDocuments.delete(doc.documentId);
continue;
}
if (deps.bypassOverlay) {
try {
await deps.bypassOverlay(tabId, false);
} catch {
// Best-effort cleanup.
}
recordFrameFailure(recording, result);
}
if (recording.armedDocuments.size > 0) {
for (const doc of [...recording.armedDocuments.values()]) {
recordFrameFailure(
recording,
{
reason: "flush_failed",
frame_id: doc.frameId,
document_id: doc.documentId,
url: doc.url,
detail: "document still armed after STOP",
},
);
}
recording.armedDocuments.clear();
}
await publishCaptureStatus(recording, deps);
if (deps.bypassOverlay) {
try {
await deps.bypassOverlay(recording.tabId, false);
} catch {
// Best-effort cleanup.
}
}
}
async function rearmRecording(
async function cancelRecordingOnAllDocuments(
recording: ActiveRecording,
deps: RecordDeps,
): Promise<void> {
const cancelMsg: RecordCancelMessage = { type: RECORD_CANCEL, requestId: recording.requestId };
const frameDeps = frameSendDeps(deps);
for (const doc of [...recording.armedDocuments.values()]) {
await cancelDocumentCapture(doc, cancelMsg, frameDeps);
}
recording.armedDocuments.clear();
}
async function rearmRecordingDocuments(
recording: ActiveRecording,
targetTabId: number,
deps: RecordDeps,
targetDocumentId?: string,
): Promise<boolean> {
// Do NOT toggle automation-bypass here: each retry used to increment the
// content-script counter, and a single stop decrement left the ControlOverlay
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
// already hides the control chrome while activeRecord is set.
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
const startMsg: RecordStartMessage = {
type: RECORD_START,
requestId: recording.requestId,
startedAtMs: recording.startedAtMs,
};
try {
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
recording.tabId = targetTabId;
return true;
} catch {
let resolveTracked!: () => void;
const tracked = new Promise<void>((resolve) => {
resolveTracked = resolve;
});
recording.rearmCallbacks.add(tracked);
const frameDeps = frameSendDeps(deps);
try {
const attemptLimit = targetDocumentId ? 1 : RECORD_REARM_MAX_ATTEMPTS;
for (let attempt = 0; attempt < attemptLimit; attempt += 1) {
if (recording.settled || recording.finishing || !recording.acceptingNewFrames) return false;
const { injectable, failures } = await listInjectableFrames(targetTabId, frameDeps);
for (const failure of failures) recordFrameFailure(recording, failure);
const targets = targetDocumentId
? injectable.filter((frame) => frame.documentId === targetDocumentId)
: injectable.filter((frame) => !recording.armedDocuments.has(frame.documentId ?? ""));
if (targets.length === 0) {
if (targetDocumentId) {
recordFrameFailure(recording, {
reason: "rearm_failed",
frame_id: 0,
document_id: targetDocumentId,
detail: "document not injectable",
});
await publishCaptureStatus(recording, deps);
return false;
}
recording.tabId = targetTabId;
return injectable.some((frame) => recording.armedDocuments.has(frame.documentId ?? ""));
}
let armedAny = false;
for (const frame of targets) {
const startMsg: RecordStartMessage = {
type: RECORD_START,
requestId: recording.requestId,
startedAtMs: recording.startedAtMs,
showOverlay: frame.frameId === 0,
frameMode: frame.frameId === 0 ? "top" : "child",
};
const result = await armDocumentCapture(targetTabId, frame, startMsg, frameDeps);
if (result.armed) {
recording.armedDocuments.set(result.armed.documentId, result.armed);
armedAny = true;
}
if (result.failure) recordFrameFailure(recording, result.failure);
}
if (armedAny) {
recording.tabId = targetTabId;
await publishCaptureStatus(recording, deps);
if (deps.cdp && recording.acceptingNewFrames) {
void captureAndRegisterObservation(
recording.observation,
deps.cdp,
targetTabId,
deps.tabsApi,
).catch(() => {});
}
return true;
}
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
await sleep(RECORD_REARM_RETRY_DELAY_MS);
}
}
await publishCaptureStatus(recording, deps);
return false;
} finally {
recording.rearmCallbacks.delete(tracked);
resolveTracked();
}
return false;
}
function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
@@ -401,7 +678,7 @@ function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
rearmTimers.delete(tabId);
void (async () => {
const current = await findRecordingForTab(tabId, deps);
if (current) await rearmRecording(current, tabId, deps);
if (current) await rearmRecordingDocuments(current, tabId, deps);
})();
}, RECORD_REARM_DEBOUNCE_MS),
);
@@ -432,11 +709,86 @@ export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): ()
}
export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps()): () => void {
const observeMainFrame = (tabId: number, url?: string, causedByAction?: boolean) => {
const observeMainFrame = (
tabId: number,
url?: string,
causedByAction?: boolean,
transitionType?: string,
transitionQualifiers?: string[],
) => {
if (!url) return;
const direct = findRecordingByTabId(tabId);
const candidates = direct
? direct.acceptingNavigation
? [direct]
: []
: [...recordings.values()].filter(
(recording) => !recording.settled && recording.acceptingNavigation,
);
if (candidates.length === 0) return;
let resolveTracked!: () => void;
const tracked = new Promise<void>((resolve) => {
resolveTracked = resolve;
});
for (const candidate of candidates) candidate.navigationCallbacks.add(tracked);
void (async () => {
const recording = await findRecordingForTab(tabId, deps);
if (recording && url) {
observeRecordedNavigation(recording, url, causedByAction);
try {
const recording = await findRecordingForTab(tabId, deps);
if (!recording || !recording.acceptingNavigation || !candidates.includes(recording)) {
return;
}
enqueueRecordingAction(recording, async () => {
const result = observeRecordedNavigation(
recording,
url,
causedByAction,
transitionType,
transitionQualifiers,
);
const cdp = recording.cdp ?? deps.cdp;
if (!cdp) return;
if (result.kind === "coalesce_redirect") {
scheduleRedirectLandingFlush(
recording.observation,
recording.steps,
cdp,
tabId,
deps.tabsApi,
result.url,
{ cancelled: () => recording.settled },
);
return;
}
if (result.kind === "noop") return;
// A concrete navigation supersedes any in-flight redirect coalesce.
clearPendingRedirectLanding(recording.observation);
const draftIndex = result.index;
const lastDraft = recording.steps[draftIndex];
if (!lastDraft) return;
if (result.kind === "appended") {
lastDraft.preStateId = recording.observation.lastSettled?.stateId;
rememberStepOnPage(recording.observation, lastDraft, draftIndex + 1);
}
scheduleDraftSettle(
recording.observation,
draftIndex,
draftIndex + 1,
cdp,
tabId,
deps.tabsApi,
recording.steps,
);
});
} finally {
for (const candidate of candidates) candidate.navigationCallbacks.delete(tracked);
resolveTracked();
}
})();
};
@@ -447,29 +799,36 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
})();
};
const onSubFrameComplete = (tabId: number, documentId?: string) => {
if (!documentId) return;
void (async () => {
const recording = await findRecordingForTab(tabId, deps);
if (!recording || !recording.acceptingNewFrames) return;
await rearmRecordingDocuments(recording, tabId, deps, documentId);
})();
};
if (chrome.webNavigation?.onCompleted) {
const completedListener = (
details: chrome.webNavigation.WebNavigationFramedCallbackDetails,
) => {
if (details.frameId !== 0) return;
onMainFrameComplete(details.tabId, details.url);
if (details.frameId === 0) {
onMainFrameComplete(details.tabId, details.url);
return;
}
onSubFrameComplete(details.tabId, details.documentId);
};
const committedListener = (
details: chrome.webNavigation.WebNavigationTransitionCallbackDetails,
) => {
if (details.frameId !== 0) return;
const causedByAction =
details.transitionType === "link" || details.transitionType === "form_submit"
? true
: details.transitionType === "typed" ||
details.transitionType === "auto_bookmark" ||
details.transitionType === "generated" ||
details.transitionType === "keyword" ||
details.transitionType === "keyword_generated" ||
details.transitionType === "reload"
? false
: undefined;
observeMainFrame(details.tabId, details.url, causedByAction);
observeMainFrame(
details.tabId,
details.url,
undefined,
details.transitionType,
details.transitionQualifiers,
);
};
chrome.webNavigation.onCompleted.addListener(completedListener);
chrome.webNavigation.onCommitted?.addListener(committedListener);
@@ -487,6 +846,46 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
return () => chrome.tabs.onUpdated.removeListener(listener);
}
const MAX_FINISH_DRAIN_ROUNDS = 10;
async function drainRecordingToStability(
recording: ActiveRecording,
deps: RecordDeps,
): Promise<boolean> {
const cdp = recording.cdp ?? deps.cdp;
for (let round = 0; round < MAX_FINISH_DRAIN_ROUNDS; round += 1) {
await Promise.all([...recording.navigationCallbacks]);
const actionTail = recording.actionQueue;
await actionTail;
await flushPendingRedirectLanding(
recording.observation,
recording.steps,
cdp,
recording.tabId,
deps.tabsApi,
{ cancelled: () => recording.settled },
);
await flushPendingSettles(recording.observation);
const redirectTail = recording.observation.redirectFlushQueue;
const settleTail = recording.observation.settleQueue;
await Promise.all([redirectTail, settleTail]);
if (
recording.navigationCallbacks.size === 0 &&
recording.actionQueue === actionTail &&
recording.observation.redirectFlushQueue === redirectTail &&
recording.observation.settleQueue === settleTail
) {
// No event or promise continuation can interleave with this synchronous
// check-and-close, so work accepted before the cutoff is fully drained.
recording.acceptingNavigation = false;
return true;
}
}
console.warn("[bsk record] navigation/action queues did not stabilize at stop");
return false;
}
export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()): () => void {
const listener = (
message: unknown,
@@ -505,11 +904,15 @@ export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()):
sendResponse({ active: false });
return;
}
await rearmRecording(recording, tabId, deps);
await rearmRecordingDocuments(recording, tabId, deps, sender.documentId);
sendResponse({
active: true,
requestId: recording.requestId,
startedAtMs: recording.startedAtMs,
captureStatus: recording.frameCaptureStatus,
...(recording.frameCaptureFailures.length > 0
? { captureFailures: recording.frameCaptureFailures }
: {}),
});
})();
return true;
@@ -527,23 +930,46 @@ async function finishRecordingByRequest(
if (recording.requestId !== requestId || recording.settled) continue;
const match = await findRecordingForTab(tabId, deps);
if (match !== recording) continue;
await finishRecording(sessionId, deps);
await finishRecording(sessionId, deps, "user_finish");
return;
}
}
async function finishRecording(sessionId: string, deps: RecordDeps): Promise<Trace | null> {
async function finishRecording(
sessionId: string,
deps: RecordDeps,
stoppedBy: StopReason,
): Promise<Trace | null> {
const recording = recordings.get(sessionId);
if (!recording || recording.settled || recording.finishing) return null;
if (!recording || recording.settled) return null;
if (recording.finishing) return recording.finishPromise;
recording.finishing = true;
recording.stoppedBy = stoppedBy;
await clearRearmTimersForRecording(recording, deps);
await Promise.all([...recording.rearmCallbacks]);
try {
await stopRecordingOnAllAgentTabs(recording, deps);
await stopRecordingOnAllDocuments(recording, deps);
} catch {
recording.finishing = false;
return null;
}
if (!(await drainRecordingToStability(recording, deps))) {
recording.finishing = false;
return null;
}
cancelPendingSettles(recording.observation);
inferMissingPostStates(recording.steps);
const cdp = recording.cdp ?? deps.cdp;
if (cdp) {
await settleUnsettledDrafts(
recording.observation,
cdp,
recording.tabId,
deps.tabsApi,
recording.steps,
);
}
recording.settled = true;
recordings.delete(sessionId);
@@ -582,6 +1008,8 @@ export async function handleRecordStart(
});
const navigateUrl = params.url ?? RECORD_DEFAULT_START_URL;
const startedAtMs = Date.now();
const maxPageTokens = params.max_page_tokens;
const redactValues = params.redact_values ?? false;
recordings.set(params.session_id, {
requestId,
tabId: target.tabId,
@@ -599,23 +1027,30 @@ export async function handleRecordStart(
currentUrl: navigateUrl,
pendingNavigation: false,
pendingNavigationDeadline: undefined,
observation: createObservationState({ maxPageTokens, redactValues }),
stoppedBy: "user_finish",
maxPageTokens: maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS,
redactValues,
armedDocuments: new Map(),
frameCaptureFailures: [],
frameCaptureStatus: "complete",
acceptingNewFrames: true,
rearmCallbacks: new Set(),
navigationCallbacks: new Set(),
acceptingNavigation: true,
actionQueue: Promise.resolve(),
...(deps.cdp ? { cdp: deps.cdp } : {}),
});
// Observe navigations for the whole recording lifetime; attach before
// optional navigate so the destination load can rearm capture.
ensureBrowserObservationListeners(deps);
const abortPending = async (notifyContent: boolean) => {
const pending = recordings.get(params.session_id);
recordings.delete(params.session_id);
releaseBrowserObservationListenersIfIdle();
if (notifyContent) {
try {
await deps.sendToTab(target.tabId, {
type: RECORD_CANCEL,
requestId,
});
} catch {
// Content may never have received RECORD_START.
}
if (notifyContent && pending) {
await cancelRecordingOnAllDocuments(pending, deps);
}
if (deps.bypassOverlay) {
try {
@@ -718,10 +1153,28 @@ export async function handleRecordStart(
if (cancelled) return cancelled;
}
const startMsg: RecordStartMessage = { type: RECORD_START, requestId, startedAtMs };
const startMsg: RecordStartMessage = {
type: RECORD_START,
requestId,
startedAtMs,
showOverlay: true,
frameMode: "top",
};
try {
await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
const frameDeps = frameSendDeps(deps);
const { armed, failures } = await armAllDocumentsForTab(target.tabId, startMsg, frameDeps);
for (const failure of failures) recordFrameFailure(active, failure);
for (const doc of armed) active.armedDocuments.set(doc.documentId, doc);
if (active.armedDocuments.size === 0) {
await abortPending(true);
return {
code: "protocol_error",
message:
"failed to start recording in any frame — reload the BrowserSkill extension, then retry",
};
}
await publishCaptureStatus(active, deps);
} catch {
await abortPending(true);
return {
@@ -731,6 +1184,23 @@ export async function handleRecordStart(
};
}
if (deps.cdp) {
const activeRecording = recordings.get(params.session_id);
if (activeRecording) {
try {
await captureAndRegisterObservation(
activeRecording.observation,
deps.cdp,
target.tabId,
deps.tabsApi,
startUrl,
);
} catch {
// Proceed without initial observation; steps may be dropped by reducer.
}
}
}
{
const cancelled = await abortIfCancelled(true);
if (cancelled) return cancelled;
@@ -755,7 +1225,7 @@ export async function handleRecordStop(
};
}
const trace = await finishRecording(params.session_id, deps);
const trace = await finishRecording(params.session_id, deps, "cli_stop");
if (!trace) {
return {
code: "protocol_error",
@@ -677,7 +677,7 @@ describe("captureViewModel", () => {
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
});
it("subtracts scroll offset to make rects viewport-relative", async () => {
it("preserves document rects while deriving viewport rects after scroll", async () => {
const cdp = {
send: vi.fn(async (_t: number, method: string) => {
if (method === "DOMSnapshot.enable") return {};
@@ -691,11 +691,10 @@ describe("captureViewModel", () => {
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
};
const { nodes } = await captureViewModel(cdp, 4);
const div = nodes.find((n) => n.backendNodeId === 12);
// local rect preserves viewport-relative scroll math.
expect(div?.localRect?.y).toBe(-200);
// exported rect is clipped to the top-level viewport before VOM consumes it.
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
const input = nodes.find((n) => n.backendNodeId === 13);
expect(input?.documentRect).toEqual({ x: 400, y: 300, w: 200, h: 40 });
expect(input?.localRect).toEqual({ x: 400, y: 100, w: 200, h: 40 });
expect(input?.rect).toEqual({ x: 400, y: 100, w: 200, h: 40 });
});
it("normalizes device-pixel bounds by devicePixelRatio", async () => {
+13 -3
View File
@@ -25,6 +25,8 @@ export interface CapturedNode {
rect: Rect | null;
/** Frame-local viewport-relative CSS px before top-level translation/clipping. */
localRect?: Rect | null;
/** Frame-local document-relative CSS px as reported by DOMSnapshot. */
documentRect?: Rect | null;
paintOrder: number;
position: string;
pointerEvents: string;
@@ -788,6 +790,7 @@ function parseDocumentNodes(
let rect: Rect | null = null;
let localRect: Rect | null = null;
let documentRect: Rect | null = null;
let paintOrder = 0;
let position = "static";
let pointerEvents = "auto";
@@ -796,12 +799,18 @@ function parseDocumentNodes(
if (li !== undefined) {
const b = dl?.bounds?.[li];
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
localRect = {
x: b[0] / dpr - context.scrollX,
y: b[1] / dpr - context.scrollY,
documentRect = {
x: b[0] / dpr,
y: b[1] / dpr,
w: b[2] / dpr,
h: b[3] / dpr,
};
localRect = {
x: documentRect.x - context.scrollX,
y: documentRect.y - context.scrollY,
w: documentRect.w,
h: documentRect.h,
};
rect = rectInTop(localRect, context);
}
paintOrder = dl?.paintOrders?.[li] ?? 0;
@@ -825,6 +834,7 @@ function parseDocumentNodes(
attrs,
rect,
localRect,
documentRect,
paintOrder,
position,
pointerEvents,
@@ -65,6 +65,11 @@ function deferredFakeTransport(): { transport: Transport; emit: (frame: Protocol
}
describe("performHandshake", () => {
it("advertises the Trace v3 compatibility boundary", () => {
expect(PROTOCOL_VERSION).toBe("1.1");
expect(MIN_COMPATIBLE_PROTOCOL).toBe("1.1");
});
it("sends system.handshake with identity and both compat fields", async () => {
let sentFrame: ProtocolFrame | null = null;
const transport = fakeTransport((req) => {
@@ -74,9 +79,9 @@ describe("performHandshake", () => {
result: {
server: "browser-skill-daemon",
version: "0.1.0",
protocol_version: "1.0",
protocol_version: "1.1",
min_compatible_peer: "0.0.0",
min_compatible_protocol: "1.0",
min_compatible_protocol: "1.1",
},
};
});
@@ -102,7 +107,7 @@ describe("performHandshake", () => {
},
});
expect(outcome.result.min_compatible_peer).toBe("0.0.0");
expect(outcome.result.min_compatible_protocol).toBe("1.0");
expect(outcome.result.min_compatible_protocol).toBe("1.1");
});
it("accepts legacy daemon reply with only min_compatible_peer", async () => {
@@ -129,8 +134,8 @@ describe("performHandshake", () => {
const response = {
server: "browser-skill-daemon",
version: "0.1.0",
protocol_version: "1.0",
min_compatible_protocol: "1.0",
protocol_version: "1.1",
min_compatible_protocol: "1.1",
} satisfies HandshakeResult;
const transport = fakeTransport((req) => ({
id: (req as { id: string }).id,
@@ -145,7 +150,7 @@ describe("performHandshake", () => {
});
expect(outcome.result.min_compatible_peer).toBeUndefined();
expect(outcome.result.min_compatible_protocol).toBe("1.0");
expect(outcome.result.min_compatible_protocol).toBe("1.1");
});
it("rejects when the daemon responds with an error", async () => {
@@ -178,9 +183,9 @@ describe("performHandshake", () => {
result: {
server: "browser-skill-daemon",
version: "0.1.0",
protocol_version: "1.0",
protocol_version: "1.1",
min_compatible_peer: "0.0.0",
min_compatible_protocol: "1.0",
min_compatible_protocol: "1.1",
},
});
emit({
@@ -188,14 +193,14 @@ describe("performHandshake", () => {
result: {
server: "browser-skill-daemon",
version: "0.1.0",
protocol_version: "1.0",
protocol_version: "1.1",
min_compatible_peer: "0.0.0",
min_compatible_protocol: "1.0",
min_compatible_protocol: "1.1",
},
});
await expect(pending).resolves.toMatchObject({
result: { server: "browser-skill-daemon", protocol_version: "1.0" },
result: { server: "browser-skill-daemon", protocol_version: "1.1" },
});
});
+2 -2
View File
@@ -7,7 +7,7 @@ import type {
ResponseFrame,
} from "./types";
export const PROTOCOL_VERSION = "1.0";
export const PROTOCOL_VERSION = "1.1";
/**
* Extension semver, injected at build time from `package.json` via
* Vite's `define` (see `wxt.config.ts` and `vitest.config.ts`).
@@ -18,7 +18,7 @@ export const EXTENSION_VERSION: string =
* Lowest **protocol** version this extension accepts (e.g. `"1.0"`).
* Must stay in sync with daemon `MIN_COMPATIBLE_PROTOCOL`.
*/
export const MIN_COMPATIBLE_PROTOCOL = "1.0";
export const MIN_COMPATIBLE_PROTOCOL = "1.1";
/**
* **Deprecated** — legacy app-semver floor for wire compat with old
* daemons. New code sends `"0.0.0"`; compat decisions ignore this.
+149 -19
View File
@@ -3,6 +3,8 @@
// stay structural (interface, not class) so the same JSON parses on both
// sides without extra adapters.
import type { CaptureTargetDescriptor } from "@/lib/describe-target";
export type RpcId = string;
export type ErrorCode =
@@ -656,26 +658,72 @@ export interface EmulateResult {
}
// --------------------------------------------------------------------------
// Semantic record payloads — mirror bsk-protocol record.rs (Trace v2)
// Semantic record payloads — mirror bsk-protocol record.rs (Trace v3)
// --------------------------------------------------------------------------
export const TRACE_VERSION = 3;
export const VOM_FORMAT_VERSION = 1;
export interface TargetDescriptor {
ref?: string;
role?: string;
name?: string;
tag: string;
name_attr?: string;
placeholder?: string;
nearby_label?: string;
ctx?: string;
unmatched?: boolean;
}
export interface TraceEntry {
start_url: string;
}
export interface PageRef {
export interface RecorderInfo {
bsk: string;
vom: number;
}
export type FrameCaptureStatus = "complete" | "partial";
export type FrameCaptureFailureReason =
| "not_injectable"
| "arm_failed"
| "rearm_failed"
| "drain_failed"
| "flush_failed";
export interface FrameCaptureFailure {
reason: FrameCaptureFailureReason;
frame_id: number;
document_id?: string;
url?: string;
detail?: string;
}
export interface FrameCaptureInfo {
status: FrameCaptureStatus;
failures?: FrameCaptureFailure[];
}
export type StopReason = "user_finish" | "cli_stop";
export interface TraceState {
id: string;
url: string;
title?: string;
/** Wire-only: full page observation text. */
body?: string;
/** Disk-only: filename under the bundle `pages/` directory. */
page?: string;
truncated?: boolean;
}
export interface StepResult {
state: string;
}
export interface StepCommon {
id: number;
state: string;
result: StepResult;
}
export interface SelectedOption {
@@ -683,66 +731,140 @@ export interface SelectedOption {
label?: string;
}
export interface StepEffect {
navigated_to: string;
export type NavigationCause =
| "user_typed"
| "link"
| "form_submit"
| "reload"
| "history"
| "script"
| "browser";
export type FillCommit = "enter" | "suggestion" | "blur";
/** Geometry reported by the content script at capture time. */
export interface CaptureGeometry {
rect: { x: number; y: number; w: number; h: number };
scrollX: number;
scrollY: number;
position: string;
tag: string;
ownerFrameBackendNodeId?: number | null;
}
export interface StepCommon {
id: number;
page: string;
effect?: StepEffect;
}
/** Capture/buffer draft before v2 reduction. */
/** Capture/buffer draft before v3 reduction. */
export type DraftTraceStep =
| {
op: "click";
target: TargetDescriptor;
captureTarget?: CaptureTargetDescriptor;
geometry?: CaptureGeometry;
navigated_to?: string;
page_url?: string;
title?: string;
preStateId?: string;
postStateId?: string;
observationBody?: string;
observationHash?: string;
truncated?: boolean;
}
| {
op: "hover";
target: TargetDescriptor;
captureTarget?: CaptureTargetDescriptor;
geometry?: CaptureGeometry;
page_url?: string;
title?: string;
preStateId?: string;
postStateId?: string;
observationBody?: string;
observationHash?: string;
truncated?: boolean;
}
| {
op: "fill";
target: TargetDescriptor;
captureTarget?: CaptureTargetDescriptor;
geometry?: CaptureGeometry;
value: string;
commit?: FillCommit;
redacted?: boolean;
navigated_to?: string;
page_url?: string;
title?: string;
preStateId?: string;
postStateId?: string;
observationBody?: string;
observationHash?: string;
truncated?: boolean;
}
| {
op: "press";
key: string;
target?: TargetDescriptor;
captureTarget?: CaptureTargetDescriptor;
geometry?: CaptureGeometry;
modifiers?: KeyModifier[];
navigated_to?: string;
page_url?: string;
title?: string;
preStateId?: string;
postStateId?: string;
observationBody?: string;
observationHash?: string;
truncated?: boolean;
}
| {
op: "select";
target: TargetDescriptor;
captureTarget?: CaptureTargetDescriptor;
geometry?: CaptureGeometry;
values: string[];
labels?: string[];
navigated_to?: string;
page_url?: string;
title?: string;
preStateId?: string;
postStateId?: string;
observationBody?: string;
observationHash?: string;
truncated?: boolean;
}
| {
op: "scroll";
page_url?: string;
title?: string;
preStateId?: string;
postStateId?: string;
observationBody?: string;
observationHash?: string;
truncated?: boolean;
}
| {
op: "navigate";
url: string;
cause?: NavigationCause;
page_url?: string;
title?: string;
preStateId?: string;
postStateId?: string;
observationBody?: string;
observationHash?: string;
truncated?: boolean;
/** Raw webNavigation transition metadata for cause mapping. */
transitionType?: string;
transitionQualifiers?: string[];
};
/** Exported record-only step (trace v2). */
/** Exported record-only step (trace v3). */
export type Step =
| ({ op: "navigate" } & StepCommon & { to: string })
| ({ op: "navigate" } & StepCommon & { to: string; cause: NavigationCause })
| ({ op: "click" } & StepCommon & { target: TargetDescriptor })
| ({ op: "hover" } & StepCommon & { target: TargetDescriptor })
| ({ op: "fill" } & StepCommon & {
target: TargetDescriptor;
value: string;
commit: FillCommit;
redacted?: boolean;
})
| ({ op: "select" } & StepCommon & {
@@ -753,14 +875,20 @@ export type Step =
key: string;
modifiers?: KeyModifier[];
target?: TargetDescriptor;
});
})
| ({ op: "scroll" } & StepCommon);
export interface Trace {
version: number;
recorded_at: string;
started_at?: string;
purpose?: string;
stopped_by: StopReason;
entry: TraceEntry;
pages: PageRef[];
recorder: RecorderInfo;
/** Present when one or more frames could not be armed, rearmed, or flushed. */
frame_capture?: FrameCaptureInfo;
states: TraceState[];
steps: Step[];
}
@@ -769,6 +897,8 @@ export interface RecordStartParams {
tab_id?: number;
url?: string;
purpose?: string;
max_page_tokens?: number;
redact_values?: boolean;
}
export interface RecordStartResult {
+14 -6
View File
@@ -265,19 +265,27 @@ retry — complete the task autonomously or stop gracefully.
### Recording — `bsk record`
Capture the user's own actions in the Agent Window to a `trace.json`, for later LLM-driven automation:
Capture the user's own actions in the Agent Window to a **trace v3 bundle**, for later LLM-driven automation:
```bash
bsk record start --browser <instance-id-or-label> [--url https://…] [--purpose "publish a wiki doc"] [--output trace.json]
bsk record start --browser <instance-id-or-label> \
[--url https://…] [--purpose "publish a wiki doc"] \
[--max-page-tokens 3000] [--redact-values] \
[--output trace]
# `--url` is optional; default https://example.com/ when omitted (must be http(s)).
# Blocks until the user clicks Finish in the recording panel, then writes ./trace.json and closes the window.
# Blocks until the user clicks Finish in the recording panel, then writes:
# trace/trace.json — action chain + state index (version 3)
# trace/pages/ — one `.vom.txt` observe snapshot per settled page state
bsk record stop [--output trace.json] # terminal fallback if the browser panel is unavailable
bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable
```
- The trace is a **record-only action log** (a `pages[]` dictionary + `navigate`/`click`/`fill`/`select`/`press` steps with `target` descriptors). It records *what the user did*; deciding which inputs are variable is left to the executing agent.
- **Bundle layout:** `--output` is a directory (default `./trace`) containing `trace.json` and `pages/`. `trace.json` lists `states[]` (page observation ids) and `steps[]` (each step binds `state` = observe snapshot *before* the action and `result.state` = snapshot *after* settle). Page bodies live in `pages/sN.vom.txt` using the same VOM format as `bsk observe`.
- Each `states[]` entry is one **settled page observation** (captured after recording start, navigation landing, or action settle — not on a timer).
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
- `--purpose` is optional context metadata; it does **not** change what gets captured.
- There is **no** `bsk replay` — to redo a flow, read the trace and reuse the existing `session` / `snapshot` / `@eN` / `click` / `fill` tools. Follow **Stop when the goal is met**.
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
- Recording captures interactions in the **top frame and injectable child frames** (same-origin and cross-origin where the extension can attach). Embedded frames that cannot be injected are omitted; the trace may include `frame_capture.status: "partial"` with failure details.
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
## Error handling
+5 -5
View File
@@ -480,7 +480,7 @@ mod m2_tests {
fn fake_status(browsers: Vec<BrowserStatusEntry>, skew: Vec<VersionSkewEntry>) -> StatusResult {
StatusResult {
daemon_version: env!("CARGO_PKG_VERSION").into(),
protocol_version: "1.0".into(),
protocol_version: "1.1".into(),
pid: 1,
uptime_secs: 0,
ws_port: 0,
@@ -545,7 +545,7 @@ mod m2_tests {
session_count: 0,
connected_at_ms: 1,
version_skew: false,
extension_protocol_version: "1.0".into(),
extension_protocol_version: "1.1".into(),
}],
Vec::new(),
);
@@ -566,7 +566,7 @@ mod m2_tests {
session_count: 0,
connected_at_ms: 1,
version_skew: true,
extension_protocol_version: "1.1".into(),
extension_protocol_version: "1.2".into(),
}],
vec![VersionSkewEntry {
instance_id: "alpha".into(),
@@ -574,8 +574,8 @@ mod m2_tests {
label: "Personal".into(),
server_version: env!("CARGO_PKG_VERSION").into(),
client_version: "0.0.9".into(),
server_protocol_version: "1.0".into(),
client_protocol_version: "1.1".into(),
server_protocol_version: "1.1".into(),
client_protocol_version: "1.2".into(),
}],
);
let check = check_browsers_protocol_compatible(Some(&status));
+604 -50
View File
@@ -1,15 +1,17 @@
//! `bsk record start|stop` — capture user actions in the Agent Window.
use std::fs;
use std::path::PathBuf;
use std::collections::HashSet;
use std::fs::{self, OpenOptions};
use std::io;
use std::path::{Path, PathBuf};
use std::time::Duration;
use anyhow::Context;
use bsk_protocol::Method;
use bsk_protocol::tools::{
RecordAwaitParams, RecordAwaitResult, RecordStartParams, RecordStartResult, RecordStopParams,
RecordStopResult, Trace,
FrameCaptureStatus, RecordAwaitParams, RecordAwaitResult, RecordStartParams, RecordStartResult,
RecordStopParams, RecordStopResult, TRACE_VERSION, Trace, TraceState,
};
use bsk_protocol::{ErrorCode, Method};
use clap::{Args, Subcommand};
use crate::cli::TOOL_IPC_TIMEOUT;
@@ -18,10 +20,34 @@ use crate::cli::ensure_daemon::ensure_daemon;
use crate::cli::error::{CliError, Format};
use crate::cli::record_state;
use crate::cli::session::{SessionStartOptions, start_session, stop_session};
use crate::daemon::ipc::STOP_IN_PROGRESS_REASON;
/// Max wait for the user to click 结束 in the browser (24 hours).
const RECORD_AWAIT_TIMEOUT_MS: u32 = 86_400_000;
/// Tear down the recording session, tolerating the teardown the other half of
/// the pair already started. `record start` blocks in `record_await` and stops
/// the session once the trace arrives; the documented `record stop` fallback
/// stops it too. Whichever loses that race must not report a failure — the
/// trace is already written by then.
fn stop_recording_session(sock: PathBuf, session_id: &str) -> Result<(), CliError> {
match stop_session(sock, session_id) {
Ok(_) => Ok(()),
Err(err) if session_teardown_raced(&err) => Ok(()),
Err(err) => Err(err),
}
}
fn session_teardown_raced(err: &CliError) -> bool {
if err.code() == Some(ErrorCode::NotFound) {
return true;
}
err.data()
.and_then(|data| data.get("reason"))
.and_then(serde_json::Value::as_str)
== Some(STOP_IN_PROGRESS_REASON)
}
/// `bsk record …` subcommand tree.
#[derive(Debug, Clone, Args)]
pub struct RecordCmd {
@@ -58,15 +84,25 @@ pub struct RecordStartArgs {
#[arg(long)]
pub purpose: Option<String>,
/// Output path for the trace JSON (default `./trace.json`).
#[arg(long, default_value = "trace.json")]
/// Max VOM tokens per page observation file (default 3000).
#[arg(long = "max-page-tokens")]
pub max_page_tokens: Option<u32>,
/// Redact all form values in page observations (`[filled]` only).
#[arg(long = "redact-values")]
pub redact_values: bool,
/// Output directory for the trace bundle (default `./trace`).
/// Writes `<dir>/trace.json` and `<dir>/pages/*.vom.txt`.
#[arg(long, default_value = "trace")]
pub output: PathBuf,
}
#[derive(Debug, Clone, Args)]
pub struct RecordStopArgs {
/// Output path for the trace JSON (default `./trace.json`).
#[arg(long, default_value = "trace.json")]
/// Output directory for the trace bundle (default `./trace`).
/// Writes `<dir>/trace.json` and `<dir>/pages/*.vom.txt`.
#[arg(long, default_value = "trace")]
pub output: PathBuf,
}
@@ -98,6 +134,8 @@ fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError>
tab_id: args.tab_id,
url: args.url,
purpose: args.purpose.clone(),
max_page_tokens: args.max_page_tokens,
redact_values: Some(args.redact_values),
};
let start_result = business_rpc::call::<RecordStartParams, RecordStartResult>(
info.sock_path.clone(),
@@ -142,13 +180,13 @@ fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError>
// Keep write/render inside a Result so `?` cannot skip session teardown.
let run_result: Result<(), CliError> = match await_result {
Ok(result) => (|| {
write_trace_file(&args.output, &result.trace)?;
render_finish(&result.trace, &args.output, format)
let pages_dir = write_trace_bundle(&args.output, &result.trace)?;
render_finish(&result.trace, &args.output, &pages_dir, format)
})(),
Err(err) => Err(err),
};
let session_stop_result = stop_session(info.sock_path, &session.session_id);
let session_stop_result = stop_recording_session(info.sock_path, &session.session_id);
record_state::clear();
run_result?;
@@ -161,22 +199,23 @@ fn dispatch_stop(args: RecordStopArgs, format: Format) -> Result<(), CliError> {
let info = ensure_daemon().context("ensure daemon is running")?;
let session_id = state.session_id.clone();
let params = RecordStopParams {
session_id: session_id.clone(),
};
let result = business_rpc::call::<RecordStopParams, RecordStopResult>(
info.sock_path.clone(),
"record-stop",
Method::ToolRecordStop,
Some(params),
TOOL_IPC_TIMEOUT,
)?;
let run_result: Result<(), CliError> = (|| {
let params = RecordStopParams {
session_id: session_id.clone(),
};
let result = business_rpc::call::<RecordStopParams, RecordStopResult>(
info.sock_path.clone(),
"record-stop",
Method::ToolRecordStop,
Some(params),
TOOL_IPC_TIMEOUT,
)?;
write_trace_file(&args.output, &result.trace)?;
render_stop(&result, &args.output, format)
let pages_dir = write_trace_bundle(&args.output, &result.trace)?;
render_stop(&result, &args.output, &pages_dir, format)
})();
let session_stop_result = stop_session(info.sock_path, &session_id);
let session_stop_result = stop_recording_session(info.sock_path, &session_id);
record_state::clear();
run_result?;
@@ -190,30 +229,313 @@ fn record_await_ipc_timeout(timeout_ms: u32) -> Duration {
.unwrap_or(Duration::from_secs(u64::from(timeout_ms / 1_000) + 15))
}
fn write_trace_file(output: &PathBuf, trace: &Trace) -> Result<(), CliError> {
let json = serde_json::to_string_pretty(trace)
.context("serialize trace JSON")
.map_err(CliError::Local)?;
if let Some(parent) = output.parent() {
if !parent.as_os_str().is_empty() {
fs::create_dir_all(parent)
.with_context(|| format!("create output directory {}", parent.display()))
.map_err(CliError::Local)?;
fn pages_dir_for_output(output_dir: &Path) -> PathBuf {
output_dir.join("pages")
}
fn trace_json_path(output_dir: &Path) -> PathBuf {
output_dir.join("trace.json")
}
fn is_canonical_state_id(id: &str) -> bool {
let Some(number) = id.strip_prefix('s') else {
return false;
};
let mut digits = number.bytes();
matches!(digits.next(), Some(b'1'..=b'9')) && digits.all(|byte| byte.is_ascii_digit())
}
fn validate_bundle_directory(path: &Path, description: &str) -> Result<(), CliError> {
match fs::symlink_metadata(path) {
Ok(metadata) if metadata.file_type().is_symlink() => Err(CliError::Local(anyhow::anyhow!(
"{description} {} must not be a symlink",
path.display()
))),
Ok(metadata) if metadata.is_dir() => Ok(()),
Ok(_) => Err(CliError::Local(anyhow::anyhow!(
"{description} {} is not a directory",
path.display()
))),
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
Err(err) => Err(CliError::Local(
anyhow::Error::new(err).context(format!("inspect {description} {}", path.display())),
)),
}
}
fn validate_replaceable_file(path: &Path, description: &str) -> Result<(), CliError> {
match fs::symlink_metadata(path) {
Ok(metadata) if metadata.file_type().is_file() => Ok(()),
Ok(_) => Err(CliError::Local(anyhow::anyhow!(
"{description} {} is not a regular file",
path.display()
))),
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
Err(err) => Err(CliError::Local(
anyhow::Error::new(err).context(format!("inspect {description} {}", path.display())),
)),
}
}
fn commit_staged_file(staged: &Path, target: &Path, backup: &Path) -> io::Result<Option<PathBuf>> {
let old_file = match fs::symlink_metadata(target) {
Ok(_) => {
fs::rename(target, backup)?;
Some(backup.to_path_buf())
}
Err(err) if err.kind() == io::ErrorKind::NotFound => None,
Err(err) => return Err(err),
};
if let Err(err) = fs::rename(staged, target) {
if let Some(old_file) = old_file.as_deref() {
if let Err(restore_err) = fs::rename(old_file, target) {
return Err(io::Error::new(
restore_err.kind(),
format!(
"{err}; additionally failed to restore {}: {restore_err}",
target.display()
),
));
}
}
return Err(err);
}
Ok(old_file)
}
fn rollback_installed_files(installed: &[(PathBuf, Option<PathBuf>)]) -> io::Result<()> {
let mut rollback_error = None;
for (target, backup) in installed.iter().rev() {
match fs::remove_file(target) {
Ok(()) => {}
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => {
rollback_error.get_or_insert(err);
continue;
}
}
if let Some(backup) = backup {
if let Err(err) = fs::rename(backup, target) {
rollback_error.get_or_insert(err);
}
}
}
match rollback_error {
Some(err) => Err(err),
None => Ok(()),
}
}
/// Drop page files left by an earlier, longer recording without deleting the
/// directory itself: `record start` and the `record stop` fallback export the
/// same recording concurrently, and a `remove_dir_all` here would fail the
/// writer whose files the other one just erased.
fn prune_stale_pages(pages_dir: &Path, keep: &HashSet<String>) -> Result<(), CliError> {
let entries = match fs::read_dir(pages_dir) {
Ok(entries) => entries,
Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(()),
Err(err) => {
return Err(CliError::Local(
anyhow::Error::new(err).context(format!("read pages dir {}", pages_dir.display())),
));
}
};
for entry in entries.flatten() {
let name = entry.file_name();
let Some(name) = name.to_str() else { continue };
if !name.ends_with(".vom.txt") || keep.contains(name) {
continue;
}
match fs::remove_file(entry.path()) {
Ok(()) => {}
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
Err(err) => {
return Err(CliError::Local(
anyhow::Error::new(err)
.context(format!("remove stale page {}", entry.path().display())),
));
}
}
}
fs::write(output, format!("{json}\n"))
.with_context(|| format!("write trace to {}", output.display()))
.map_err(CliError::Local)?;
Ok(())
}
fn render_finish(trace: &Trace, output: &PathBuf, format: Format) -> Result<(), CliError> {
fn write_trace_bundle(output_dir: &Path, trace: &Trace) -> Result<PathBuf, CliError> {
let pages_dir = pages_dir_for_output(output_dir);
let trace_path = trace_json_path(output_dir);
if trace.version != TRACE_VERSION {
return Err(CliError::Local(anyhow::anyhow!(
"trace version {} is not supported (expected {})",
trace.version,
TRACE_VERSION
)));
}
let mut state_ids = HashSet::with_capacity(trace.states.len());
for state in &trace.states {
if !is_canonical_state_id(&state.id) {
return Err(CliError::Local(anyhow::anyhow!(
"state id {:?} is not canonical (expected s1, s2, ...)",
state.id
)));
}
if !state_ids.insert(state.id.as_str()) {
return Err(CliError::Local(anyhow::anyhow!(
"duplicate state id {:?}",
state.id
)));
}
}
let mut disk_states: Vec<TraceState> = Vec::with_capacity(trace.states.len());
let mut page_writes = Vec::new();
let mut keep = HashSet::new();
for state in &trace.states {
let mut disk = state.clone();
if let Some(body) = state.body.as_deref() {
let filename = format!("{}.vom.txt", state.id);
let page_path = pages_dir.join(&filename);
if page_path.parent() != Some(pages_dir.as_path()) {
return Err(CliError::Local(anyhow::anyhow!(
"page path {} escapes pages directory {}",
page_path.display(),
pages_dir.display()
)));
}
keep.insert(filename.clone());
page_writes.push((filename.clone(), body));
disk.body = None;
disk.page = Some(filename);
}
disk_states.push(disk);
}
let mut disk_trace = trace.clone();
disk_trace.states = disk_states;
let json = serde_json::to_string_pretty(&disk_trace)
.context("serialize trace JSON")
.map_err(CliError::Local)?;
validate_bundle_directory(output_dir, "output bundle directory")?;
fs::create_dir_all(output_dir)
.with_context(|| format!("create output dir {}", output_dir.display()))
.map_err(CliError::Local)?;
validate_bundle_directory(&pages_dir, "pages directory")?;
// `record start` and `record stop` can export the same recording at once.
// Serialize writers without replacing the shared directory underneath one
// another; the lock is released when this file handle is dropped.
let lock_path = output_dir.join(".bsk-record-export.lock");
let lock_file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(false)
.open(&lock_path)
.with_context(|| format!("open export lock {}", lock_path.display()))
.map_err(CliError::Local)?;
fs2::FileExt::lock_exclusive(&lock_file)
.with_context(|| format!("lock export bundle {}", output_dir.display()))
.map_err(CliError::Local)?;
fs::create_dir_all(&pages_dir)
.with_context(|| format!("create pages dir {}", pages_dir.display()))
.map_err(CliError::Local)?;
for (filename, _) in &page_writes {
validate_replaceable_file(&pages_dir.join(filename), "page observation")?;
}
validate_replaceable_file(&trace_path, "trace JSON")?;
let transaction_id = uuid::Uuid::new_v4();
let staging_dir = output_dir.join(format!(".bsk-record-stage-{transaction_id}"));
let staging_pages = staging_dir.join("pages");
fs::create_dir_all(&staging_pages)
.with_context(|| format!("create staging dir {}", staging_pages.display()))
.map_err(CliError::Local)?;
let stage_result: Result<(), CliError> = (|| {
for (filename, body) in &page_writes {
let staged_path = staging_pages.join(filename);
fs::write(&staged_path, body)
.with_context(|| format!("write staged page observation {}", staged_path.display()))
.map_err(CliError::Local)?;
}
let staged_trace = staging_dir.join("trace.json");
fs::write(&staged_trace, format!("{json}\n"))
.with_context(|| format!("write staged trace to {}", staged_trace.display()))
.map_err(CliError::Local)?;
Ok(())
})();
if let Err(err) = stage_result {
let _ = fs::remove_dir_all(&staging_dir);
return Err(err);
}
let mut replacements: Vec<(PathBuf, PathBuf, PathBuf)> =
Vec::with_capacity(page_writes.len() + 1);
for (filename, _) in &page_writes {
replacements.push((
staging_pages.join(filename),
pages_dir.join(filename),
pages_dir.join(format!(".{filename}.{transaction_id}.bak")),
));
}
replacements.push((
staging_dir.join("trace.json"),
trace_path.clone(),
output_dir.join(format!(".trace.json.{transaction_id}.bak")),
));
let mut installed = Vec::with_capacity(replacements.len());
for (staged, target, backup) in replacements {
match commit_staged_file(&staged, &target, &backup) {
Ok(old_file) => installed.push((target, old_file)),
Err(err) => {
let rollback_result = rollback_installed_files(&installed);
let _ = fs::remove_dir_all(&staging_dir);
let message = match rollback_result {
Ok(()) => format!("commit replacement {}: {err}", target.display()),
Err(rollback_err) => format!(
"commit replacement {}: {err}; rollback also failed: {rollback_err}",
target.display()
),
};
return Err(CliError::Local(anyhow::anyhow!(message)));
}
}
}
for (_, backup) in &installed {
if let Some(backup) = backup {
let _ = fs::remove_file(backup);
}
}
let _ = fs::remove_dir_all(&staging_dir);
prune_stale_pages(&pages_dir, &keep)?;
Ok(pages_dir)
}
fn render_finish(
trace: &Trace,
output_dir: &PathBuf,
pages_dir: &PathBuf,
format: Format,
) -> Result<(), CliError> {
let trace_path = trace_json_path(output_dir);
match format {
Format::Json => {
println!(
"{}",
serde_json::to_string(&serde_json::json!({
"output": output,
"output": output_dir,
"trace_json": trace_path,
"pages_dir": pages_dir,
"trace": trace,
"window_closed": true,
}))
@@ -221,7 +543,20 @@ fn render_finish(trace: &Trace, output: &PathBuf, format: Format) -> Result<(),
);
}
Format::Human => {
println!("saved {} steps to {}", trace.steps.len(), output.display());
println!(
"saved {} steps and {} states to {} (+ {})",
trace.steps.len(),
trace.states.len(),
trace_path.display(),
pages_dir.display()
);
if let Some(frame_capture) = &trace.frame_capture {
if matches!(frame_capture.status, FrameCaptureStatus::Partial) {
eprintln!(
"warning: trace is partial — some embedded frames were not recorded (see frame_capture.failures in trace.json)"
);
}
}
}
}
Ok(())
@@ -230,33 +565,252 @@ fn render_finish(trace: &Trace, output: &PathBuf, format: Format) -> Result<(),
fn render_stop(
result: &RecordStopResult,
output: &PathBuf,
pages_dir: &PathBuf,
format: Format,
) -> Result<(), CliError> {
render_finish(&result.trace, output, format)
render_finish(&result.trace, output, pages_dir, format)
}
#[cfg(test)]
mod tests {
use super::*;
use bsk_protocol::tools::{
NavigationCause, RecorderInfo, Step, StepCommon, StepResult, StopReason, TraceEntry,
VOM_FORMAT_VERSION,
};
#[test]
fn default_output_is_trace_json() {
let args = RecordStopArgs {
output: PathBuf::from("trace.json"),
};
assert_eq!(args.output, PathBuf::from("trace.json"));
fn sample_trace(state_id: &str, body: &str) -> Trace {
Trace {
version: TRACE_VERSION,
recorded_at: "2026-08-10T02:12:55.360Z".into(),
stopped_by: StopReason::UserFinish,
entry: TraceEntry {
start_url: "https://example.com/".into(),
},
recorder: RecorderInfo {
bsk: "0.1.10".into(),
vom: VOM_FORMAT_VERSION,
},
frame_capture: None,
states: vec![TraceState {
id: state_id.into(),
url: "https://example.com/".into(),
title: Some("Example".into()),
body: Some(body.into()),
page: None,
truncated: false,
}],
steps: vec![Step::Navigate {
common: StepCommon {
id: 1,
state: state_id.into(),
result: StepResult {
state: state_id.into(),
},
},
to: "https://example.com/".into(),
cause: NavigationCause::UserTyped,
}],
purpose: None,
started_at: None,
}
}
#[test]
fn start_args_default_output_is_trace_json() {
fn concurrent_teardown_is_not_a_record_failure() {
let stopping = CliError::from_rpc(bsk_protocol::RpcError {
code: ErrorCode::Timeout,
message: "session stop is already in progress".into(),
data: Some(serde_json::json!({ "reason": STOP_IN_PROGRESS_REASON })),
});
assert!(session_teardown_raced(&stopping));
let gone = CliError::from_rpc(bsk_protocol::RpcError {
code: ErrorCode::NotFound,
message: "session is not registered".into(),
data: None,
});
assert!(session_teardown_raced(&gone));
}
#[test]
fn unrelated_stop_timeout_still_fails() {
let stalled = CliError::from_rpc(bsk_protocol::RpcError {
code: ErrorCode::Timeout,
message: "tool.session_stop timed out".into(),
data: None,
});
assert!(!session_teardown_raced(&stalled));
}
#[test]
fn default_output_is_trace_dir() {
let args = RecordStopArgs {
output: PathBuf::from("trace"),
};
assert_eq!(args.output, PathBuf::from("trace"));
}
#[test]
fn start_args_default_output_is_trace_dir() {
let args = RecordStartArgs {
browser: None,
tab_id: None,
url: None,
purpose: None,
output: PathBuf::from("trace.json"),
max_page_tokens: None,
redact_values: false,
output: PathBuf::from("trace"),
};
assert_eq!(args.output, PathBuf::from("trace.json"));
assert_eq!(args.output, PathBuf::from("trace"));
}
#[test]
fn write_trace_bundle_writes_dir_layout() {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
let trace = sample_trace("s1", "@vom 1\nL1 page");
let pages_dir = write_trace_bundle(&output, &trace).unwrap();
let trace_path = output.join("trace.json");
assert!(trace_path.exists());
assert_eq!(pages_dir, output.join("pages"));
assert!(pages_dir.join("s1.vom.txt").exists());
let disk: serde_json::Value =
serde_json::from_str(&fs::read_to_string(&trace_path).unwrap()).unwrap();
assert_eq!(disk["states"][0]["page"], "s1.vom.txt");
assert!(disk["states"][0].get("body").is_none());
assert_eq!(
fs::read_to_string(pages_dir.join("s1.vom.txt")).unwrap(),
"@vom 1\nL1 page"
);
// Re-exporting drops pages from the previous run but keeps the
// directory in place for the concurrent writer.
fs::write(pages_dir.join("s9.vom.txt"), "stale").unwrap();
fs::write(pages_dir.join("notes.md"), "keep me").unwrap();
write_trace_bundle(&output, &trace).unwrap();
assert!(!pages_dir.join("s9.vom.txt").exists());
assert!(pages_dir.join("s1.vom.txt").exists());
assert!(pages_dir.join("notes.md").exists());
}
#[test]
fn write_trace_bundle_rejects_traversal_state_id() {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
let trace = sample_trace("../escaped", "@vom 1\nL1 page");
let err = write_trace_bundle(&output, &trace).unwrap_err();
assert!(err.to_string().contains("state id"));
assert!(!output.join("escaped.vom.txt").exists());
assert!(!output.join("trace.json").exists());
}
#[test]
fn write_trace_bundle_rejects_duplicate_state_id() {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
let mut trace = sample_trace("s1", "@vom 1\nL1 first");
trace.states.push(TraceState {
body: Some("@vom 1\nL1 duplicate".into()),
..trace.states[0].clone()
});
let err = write_trace_bundle(&output, &trace).unwrap_err();
assert!(err.to_string().contains("duplicate state id"));
assert!(!output.exists());
}
#[test]
fn unsupported_version_preserves_existing_bundle() {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
write_trace_bundle(&output, &sample_trace("s1", "original")).unwrap();
let original_json = fs::read(output.join("trace.json")).unwrap();
let mut replacement = sample_trace("s2", "replacement");
replacement.version += 1;
let err = write_trace_bundle(&output, &replacement).unwrap_err();
assert!(err.to_string().contains("not supported"));
assert_eq!(fs::read(output.join("trace.json")).unwrap(), original_json);
assert_eq!(
fs::read_to_string(output.join("pages/s1.vom.txt")).unwrap(),
"original"
);
assert!(!output.join("pages/s2.vom.txt").exists());
}
#[test]
fn page_replacement_failure_preserves_existing_bundle() {
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
write_trace_bundle(&output, &sample_trace("s1", "original")).unwrap();
let original_json = fs::read(output.join("trace.json")).unwrap();
fs::write(output.join("pages/notes.md"), "keep me").unwrap();
fs::create_dir(output.join("pages/s2.vom.txt")).unwrap();
let err = write_trace_bundle(&output, &sample_trace("s2", "replacement")).unwrap_err();
assert!(err.to_string().contains("page"));
assert_eq!(fs::read(output.join("trace.json")).unwrap(), original_json);
assert_eq!(
fs::read_to_string(output.join("pages/s1.vom.txt")).unwrap(),
"original"
);
assert_eq!(
fs::read_to_string(output.join("pages/notes.md")).unwrap(),
"keep me"
);
}
#[cfg(unix)]
#[test]
fn write_trace_bundle_rejects_symlinked_pages_directory() {
use std::os::unix::fs::symlink;
let dir = tempfile::tempdir().unwrap();
let output = dir.path().join("flow");
let external_pages = dir.path().join("external-pages");
fs::create_dir_all(&output).unwrap();
fs::create_dir_all(&external_pages).unwrap();
fs::write(external_pages.join("s9.vom.txt"), "external").unwrap();
symlink(&external_pages, output.join("pages")).unwrap();
let err = write_trace_bundle(&output, &sample_trace("s1", "replacement")).unwrap_err();
assert!(err.to_string().contains("symlink"));
assert_eq!(
fs::read_to_string(external_pages.join("s9.vom.txt")).unwrap(),
"external"
);
assert!(!external_pages.join("s1.vom.txt").exists());
assert!(!output.join("trace.json").exists());
}
#[cfg(unix)]
#[test]
fn write_trace_bundle_rejects_symlinked_output_directory() {
use std::os::unix::fs::symlink;
let dir = tempfile::tempdir().unwrap();
let external_output = dir.path().join("external-output");
let output = dir.path().join("flow");
fs::create_dir_all(&external_output).unwrap();
fs::write(external_output.join("notes.md"), "external").unwrap();
symlink(&external_output, &output).unwrap();
let err = write_trace_bundle(&output, &sample_trace("s1", "replacement")).unwrap_err();
assert!(err.to_string().contains("symlink"));
assert_eq!(
fs::read_to_string(external_output.join("notes.md")).unwrap(),
"external"
);
assert!(!external_output.join("trace.json").exists());
assert!(!external_output.join("pages").exists());
}
#[test]
+27 -3
View File
@@ -141,9 +141,9 @@ pub fn info_for(code: ErrorCode) -> RenderInfo {
exit_code: 2,
},
ErrorCode::VersionTooOld => RenderInfo {
summary: "peer version is too old to communicate with this build",
summary: "the installed bsk CLI and BrowserSkill extension are incompatible",
hint: Some(
"upgrade both bsk CLI and the browser-skill extension so both sides satisfy `min_compatible_protocol`",
"update bsk and the BrowserSkill extension, reload the extension, then run `bsk daemon restart`",
),
exit_code: 5,
},
@@ -157,7 +157,7 @@ pub fn info_for(code: ErrorCode) -> RenderInfo {
ErrorCode::NoBrowserConnected => RenderInfo {
summary: "no browser is connected to the daemon",
hint: Some(
"open the browser-skill extension in your browser and wait for the popup to show \"connected\"",
"open the BrowserSkill extension and wait for it to show \"connected\"; if the extension is already open, update and reload it, then run `bsk daemon restart`",
),
exit_code: 1,
},
@@ -310,6 +310,30 @@ mod tests {
assert_eq!(exit_code_for(ErrorCode::VersionTooOld), 5);
}
#[test]
fn version_error_names_the_components_and_gives_update_steps() {
let info = info_for(ErrorCode::VersionTooOld);
assert!(info.summary.contains("bsk CLI"));
assert!(info.summary.contains("BrowserSkill extension"));
let hint = info
.hint
.expect("version mismatch should have a repair hint");
assert!(hint.contains("update bsk"));
assert!(hint.contains("reload the extension"));
assert!(hint.contains("bsk daemon restart"));
assert!(!hint.contains("min_compatible_protocol"));
}
#[test]
fn no_browser_hint_explains_version_mismatch_recovery() {
let hint = info_for(ErrorCode::NoBrowserConnected)
.hint
.expect("no browser should have a repair hint");
assert!(hint.contains("extension is already open"));
assert!(hint.contains("update"));
assert!(hint.contains("bsk daemon restart"));
}
/// Every code must carry a non-empty summary; an empty `summary`
/// would render `error: ` with nothing after it.
#[test]
+12
View File
@@ -786,7 +786,19 @@ async fn handle_session_stop(state: &Arc<DaemonState>, params: Value) -> Result<
}
}
/// Marks a `session.stop` rejected because another teardown already owns the
/// session, so callers that race on purpose (`bsk record`) can tell it apart
/// from a browser that stopped responding.
pub const STOP_IN_PROGRESS_REASON: &str = "session_stop_in_progress";
fn map_stop_error(err: StopSessionError) -> RpcError {
if matches!(err, StopSessionError::Stopping) {
return RpcError {
code: ErrorCode::Timeout,
message: err.to_string(),
data: Some(serde_json::json!({ "reason": STOP_IN_PROGRESS_REASON })),
};
}
let code = match &err {
StopSessionError::NotFound => ErrorCode::NotFound,
StopSessionError::Stopping => ErrorCode::Timeout,
+2 -2
View File
@@ -27,7 +27,7 @@ use crate::daemon::{
browsers::{BROWSER_LIVENESS_TICK, BROWSER_LIVENESS_TIMEOUT, EXTENSION_CONNECT_WAIT},
info as daemon_info, ipc, lockfile, paths,
sessions::{StopSessionError, forget_session, stop_session},
state::DaemonState,
state::{DaemonState, PROTOCOL_VERSION},
ws,
};
@@ -299,7 +299,7 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
ws_port,
sock_path: sock_path.clone(),
daemon_version: env!("CARGO_PKG_VERSION"),
protocol_version: "1.0",
protocol_version: PROTOCOL_VERSION,
};
let handler = ipc::full_handler(status, Arc::clone(&state));
+3 -3
View File
@@ -16,9 +16,9 @@ use super::start::DaemonConfig;
use super::ws::WsHandle;
pub const DAEMON_VERSION: &str = env!("CARGO_PKG_VERSION");
pub const PROTOCOL_VERSION: &str = "1.0";
/// Lowest **protocol** version peers must speak (e.g. `"1.0"`).
pub const MIN_COMPATIBLE_PROTOCOL: &str = "1.0";
pub const PROTOCOL_VERSION: &str = "1.1";
/// Lowest **protocol** version peers must speak (e.g. `"1.1"`).
pub const MIN_COMPATIBLE_PROTOCOL: &str = "1.1";
/// Legacy app-semver floor used only when `HandshakeResult.min_compatible_peer`
/// is emitted for old extensions. New code ignores this on read.
pub const LEGACY_MIN_COMPATIBLE_PEER: &str = "0.0.0";
+2 -2
View File
@@ -112,14 +112,14 @@ async fn handshake_as_ext(
let params = HandshakeParams {
client: "browser-skill-extension".into(),
version: "0.1.0-dev.0".parse().unwrap(),
protocol_version: "1.0".into(),
protocol_version: "1.1".into(),
instance_id: TEST_EXT_ID.into(),
browser: BrowserPeerInfo {
name: "chrome".into(),
version: "131.0".into(),
},
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
min_compatible_protocol: Some("1.0".into()),
min_compatible_protocol: Some("1.1".into()),
label: "Test".into(),
};
let req = RequestFrame {
+2 -2
View File
@@ -75,7 +75,7 @@ async fn handshake_as_ext(
let params = HandshakeParams {
client: "browser-skill-extension".into(),
version: "0.1.0-dev.0".parse().unwrap(),
protocol_version: "1.0".into(),
protocol_version: "1.1".into(),
instance_id: TEST_EXT_ID.into(),
browser: BrowserPeerInfo {
name: "chrome".into(),
@@ -83,7 +83,7 @@ async fn handshake_as_ext(
},
label: "Test".into(),
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
min_compatible_protocol: Some("1.0".into()),
min_compatible_protocol: Some("1.1".into()),
};
let req = RequestFrame {
id: "hs".into(),
+33 -11
View File
@@ -108,12 +108,12 @@ async fn send_handshake_with_floors(
async fn handshake_ok_when_protocol_matches() {
let (handle, _sock) = spawn_daemon().await;
let mut ws = open_ws(handle.ws_addr()).await;
let resp = send_handshake(&mut ws, "1.0", env!("CARGO_PKG_VERSION")).await;
let resp = send_handshake(&mut ws, "1.1", env!("CARGO_PKG_VERSION")).await;
let result: HandshakeResult = match resp.body {
ResponseBody::Ok(v) => serde_json::from_value(v).unwrap(),
ResponseBody::Err(e) => panic!("expected ok handshake, got {e:?}"),
};
assert_eq!(result.protocol_version, "1.0");
assert_eq!(result.protocol_version, "1.1");
assert_eq!(
result
.min_compatible_peer
@@ -124,7 +124,7 @@ async fn handshake_ok_when_protocol_matches() {
);
assert_eq!(
result.min_compatible_protocol.as_deref(),
Some("1.0"),
Some("1.1"),
"daemon must advertise protocol floor for new extensions"
);
handle.shutdown().await;
@@ -135,7 +135,7 @@ async fn handshake_ok_when_app_versions_differ_but_protocol_matches() {
let (handle, _sock) = spawn_daemon().await;
let mut ws = open_ws(handle.ws_addr()).await;
let resp =
send_handshake_with_floors(&mut ws, "1.0", "9.9.9", Some("0.0.0"), Some("1.0")).await;
send_handshake_with_floors(&mut ws, "1.1", "9.9.9", Some("0.0.0"), Some("1.1")).await;
match resp.body {
ResponseBody::Ok(_) => {}
other => panic!("expected ok when protocol matches, got {other:?}"),
@@ -149,10 +149,10 @@ async fn handshake_skew_when_protocol_minor_differs() {
let mut ws = open_ws(handle.ws_addr()).await;
let resp = send_handshake_with_floors(
&mut ws,
"1.1",
"1.2",
env!("CARGO_PKG_VERSION"),
Some("0.0.0"),
Some("1.0"),
Some("1.1"),
)
.await;
match resp.body {
@@ -168,6 +168,28 @@ async fn handshake_skew_when_protocol_minor_differs() {
handle.shutdown().await;
}
#[tokio::test]
async fn handshake_rejects_pre_trace_v3_extension() {
let (handle, _sock) = spawn_daemon().await;
let mut ws = open_ws(handle.ws_addr()).await;
let resp = send_handshake_with_floors(
&mut ws,
"1.0",
env!("CARGO_PKG_VERSION"),
Some("0.0.0"),
Some("1.0"),
)
.await;
match resp.body {
ResponseBody::Err(e) => {
assert_eq!(e.code, ErrorCode::VersionTooOld);
assert!(e.message.contains("below local min_compatible_protocol"));
}
other => panic!("pre-Trace-v3 extension must be rejected, got {other:?}"),
}
handle.shutdown().await;
}
#[tokio::test]
async fn handshake_rejected_on_protocol_major_mismatch() {
let (handle, _sock) = spawn_daemon().await;
@@ -210,7 +232,7 @@ async fn handshake_legacy_ext_without_protocol_floor_still_ok() {
let mut ws = open_ws(handle.ws_addr()).await;
let resp = send_handshake_with_floors(
&mut ws,
"1.0",
"1.1",
env!("CARGO_PKG_VERSION"),
Some("0.1.0"),
None,
@@ -235,7 +257,7 @@ async fn status_surfaces_version_skew_for_skewed_browser() {
browser_name: "chrome".into(),
browser_version: "131.0".into(),
extension_version: "9.9.9".into(),
extension_protocol_version: "1.1".into(),
extension_protocol_version: "1.2".into(),
label: "Older".into(),
sink: bsk::daemon::browsers::BrowserSink { tx },
pending: Mutex::new(bsk::daemon::browsers::Pending::default()),
@@ -263,8 +285,8 @@ async fn status_surfaces_version_skew_for_skewed_browser() {
.iter()
.find(|s| s.instance_id == "skew-only-test")
.expect("status must list our skew client");
assert_eq!(skew.client_protocol_version, "1.1");
assert_eq!(skew.server_protocol_version, "1.0");
assert_eq!(skew.client_protocol_version, "1.2");
assert_eq!(skew.server_protocol_version, "1.1");
assert_eq!(skew.client_version, "9.9.9");
let entry = status
.browsers
@@ -281,7 +303,7 @@ async fn handshake_rejects_when_local_below_peer_min_compatible_protocol() {
let mut ws = open_ws(handle.ws_addr()).await;
let resp = send_handshake_with_floors(
&mut ws,
"1.0",
"1.1",
env!("CARGO_PKG_VERSION"),
Some("0.0.0"),
Some("99.0.0"),
+2 -2
View File
@@ -81,14 +81,14 @@ async fn handshake_as_ext(
let params = HandshakeParams {
client: "browser-skill-extension".into(),
version: "0.1.0-dev.0".parse().unwrap(),
protocol_version: "1.0".into(),
protocol_version: "1.1".into(),
instance_id: TEST_EXT_ID.into(),
browser: BrowserPeerInfo {
name: "chrome".into(),
version: "131.0".into(),
},
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
min_compatible_protocol: Some("1.0".into()),
min_compatible_protocol: Some("1.1".into()),
label: "Test".into(),
};
let req = RequestFrame {
+272
View File
@@ -0,0 +1,272 @@
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::time::Duration;
use bsk::cli::record_state;
use bsk::daemon::info::DaemonInfo;
use bsk::daemon::paths::BSK_HOME_ENV;
use bsk::daemon::{self, DaemonConfig};
use bsk::ipc_client::IpcClient;
use bsk_protocol::system::{HandshakeParams, HandshakeResult};
use bsk_protocol::tools::{
RecordStopResult, RecorderInfo, SessionStartParams, SessionStartResult, StopReason, Trace,
TraceEntry, TraceState, VOM_FORMAT_VERSION,
};
use bsk_protocol::{
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
};
use futures_util::{SinkExt, StreamExt};
use serde_json::json;
use tokio::process::Command;
use tokio_tungstenite::tungstenite::handshake::client::generate_key;
use tokio_tungstenite::tungstenite::http::Request;
use tokio_tungstenite::tungstenite::protocol::Message;
const TEST_EXT_ID: &str = "abcdefghijklmnopabcdefghijklmnop";
type Ws =
tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>>;
fn bsk_bin() -> PathBuf {
PathBuf::from(env!("CARGO_BIN_EXE_bsk"))
}
async fn connect_extension(addr: std::net::SocketAddr) -> Ws {
let origin = format!("chrome-extension://{TEST_EXT_ID}");
let request = Request::builder()
.method("GET")
.uri(format!("ws://{addr}/"))
.header("Host", addr.to_string())
.header("Upgrade", "websocket")
.header("Connection", "Upgrade")
.header("Sec-WebSocket-Version", "13")
.header("Sec-WebSocket-Key", generate_key())
.header("Origin", origin)
.body(())
.unwrap();
tokio_tungstenite::connect_async(request).await.unwrap().0
}
async fn handshake(ws: &mut Ws) {
let request = RequestFrame {
id: "handshake".into(),
method: Method::SystemHandshake,
params: Some(
serde_json::to_value(HandshakeParams {
client: "browser-skill-extension".into(),
version: "0.1.10".parse().unwrap(),
protocol_version: "1.1".into(),
instance_id: TEST_EXT_ID.into(),
browser: BrowserPeerInfo {
name: "chrome".into(),
version: "131.0".into(),
},
min_compatible_peer: Some("0.1.10".parse().unwrap()),
min_compatible_protocol: Some("1.1".into()),
label: "Record stop retry test".into(),
})
.unwrap(),
),
};
ws.send(Message::Text(serde_json::to_string(&request).unwrap()))
.await
.unwrap();
let Message::Text(response) = ws.next().await.unwrap().unwrap() else {
panic!("expected text handshake response");
};
let response: ResponseFrame = serde_json::from_str(&response).unwrap();
let ResponseBody::Ok(value) = response.body else {
panic!("handshake failed");
};
let _: HandshakeResult = serde_json::from_value(value).unwrap();
}
fn sample_trace() -> Trace {
Trace {
version: 3,
purpose: Some("retry record stop".into()),
started_at: Some("2026-08-12T10:00:00Z".into()),
recorded_at: "2026-08-12T10:01:00Z".into(),
stopped_by: StopReason::CliStop,
entry: TraceEntry {
start_url: "https://example.com/".into(),
},
recorder: RecorderInfo {
bsk: "0.1.10".into(),
vom: VOM_FORMAT_VERSION,
},
frame_capture: None,
states: vec![TraceState {
id: "s1".into(),
url: "https://example.com/".into(),
title: Some("Example".into()),
body: Some("@vom 1\nL1 page".into()),
page: None,
truncated: false,
}],
steps: vec![],
}
}
fn run_extension(
mut ws: Ws,
record_stop_calls: Arc<AtomicUsize>,
session_stop_calls: Arc<AtomicUsize>,
) {
tokio::spawn(async move {
while let Some(Ok(Message::Text(text))) = ws.next().await {
let Ok(Frame::Request(request)) = serde_json::from_str::<Frame>(&text) else {
continue;
};
let body = match request.method {
Method::ToolSessionStart => {
let _: SessionStartParams =
serde_json::from_value(request.params.clone().unwrap()).unwrap();
ResponseBody::Ok(
serde_json::to_value(SessionStartResult {
agent_window_id: Some(100),
})
.unwrap(),
)
}
Method::ToolRecordStop => {
if record_stop_calls.fetch_add(1, Ordering::SeqCst) == 0 {
ResponseBody::Err(RpcError {
code: ErrorCode::ProtocolError,
message: "recording remains active; retry record stop".into(),
data: Some(json!({ "recording": true, "retryable": true })),
})
} else {
ResponseBody::Ok(
serde_json::to_value(RecordStopResult {
trace: sample_trace(),
})
.unwrap(),
)
}
}
Method::ToolSessionStop => {
session_stop_calls.fetch_add(1, Ordering::SeqCst);
ResponseBody::Ok(json!({}))
}
_ => ResponseBody::Err(RpcError {
code: ErrorCode::ProtocolError,
message: format!("unexpected method {:?}", request.method),
data: None,
}),
};
let response = ResponseFrame {
id: request.id,
body,
};
ws.send(Message::Text(serde_json::to_string(&response).unwrap()))
.await
.unwrap();
}
});
}
async fn start_session(sock: &Path) -> String {
#[derive(serde::Deserialize)]
struct StartResult {
session_id: String,
}
let mut client = IpcClient::connect(sock).await.unwrap();
client
.call::<(), StartResult>(
"session-start",
Method::SessionStart,
None,
Duration::from_secs(5),
)
.await
.unwrap()
.unwrap()
.session_id
}
async fn run_record_stop(home: &Path, output: &Path) -> std::process::Output {
Command::new(bsk_bin())
.arg("record")
.arg("stop")
.arg("--output")
.arg(output)
.env(BSK_HOME_ENV, home)
.output()
.await
.unwrap()
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn failed_record_stop_keeps_state_and_session_for_retry() {
let temp = tempfile::Builder::new()
.prefix(".record-stop-retry-")
.tempdir_in(env!("CARGO_MANIFEST_DIR"))
.unwrap();
let home = temp.path().join("bsk-home");
let sock = temp.path().join("daemon.sock");
let output = temp.path().join("trace");
let daemon = daemon::run(DaemonConfig::new(0), Some(sock.clone()))
.await
.unwrap();
bsk::daemon::info::write_to_path(
&DaemonInfo::now(
std::process::id(),
sock.clone(),
daemon.ws_addr().port(),
env!("CARGO_PKG_VERSION"),
),
&home.join("daemon.json"),
)
.unwrap();
let mut extension = connect_extension(daemon.ws_addr()).await;
handshake(&mut extension).await;
let record_stop_calls = Arc::new(AtomicUsize::new(0));
let session_stop_calls = Arc::new(AtomicUsize::new(0));
run_extension(
extension,
Arc::clone(&record_stop_calls),
Arc::clone(&session_stop_calls),
);
let session_id = start_session(&sock).await;
unsafe {
std::env::set_var(BSK_HOME_ENV, &home);
}
record_state::write(&session_id).unwrap();
let first = run_record_stop(&home, &output).await;
assert!(
!first.status.success(),
"first stop unexpectedly succeeded: {}",
String::from_utf8_lossy(&first.stdout)
);
assert_eq!(record_state::read().unwrap().session_id, session_id);
assert_eq!(record_stop_calls.load(Ordering::SeqCst), 1);
assert_eq!(
session_stop_calls.load(Ordering::SeqCst),
0,
"failed record stop must leave the session active"
);
let second = run_record_stop(&home, &output).await;
assert!(
second.status.success(),
"retry failed: {}",
String::from_utf8_lossy(&second.stderr)
);
assert!(record_state::read().is_err());
assert_eq!(record_stop_calls.load(Ordering::SeqCst), 2);
assert_eq!(session_stop_calls.load(Ordering::SeqCst), 1);
assert!(output.join("trace.json").exists());
assert!(output.join("pages/s1.vom.txt").exists());
unsafe {
std::env::remove_var(BSK_HOME_ENV);
}
daemon.shutdown().await;
}
@@ -81,7 +81,7 @@ async fn handshake_as_ext(
let params = HandshakeParams {
client: "browser-skill-extension".into(),
version: "0.1.0-dev.0".parse().unwrap(),
protocol_version: "1.0".into(),
protocol_version: "1.1".into(),
instance_id: TEST_EXT_ID.into(),
browser: BrowserPeerInfo {
name: "chrome".into(),
@@ -89,7 +89,7 @@ async fn handshake_as_ext(
},
label: "Test".into(),
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
min_compatible_protocol: Some("1.0".into()),
min_compatible_protocol: Some("1.1".into()),
};
let req = RequestFrame {
id: "hs".into(),
+4 -4
View File
@@ -84,14 +84,14 @@ async fn handshake_as_ext(
let params = HandshakeParams {
client: "browser-skill-extension".into(),
version: "0.1.0-dev.0".parse().unwrap(),
protocol_version: "1.0".into(),
protocol_version: "1.1".into(),
instance_id: TEST_EXT_ID.into(),
browser: BrowserPeerInfo {
name: "chrome".into(),
version: "131.0".into(),
},
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
min_compatible_protocol: Some("1.0".into()),
min_compatible_protocol: Some("1.1".into()),
label: "Test".into(),
};
let req = RequestFrame {
@@ -400,14 +400,14 @@ async fn connect_second_ext(
let params = HandshakeParams {
client: "browser-skill-extension".into(),
version: "0.1.0-dev.0".parse().unwrap(),
protocol_version: "1.0".into(),
protocol_version: "1.1".into(),
instance_id: instance_id.into(),
browser: BrowserPeerInfo {
name: "edge".into(),
version: "130".into(),
},
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
min_compatible_protocol: Some("1.0".into()),
min_compatible_protocol: Some("1.1".into()),
label: label.into(),
};
let hs = RequestFrame {
+1 -1
View File
@@ -61,7 +61,7 @@ fn bsk_status_json_returns_structured_payload() {
assert!(parsed["pid"].as_u64().unwrap() > 0);
assert!(!parsed["daemon_version"].as_str().unwrap().is_empty());
assert_eq!(parsed["protocol_version"], "1.0");
assert_eq!(parsed["protocol_version"], "1.1");
assert!(parsed["sock_path"].as_str().is_some());
assert_eq!(parsed["browsers"], serde_json::json!([]));
assert_eq!(parsed["sessions"], serde_json::json!([]));
+2 -2
View File
@@ -73,14 +73,14 @@ async fn do_handshake(ws: &mut Ws) -> HandshakeResult {
let params = HandshakeParams {
client: "browser-skill-extension".into(),
version: "0.1.0-dev.0".parse().unwrap(),
protocol_version: "1.0".into(),
protocol_version: "1.1".into(),
instance_id: TEST_EXT_ID.into(),
browser: BrowserPeerInfo {
name: "chrome".into(),
version: "131.0".into(),
},
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
min_compatible_protocol: Some("1.0".into()),
min_compatible_protocol: Some("1.1".into()),
label: "Test".into(),
};
let req = RequestFrame {
+2 -2
View File
@@ -76,14 +76,14 @@ async fn do_handshake(ws: &mut Ws) -> HandshakeResult {
let params = HandshakeParams {
client: "browser-skill-extension".into(),
version: "0.1.0-dev.0".parse().unwrap(),
protocol_version: "1.0".into(),
protocol_version: "1.1".into(),
instance_id: TEST_EXT_ID.into(),
browser: BrowserPeerInfo {
name: "chrome".into(),
version: "131.0".into(),
},
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
min_compatible_protocol: Some("1.0".into()),
min_compatible_protocol: Some("1.1".into()),
label: "Test".into(),
};
let req = RequestFrame {
+2 -2
View File
@@ -75,14 +75,14 @@ async fn do_handshake(ws: &mut Ws) -> HandshakeResult {
let params = HandshakeParams {
client: "browser-skill-extension".into(),
version: "0.1.0-dev.0".parse().unwrap(),
protocol_version: "1.0".into(),
protocol_version: "1.1".into(),
instance_id: TEST_EXT_ID.into(),
browser: BrowserPeerInfo {
name: "chrome".into(),
version: "131.0".into(),
},
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
min_compatible_protocol: Some("1.0".into()),
min_compatible_protocol: Some("1.1".into()),
label: "Test".into(),
};
let req = RequestFrame {
+2 -2
View File
@@ -85,14 +85,14 @@ async fn do_handshake(ws: &mut Ws) -> HandshakeResult {
let params = HandshakeParams {
client: "browser-skill-extension".into(),
version: "0.1.0-dev.0".parse().unwrap(),
protocol_version: "1.0".into(),
protocol_version: "1.1".into(),
instance_id: TEST_EXT_ID.into(),
browser: BrowserPeerInfo {
name: "chrome".into(),
version: "131.0".into(),
},
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
min_compatible_protocol: Some("1.0".into()),
min_compatible_protocol: Some("1.1".into()),
label: "Test".into(),
};
let req = RequestFrame {
+3 -3
View File
@@ -53,14 +53,14 @@ pub async fn send_handshake(
let params = HandshakeParams {
client: "browser-skill-extension".into(),
version: "0.1.0-dev.0".parse().unwrap(),
protocol_version: "1.0".into(),
protocol_version: "1.1".into(),
instance_id: instance_id.into(),
browser: BrowserPeerInfo {
name: "chrome".into(),
version: "131.0".into(),
},
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
min_compatible_protocol: Some("1.0".into()),
min_compatible_protocol: Some("1.1".into()),
label: "Test Chrome".into(),
};
let req = RequestFrame {
@@ -91,7 +91,7 @@ async fn ws_handshake_registers_browser_in_state() {
let mut ws = connect_ext(handle.ws_addr(), &origin).await;
let result = send_handshake(&mut ws, TEST_EXT_ID).await;
assert_eq!(result.server, "browser-skill-daemon");
assert_eq!(result.protocol_version, "1.0");
assert_eq!(result.protocol_version, "1.1");
let state = handle.state();
let browsers = state.browsers.snapshot();
@@ -11,6 +11,14 @@
}
},
"definitions": {
"FillCommit": {
"type": "string",
"enum": [
"enter",
"suggestion",
"blur"
]
},
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
@@ -21,25 +29,32 @@
"shift"
]
},
"PageRef": {
"description": "Page context dictionary entry — referenced by steps via `page` id.",
"NavigationCause": {
"type": "string",
"enum": [
"user_typed",
"link",
"form_submit",
"reload",
"history",
"script",
"browser"
]
},
"RecorderInfo": {
"type": "object",
"required": [
"id",
"url"
"bsk",
"vom"
],
"properties": {
"id": {
"bsk": {
"type": "string"
},
"title": {
"type": [
"string",
"null"
]
},
"url": {
"type": "string"
"vom": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
}
}
},
@@ -68,21 +83,16 @@
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"cause",
"id",
"op",
"page",
"result",
"state",
"to"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
"cause": {
"$ref": "#/definitions/NavigationCause"
},
"id": {
"type": "integer",
@@ -95,8 +105,11 @@
"navigate"
]
},
"page": {
"description": "Reference into `pages[]`.",
"result": {
"$ref": "#/definitions/StepResult"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"to": {
@@ -110,20 +123,11 @@
"required": [
"id",
"op",
"page",
"result",
"state",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
@@ -135,8 +139,11 @@
"click"
]
},
"page": {
"description": "Reference into `pages[]`.",
"result": {
"$ref": "#/definitions/StepResult"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
@@ -150,20 +157,11 @@
"required": [
"id",
"op",
"page",
"result",
"state",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
@@ -175,8 +173,11 @@
"hover"
]
},
"page": {
"description": "Reference into `pages[]`.",
"result": {
"$ref": "#/definitions/StepResult"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
@@ -188,22 +189,17 @@
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"commit",
"id",
"op",
"page",
"result",
"state",
"target",
"value"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
"commit": {
"$ref": "#/definitions/FillCommit"
},
"id": {
"type": "integer",
@@ -216,15 +212,15 @@
"fill"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"redacted": {
"type": [
"boolean",
"null"
]
"type": "boolean"
},
"result": {
"$ref": "#/definitions/StepResult"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
@@ -240,21 +236,11 @@
"required": [
"id",
"op",
"page",
"selection",
"result",
"state",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
@@ -266,9 +252,8 @@
"select"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
"result": {
"$ref": "#/definitions/StepResult"
},
"selection": {
"type": "array",
@@ -276,6 +261,10 @@
"$ref": "#/definitions/SelectedOption"
}
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
}
@@ -288,19 +277,10 @@
"id",
"key",
"op",
"page"
"result",
"state"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
@@ -324,8 +304,11 @@
"press"
]
},
"page": {
"description": "Reference into `pages[]`.",
"result": {
"$ref": "#/definitions/StepResult"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
@@ -339,48 +322,74 @@
]
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"scroll"
]
},
"result": {
"$ref": "#/definitions/StepResult"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
}
]
},
"StepEffect": {
"description": "Observed navigation after a step (objective fact only).",
"StepResult": {
"type": "object",
"required": [
"navigated_to"
"state"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"state": {
"type": "string"
}
}
},
"StopReason": {
"type": "string",
"enum": [
"user_finish",
"cli_stop"
]
},
"TargetDescriptor": {
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
"description": "Stable semantic handle for an interacted element within a page observation.",
"type": "object",
"required": [
"tag"
],
"properties": {
"ctx": {
"type": [
"string",
"null"
]
},
"name": {
"type": [
"string",
"null"
]
},
"name_attr": {
"type": [
"string",
"null"
]
},
"nearby_label": {
"type": [
"string",
"null"
]
},
"placeholder": {
"ref": {
"type": [
"string",
"null"
@@ -392,8 +401,8 @@
"null"
]
},
"tag": {
"type": "string"
"unmatched": {
"type": "boolean"
}
}
},
@@ -402,42 +411,54 @@
"type": "object",
"required": [
"entry",
"pages",
"recorded_at",
"steps"
"recorder",
"states",
"steps",
"stopped_by",
"version"
],
"properties": {
"entry": {
"$ref": "#/definitions/TraceEntry"
},
"pages": {
"type": "array",
"items": {
"$ref": "#/definitions/PageRef"
}
},
"purpose": {
"description": "Optional user-provided goal from `--purpose` (metadata only).",
"type": [
"string",
"null"
]
},
"recorded_at": {
"description": "RFC 3339 timestamp when recording stopped.",
"type": "string"
},
"recorder": {
"$ref": "#/definitions/RecorderInfo"
},
"started_at": {
"type": [
"string",
"null"
]
},
"states": {
"type": "array",
"items": {
"$ref": "#/definitions/TraceState"
}
},
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/Step"
}
},
"stopped_by": {
"$ref": "#/definitions/StopReason"
},
"version": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
}
}
},
@@ -452,6 +473,45 @@
"type": "string"
}
}
},
"TraceState": {
"description": "Page observation dictionary entry — referenced by steps via `state` / `result.state`.",
"type": "object",
"required": [
"id",
"url"
],
"properties": {
"body": {
"description": "Wire-only: full page observation (front matter + VOM body + annotations).",
"type": [
"string",
"null"
]
},
"id": {
"type": "string"
},
"page": {
"description": "Disk-only: filename under the bundle `pages/` directory.",
"type": [
"string",
"null"
]
},
"title": {
"type": [
"string",
"null"
]
},
"truncated": {
"type": "boolean"
},
"url": {
"type": "string"
}
}
}
}
}
@@ -6,12 +6,26 @@
"session_id"
],
"properties": {
"max_page_tokens": {
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 0.0
},
"purpose": {
"type": [
"string",
"null"
]
},
"redact_values": {
"type": [
"boolean",
"null"
]
},
"session_id": {
"type": "string"
},
@@ -11,6 +11,14 @@
}
},
"definitions": {
"FillCommit": {
"type": "string",
"enum": [
"enter",
"suggestion",
"blur"
]
},
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
@@ -21,25 +29,32 @@
"shift"
]
},
"PageRef": {
"description": "Page context dictionary entry — referenced by steps via `page` id.",
"NavigationCause": {
"type": "string",
"enum": [
"user_typed",
"link",
"form_submit",
"reload",
"history",
"script",
"browser"
]
},
"RecorderInfo": {
"type": "object",
"required": [
"id",
"url"
"bsk",
"vom"
],
"properties": {
"id": {
"bsk": {
"type": "string"
},
"title": {
"type": [
"string",
"null"
]
},
"url": {
"type": "string"
"vom": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
}
}
},
@@ -68,21 +83,16 @@
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"cause",
"id",
"op",
"page",
"result",
"state",
"to"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
"cause": {
"$ref": "#/definitions/NavigationCause"
},
"id": {
"type": "integer",
@@ -95,8 +105,11 @@
"navigate"
]
},
"page": {
"description": "Reference into `pages[]`.",
"result": {
"$ref": "#/definitions/StepResult"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"to": {
@@ -110,20 +123,11 @@
"required": [
"id",
"op",
"page",
"result",
"state",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
@@ -135,8 +139,11 @@
"click"
]
},
"page": {
"description": "Reference into `pages[]`.",
"result": {
"$ref": "#/definitions/StepResult"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
@@ -150,20 +157,11 @@
"required": [
"id",
"op",
"page",
"result",
"state",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
@@ -175,8 +173,11 @@
"hover"
]
},
"page": {
"description": "Reference into `pages[]`.",
"result": {
"$ref": "#/definitions/StepResult"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
@@ -188,22 +189,17 @@
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"commit",
"id",
"op",
"page",
"result",
"state",
"target",
"value"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
"commit": {
"$ref": "#/definitions/FillCommit"
},
"id": {
"type": "integer",
@@ -216,15 +212,15 @@
"fill"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"redacted": {
"type": [
"boolean",
"null"
]
"type": "boolean"
},
"result": {
"$ref": "#/definitions/StepResult"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
@@ -240,21 +236,11 @@
"required": [
"id",
"op",
"page",
"selection",
"result",
"state",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
@@ -266,9 +252,8 @@
"select"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
"result": {
"$ref": "#/definitions/StepResult"
},
"selection": {
"type": "array",
@@ -276,6 +261,10 @@
"$ref": "#/definitions/SelectedOption"
}
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
}
@@ -288,19 +277,10 @@
"id",
"key",
"op",
"page"
"result",
"state"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
@@ -324,8 +304,11 @@
"press"
]
},
"page": {
"description": "Reference into `pages[]`.",
"result": {
"$ref": "#/definitions/StepResult"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
@@ -339,48 +322,74 @@
]
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"scroll"
]
},
"result": {
"$ref": "#/definitions/StepResult"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
}
]
},
"StepEffect": {
"description": "Observed navigation after a step (objective fact only).",
"StepResult": {
"type": "object",
"required": [
"navigated_to"
"state"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"state": {
"type": "string"
}
}
},
"StopReason": {
"type": "string",
"enum": [
"user_finish",
"cli_stop"
]
},
"TargetDescriptor": {
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
"description": "Stable semantic handle for an interacted element within a page observation.",
"type": "object",
"required": [
"tag"
],
"properties": {
"ctx": {
"type": [
"string",
"null"
]
},
"name": {
"type": [
"string",
"null"
]
},
"name_attr": {
"type": [
"string",
"null"
]
},
"nearby_label": {
"type": [
"string",
"null"
]
},
"placeholder": {
"ref": {
"type": [
"string",
"null"
@@ -392,8 +401,8 @@
"null"
]
},
"tag": {
"type": "string"
"unmatched": {
"type": "boolean"
}
}
},
@@ -402,42 +411,54 @@
"type": "object",
"required": [
"entry",
"pages",
"recorded_at",
"steps"
"recorder",
"states",
"steps",
"stopped_by",
"version"
],
"properties": {
"entry": {
"$ref": "#/definitions/TraceEntry"
},
"pages": {
"type": "array",
"items": {
"$ref": "#/definitions/PageRef"
}
},
"purpose": {
"description": "Optional user-provided goal from `--purpose` (metadata only).",
"type": [
"string",
"null"
]
},
"recorded_at": {
"description": "RFC 3339 timestamp when recording stopped.",
"type": "string"
},
"recorder": {
"$ref": "#/definitions/RecorderInfo"
},
"started_at": {
"type": [
"string",
"null"
]
},
"states": {
"type": "array",
"items": {
"$ref": "#/definitions/TraceState"
}
},
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/Step"
}
},
"stopped_by": {
"$ref": "#/definitions/StopReason"
},
"version": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
}
}
},
@@ -452,6 +473,45 @@
"type": "string"
}
}
},
"TraceState": {
"description": "Page observation dictionary entry — referenced by steps via `state` / `result.state`.",
"type": "object",
"required": [
"id",
"url"
],
"properties": {
"body": {
"description": "Wire-only: full page observation (front matter + VOM body + annotations).",
"type": [
"string",
"null"
]
},
"id": {
"type": "string"
},
"page": {
"description": "Disk-only: filename under the bundle `pages/` directory.",
"type": [
"string",
"null"
]
},
"title": {
"type": [
"string",
"null"
]
},
"truncated": {
"type": "boolean"
},
"url": {
"type": "string"
}
}
}
}
}
+271 -130
View File
@@ -5,45 +5,146 @@
"type": "object",
"required": [
"entry",
"pages",
"recorded_at",
"steps"
"recorder",
"states",
"steps",
"stopped_by",
"version"
],
"properties": {
"entry": {
"$ref": "#/definitions/TraceEntry"
},
"pages": {
"type": "array",
"items": {
"$ref": "#/definitions/PageRef"
}
"frame_capture": {
"anyOf": [
{
"$ref": "#/definitions/FrameCaptureInfo"
},
{
"type": "null"
}
]
},
"purpose": {
"description": "Optional user-provided goal from `--purpose` (metadata only).",
"type": [
"string",
"null"
]
},
"recorded_at": {
"description": "RFC 3339 timestamp when recording stopped.",
"type": "string"
},
"recorder": {
"$ref": "#/definitions/RecorderInfo"
},
"started_at": {
"type": [
"string",
"null"
]
},
"states": {
"type": "array",
"items": {
"$ref": "#/definitions/TraceState"
}
},
"steps": {
"type": "array",
"items": {
"$ref": "#/definitions/Step"
}
},
"stopped_by": {
"$ref": "#/definitions/StopReason"
},
"version": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
}
},
"definitions": {
"FillCommit": {
"type": "string",
"enum": [
"enter",
"suggestion",
"blur"
]
},
"FrameCaptureFailure": {
"type": "object",
"required": [
"frame_id",
"reason"
],
"properties": {
"detail": {
"type": [
"string",
"null"
]
},
"document_id": {
"type": [
"string",
"null"
]
},
"frame_id": {
"type": "integer",
"format": "int64"
},
"reason": {
"$ref": "#/definitions/FrameCaptureFailureReason"
},
"url": {
"type": [
"string",
"null"
]
}
}
},
"FrameCaptureFailureReason": {
"type": "string",
"enum": [
"not_injectable",
"arm_failed",
"rearm_failed",
"drain_failed",
"flush_failed"
]
},
"FrameCaptureInfo": {
"type": "object",
"required": [
"status"
],
"properties": {
"failures": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/FrameCaptureFailure"
}
},
"status": {
"$ref": "#/definitions/FrameCaptureStatus"
}
}
},
"FrameCaptureStatus": {
"type": "string",
"enum": [
"complete",
"partial"
]
},
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
@@ -54,25 +155,32 @@
"shift"
]
},
"PageRef": {
"description": "Page context dictionary entry — referenced by steps via `page` id.",
"NavigationCause": {
"type": "string",
"enum": [
"user_typed",
"link",
"form_submit",
"reload",
"history",
"script",
"browser"
]
},
"RecorderInfo": {
"type": "object",
"required": [
"id",
"url"
"bsk",
"vom"
],
"properties": {
"id": {
"bsk": {
"type": "string"
},
"title": {
"type": [
"string",
"null"
]
},
"url": {
"type": "string"
"vom": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
}
}
},
@@ -101,21 +209,16 @@
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"cause",
"id",
"op",
"page",
"result",
"state",
"to"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
"cause": {
"$ref": "#/definitions/NavigationCause"
},
"id": {
"type": "integer",
@@ -128,8 +231,11 @@
"navigate"
]
},
"page": {
"description": "Reference into `pages[]`.",
"result": {
"$ref": "#/definitions/StepResult"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"to": {
@@ -143,20 +249,11 @@
"required": [
"id",
"op",
"page",
"result",
"state",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
@@ -168,8 +265,11 @@
"click"
]
},
"page": {
"description": "Reference into `pages[]`.",
"result": {
"$ref": "#/definitions/StepResult"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
@@ -183,20 +283,11 @@
"required": [
"id",
"op",
"page",
"result",
"state",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
@@ -208,8 +299,11 @@
"hover"
]
},
"page": {
"description": "Reference into `pages[]`.",
"result": {
"$ref": "#/definitions/StepResult"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
@@ -221,22 +315,17 @@
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"commit",
"id",
"op",
"page",
"result",
"state",
"target",
"value"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
"commit": {
"$ref": "#/definitions/FillCommit"
},
"id": {
"type": "integer",
@@ -249,15 +338,15 @@
"fill"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"redacted": {
"type": [
"boolean",
"null"
]
"type": "boolean"
},
"result": {
"$ref": "#/definitions/StepResult"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
@@ -273,21 +362,11 @@
"required": [
"id",
"op",
"page",
"selection",
"result",
"state",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
@@ -299,9 +378,8 @@
"select"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
"result": {
"$ref": "#/definitions/StepResult"
},
"selection": {
"type": "array",
@@ -309,6 +387,10 @@
"$ref": "#/definitions/SelectedOption"
}
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
}
@@ -321,19 +403,10 @@
"id",
"key",
"op",
"page"
"result",
"state"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
@@ -357,8 +430,11 @@
"press"
]
},
"page": {
"description": "Reference into `pages[]`.",
"result": {
"$ref": "#/definitions/StepResult"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
@@ -372,48 +448,74 @@
]
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"scroll"
]
},
"result": {
"$ref": "#/definitions/StepResult"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
}
]
},
"StepEffect": {
"description": "Observed navigation after a step (objective fact only).",
"StepResult": {
"type": "object",
"required": [
"navigated_to"
"state"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"state": {
"type": "string"
}
}
},
"StopReason": {
"type": "string",
"enum": [
"user_finish",
"cli_stop"
]
},
"TargetDescriptor": {
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
"description": "Stable semantic handle for an interacted element within a page observation.",
"type": "object",
"required": [
"tag"
],
"properties": {
"ctx": {
"type": [
"string",
"null"
]
},
"name": {
"type": [
"string",
"null"
]
},
"name_attr": {
"type": [
"string",
"null"
]
},
"nearby_label": {
"type": [
"string",
"null"
]
},
"placeholder": {
"ref": {
"type": [
"string",
"null"
@@ -425,8 +527,8 @@
"null"
]
},
"tag": {
"type": "string"
"unmatched": {
"type": "boolean"
}
}
},
@@ -441,6 +543,45 @@
"type": "string"
}
}
},
"TraceState": {
"description": "Page observation dictionary entry — referenced by steps via `state` / `result.state`.",
"type": "object",
"required": [
"id",
"url"
],
"properties": {
"body": {
"description": "Wire-only: full page observation (front matter + VOM body + annotations).",
"type": [
"string",
"null"
]
},
"id": {
"type": "string"
},
"page": {
"description": "Disk-only: filename under the bundle `pages/` directory.",
"type": [
"string",
"null"
]
},
"title": {
"type": [
"string",
"null"
]
},
"truncated": {
"type": "boolean"
},
"url": {
"type": "string"
}
}
}
}
}
+115 -108
View File
@@ -7,21 +7,16 @@
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"cause",
"id",
"op",
"page",
"result",
"state",
"to"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
"cause": {
"$ref": "#/definitions/NavigationCause"
},
"id": {
"type": "integer",
@@ -34,8 +29,11 @@
"navigate"
]
},
"page": {
"description": "Reference into `pages[]`.",
"result": {
"$ref": "#/definitions/StepResult"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"to": {
@@ -49,20 +47,11 @@
"required": [
"id",
"op",
"page",
"result",
"state",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
@@ -74,8 +63,11 @@
"click"
]
},
"page": {
"description": "Reference into `pages[]`.",
"result": {
"$ref": "#/definitions/StepResult"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
@@ -89,20 +81,11 @@
"required": [
"id",
"op",
"page",
"result",
"state",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
@@ -114,8 +97,11 @@
"hover"
]
},
"page": {
"description": "Reference into `pages[]`.",
"result": {
"$ref": "#/definitions/StepResult"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
@@ -127,22 +113,17 @@
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"commit",
"id",
"op",
"page",
"result",
"state",
"target",
"value"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
"commit": {
"$ref": "#/definitions/FillCommit"
},
"id": {
"type": "integer",
@@ -155,15 +136,15 @@
"fill"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"redacted": {
"type": [
"boolean",
"null"
]
"type": "boolean"
},
"result": {
"$ref": "#/definitions/StepResult"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
@@ -179,21 +160,11 @@
"required": [
"id",
"op",
"page",
"selection",
"result",
"state",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
@@ -205,9 +176,8 @@
"select"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
"result": {
"$ref": "#/definitions/StepResult"
},
"selection": {
"type": "array",
@@ -215,6 +185,10 @@
"$ref": "#/definitions/SelectedOption"
}
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
}
@@ -227,19 +201,10 @@
"id",
"key",
"op",
"page"
"result",
"state"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
@@ -263,8 +228,11 @@
"press"
]
},
"page": {
"description": "Reference into `pages[]`.",
"result": {
"$ref": "#/definitions/StepResult"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
},
"target": {
@@ -278,9 +246,47 @@
]
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"scroll"
]
},
"result": {
"$ref": "#/definitions/StepResult"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
}
],
"definitions": {
"FillCommit": {
"type": "string",
"enum": [
"enter",
"suggestion",
"blur"
]
},
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
@@ -291,6 +297,18 @@
"shift"
]
},
"NavigationCause": {
"type": "string",
"enum": [
"user_typed",
"link",
"form_submit",
"reload",
"history",
"script",
"browser"
]
},
"SelectedOption": {
"description": "One selected option (`select` op).",
"type": "object",
@@ -309,45 +327,34 @@
}
}
},
"StepEffect": {
"description": "Observed navigation after a step (objective fact only).",
"StepResult": {
"type": "object",
"required": [
"navigated_to"
"state"
],
"properties": {
"navigated_to": {
"description": "Reference into `pages[]` for the destination page.",
"state": {
"type": "string"
}
}
},
"TargetDescriptor": {
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
"description": "Stable semantic handle for an interacted element within a page observation.",
"type": "object",
"required": [
"tag"
],
"properties": {
"ctx": {
"type": [
"string",
"null"
]
},
"name": {
"type": [
"string",
"null"
]
},
"name_attr": {
"type": [
"string",
"null"
]
},
"nearby_label": {
"type": [
"string",
"null"
]
},
"placeholder": {
"ref": {
"type": [
"string",
"null"
@@ -359,8 +366,8 @@
"null"
]
},
"tag": {
"type": "string"
"unmatched": {
"type": "boolean"
}
}
}
+291 -194
View File
@@ -1,35 +1,33 @@
//! Semantic user-action recording (`tool.record_start` / `stop` / `await`).
//!
//! Trace v2 is a **record-only** log of user actions: what was clicked, filled,
//! selected, and where navigation occurred. No LLM runs during recording, so
//! variable-vs-constant classification is **not** stored — executing agents
//! infer that at run time from raw values and control names.
//! Trace v3 is a **state-action-state** chain: each step binds to page
//! observations (VOM) captured before and after the action.
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use super::interaction::KeyModifier;
pub const TRACE_VERSION: u32 = 3;
pub const VOM_FORMAT_VERSION: u32 = 1;
// ---------------------------------------------------------------------------
// Target
// ---------------------------------------------------------------------------
/// Stable semantic handle for an interacted element.
///
/// `name` and `nearby_label` are **untrusted page text**.
/// Stable semantic handle for an interacted element within a page observation.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct TargetDescriptor {
#[serde(default, skip_serializing_if = "Option::is_none", rename = "ref")]
pub element_ref: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub role: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name: Option<String>,
pub tag: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub name_attr: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub placeholder: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub nearby_label: Option<String>,
pub ctx: Option<String>,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub unmatched: bool,
}
// ---------------------------------------------------------------------------
@@ -42,15 +40,54 @@ pub struct TraceEntry {
pub start_url: String,
}
/// Page context dictionary entry — referenced by steps via `page` id.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct PageRef {
pub struct RecorderInfo {
pub bsk: String,
pub vom: u32,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum StopReason {
UserFinish,
CliStop,
}
/// Page observation dictionary entry — referenced by steps via `state` / `result.state`.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct TraceState {
pub id: String,
pub url: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub title: Option<String>,
/// Wire-only: full page observation (front matter + VOM body + annotations).
#[serde(default, skip_serializing_if = "Option::is_none")]
pub body: Option<String>,
/// Disk-only: filename under the bundle `pages/` directory.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub page: Option<String>,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub truncated: bool,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct StepResult {
pub state: String,
}
/// Fields shared by every step variant (flattened in JSON).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct StepCommon {
pub id: u32,
/// Observation id immediately before this action.
pub state: String,
pub result: StepResult,
}
// ---------------------------------------------------------------------------
// Step op-specific payloads
// ---------------------------------------------------------------------------
/// One selected option (`select` op).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct SelectedOption {
@@ -59,27 +96,26 @@ pub struct SelectedOption {
pub label: Option<String>,
}
/// Observed navigation after a step (objective fact only).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct StepEffect {
/// Reference into `pages[]` for the destination page.
pub navigated_to: String,
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum NavigationCause {
UserTyped,
Link,
FormSubmit,
Reload,
History,
Script,
Browser,
}
/// Fields shared by every step variant (flattened in JSON).
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct StepCommon {
pub id: u32,
/// Reference into `pages[]`.
pub page: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub effect: Option<StepEffect>,
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum FillCommit {
Enter,
Suggestion,
Blur,
}
// ---------------------------------------------------------------------------
// Step op-specific payloads
// ---------------------------------------------------------------------------
/// One recorded user action — discriminated union by `op`.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
#[serde(tag = "op", rename_all = "snake_case")]
@@ -88,6 +124,7 @@ pub enum Step {
#[serde(flatten)]
common: StepCommon,
to: String,
cause: NavigationCause,
},
Click {
#[serde(flatten)]
@@ -104,13 +141,15 @@ pub enum Step {
common: StepCommon,
target: TargetDescriptor,
value: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
redacted: Option<bool>,
commit: FillCommit,
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
redacted: bool,
},
Select {
#[serde(flatten)]
common: StepCommon,
target: TargetDescriptor,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
selection: Vec<SelectedOption>,
},
Press {
@@ -122,6 +161,50 @@ pub enum Step {
#[serde(default, skip_serializing_if = "Option::is_none")]
target: Option<TargetDescriptor>,
},
Scroll {
#[serde(flatten)]
common: StepCommon,
},
}
// ---------------------------------------------------------------------------
// Frame capture metadata
// ---------------------------------------------------------------------------
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum FrameCaptureStatus {
Complete,
Partial,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum FrameCaptureFailureReason {
NotInjectable,
ArmFailed,
RearmFailed,
DrainFailed,
FlushFailed,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct FrameCaptureFailure {
pub reason: FrameCaptureFailureReason,
pub frame_id: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub document_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub url: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub detail: Option<String>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct FrameCaptureInfo {
pub status: FrameCaptureStatus,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub failures: Option<Vec<FrameCaptureFailure>>,
}
// ---------------------------------------------------------------------------
@@ -131,15 +214,18 @@ pub enum Step {
/// Persisted user-action trace exported by `tool.record_stop` / `await`.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct Trace {
/// RFC 3339 timestamp when recording stopped.
pub recorded_at: String,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub started_at: Option<String>,
/// Optional user-provided goal from `--purpose` (metadata only).
pub version: u32,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub purpose: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub started_at: Option<String>,
pub recorded_at: String,
pub stopped_by: StopReason,
pub entry: TraceEntry,
pub pages: Vec<PageRef>,
pub recorder: RecorderInfo,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub frame_capture: Option<FrameCaptureInfo>,
pub states: Vec<TraceState>,
pub steps: Vec<Step>,
}
@@ -156,6 +242,10 @@ pub struct RecordStartParams {
pub url: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub purpose: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub max_page_tokens: Option<u32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub redact_values: Option<bool>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
@@ -195,81 +285,80 @@ mod tests {
use super::*;
use serde_json::json;
fn sample_common(id: u32) -> StepCommon {
fn sample_common(id: u32, state: &str, result_state: &str) -> StepCommon {
StepCommon {
id,
page: "p1".into(),
effect: None,
state: state.into(),
result: StepResult {
state: result_state.into(),
},
}
}
fn sample_target() -> TargetDescriptor {
TargetDescriptor {
element_ref: Some("e21".into()),
role: Some("button".into()),
name: Some("发布".into()),
tag: "button".into(),
name_attr: None,
placeholder: None,
nearby_label: None,
ctx: Some("金桔柠檬 6 号".into()),
unmatched: false,
}
}
fn sample_trace() -> Trace {
Trace {
recorded_at: "2026-07-17T09:01:10Z".into(),
started_at: Some("2026-07-17T09:00:00Z".into()),
purpose: Some("发布一篇文章".into()),
version: TRACE_VERSION,
purpose: Some("把草稿商品发布上架".into()),
started_at: Some("2026-08-10T02:10:41.080Z".into()),
recorded_at: "2026-08-10T02:12:55.360Z".into(),
stopped_by: StopReason::UserFinish,
entry: TraceEntry {
start_url: "https://x.com/editor".into(),
start_url: "https://example.com/".into(),
},
pages: vec![
PageRef {
id: "p1".into(),
url: "https://x.com/editor".into(),
title: Some("写文章".into()),
recorder: RecorderInfo {
bsk: "0.1.10".into(),
vom: VOM_FORMAT_VERSION,
},
frame_capture: None,
states: vec![
TraceState {
id: "s1".into(),
url: "https://example.com/".into(),
title: Some("Example Domain".into()),
body: None,
page: Some("s1.vom.txt".into()),
truncated: false,
},
PageRef {
id: "p2".into(),
url: "https://x.com/p/99".into(),
title: None,
TraceState {
id: "s2".into(),
url: "https://shop.example.com/products?status=draft".into(),
title: Some("商品管理".into()),
body: None,
page: Some("s2.vom.txt".into()),
truncated: false,
},
],
steps: vec![
Step::Fill {
common: sample_common(1),
target: TargetDescriptor {
role: Some("textbox".into()),
name: Some("标题".into()),
tag: "input".into(),
name_attr: None,
placeholder: None,
nearby_label: None,
},
value: "我的第一篇文章".into(),
redacted: None,
Step::Navigate {
common: sample_common(1, "s1", "s2"),
to: "https://shop.example.com/products?status=draft".into(),
cause: NavigationCause::UserTyped,
},
Step::Select {
common: sample_common(3),
Step::Fill {
common: sample_common(2, "s2", "s2"),
target: TargetDescriptor {
role: Some("combobox".into()),
name: Some("分类".into()),
tag: "select".into(),
name_attr: None,
placeholder: None,
nearby_label: None,
element_ref: Some("e12".into()),
role: Some("textbox".into()),
name: Some("搜索商品".into()),
ctx: None,
unmatched: false,
},
selection: vec![SelectedOption {
value: "tech".into(),
label: Some("技术分享".into()),
}],
value: "金桔柠檬".into(),
commit: FillCommit::Enter,
redacted: false,
},
Step::Click {
common: StepCommon {
effect: Some(StepEffect {
navigated_to: "p2".into(),
}),
..sample_common(4)
},
common: sample_common(3, "s2", "s2"),
target: sample_target(),
},
],
@@ -279,35 +368,37 @@ mod tests {
#[test]
fn step_click_round_trips() {
let step = Step::Click {
common: sample_common(1),
common: sample_common(1, "s1", "s2"),
target: sample_target(),
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v.get("op").and_then(|v| v.as_str()), Some("click"));
assert_eq!(v.get("page").and_then(|v| v.as_str()), Some("p1"));
assert!(v.get("key").is_none());
assert_eq!(v.get("state").and_then(|v| v.as_str()), Some("s1"));
assert_eq!(v["result"]["state"], "s2");
assert_eq!(v["target"]["ref"], "e21");
let round: Step = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn step_fill_with_raw_value_round_trips() {
fn step_fill_with_commit_round_trips() {
let step = Step::Fill {
common: sample_common(2),
common: sample_common(2, "s2", "s3"),
target: TargetDescriptor {
element_ref: Some("e12".into()),
role: Some("textbox".into()),
name: Some("服务名称".into()),
tag: "input".into(),
name_attr: Some("serviceName".into()),
placeholder: None,
nearby_label: None,
name: Some("搜索商品".into()),
ctx: None,
unmatched: false,
},
value: "my-svc".into(),
redacted: None,
value: "browser skill".into(),
commit: FillCommit::Enter,
redacted: false,
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["op"], "fill");
assert_eq!(v["value"], "my-svc");
assert_eq!(v["commit"], "enter");
assert!(v.get("redacted").is_none());
let round: Step = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
@@ -315,17 +406,17 @@ mod tests {
#[test]
fn step_fill_password_is_redacted() {
let step = Step::Fill {
common: sample_common(1),
common: sample_common(1, "s1", "s1"),
target: TargetDescriptor {
element_ref: Some("e3".into()),
role: Some("textbox".into()),
name: Some("密码".into()),
tag: "input".into(),
name_attr: None,
placeholder: None,
nearby_label: None,
ctx: None,
unmatched: false,
},
value: "***".into(),
redacted: Some(true),
commit: FillCommit::Blur,
redacted: true,
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["value"], "***");
@@ -333,131 +424,137 @@ mod tests {
}
#[test]
fn step_select_uses_object_array() {
let step = Step::Select {
common: sample_common(3),
target: TargetDescriptor {
role: Some("combobox".into()),
name: Some("分类".into()),
tag: "select".into(),
name_attr: None,
placeholder: None,
nearby_label: None,
},
selection: vec![SelectedOption {
value: "tech".into(),
label: Some("技术分享".into()),
}],
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["selection"][0]["value"], "tech");
assert_eq!(v["selection"][0]["label"], "技术分享");
let round: Step = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn step_navigate_round_trips() {
fn step_navigate_with_cause_round_trips() {
let step = Step::Navigate {
common: sample_common(1),
common: sample_common(1, "s1", "s2"),
to: "https://example.com".into(),
cause: NavigationCause::UserTyped,
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["op"], "navigate");
assert_eq!(v["to"], "https://example.com");
assert_eq!(v["cause"], "user_typed");
let round: Step = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn step_press_with_modifiers_round_trips() {
let step = Step::Press {
common: StepCommon {
effect: Some(StepEffect {
navigated_to: "p2".into(),
}),
..sample_common(2)
},
key: "Enter".into(),
modifiers: Some(vec![KeyModifier::Ctrl, KeyModifier::Shift]),
target: Some(sample_target()),
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["key"], "Enter");
assert_eq!(v["modifiers"], json!(["ctrl", "shift"]));
assert_eq!(v["effect"]["navigated_to"], "p2");
let round: Step = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn trace_record_only_round_trips() {
fn trace_v3_round_trips() {
let trace = sample_trace();
let v = serde_json::to_value(&trace).unwrap();
assert!(v.get("version").is_none());
assert_eq!(
v.get("purpose").and_then(|v| v.as_str()),
Some("发布一篇文章")
);
assert!(v.get("parameters").is_none());
assert!(v.get("site").is_none());
assert!(v.get("goal").is_none());
assert_eq!(
v.get("entry")
.and_then(|e| e.get("start_url"))
.and_then(|u| u.as_str()),
Some("https://x.com/editor")
);
assert_eq!(v.get("version").and_then(|v| v.as_u64()), Some(3));
assert!(v.get("pages").is_none());
assert_eq!(v["recorder"]["vom"], 1);
assert_eq!(v["states"].as_array().unwrap().len(), 2);
let round: Trace = serde_json::from_value(v).unwrap();
assert_eq!(round, trace);
}
#[test]
fn discriminated_union_click_ignores_extra_key_field() {
let bad = json!({
"op": "click",
"id": 1,
"page": "p1",
"target": { "tag": "button", "name": "OK" },
"key": "Enter"
});
let step: Step = serde_json::from_value(bad).unwrap();
assert!(matches!(step, Step::Click { .. }));
fn default_fields_are_omitted() {
let step = Step::Click {
common: sample_common(1, "s1", "s1"),
target: TargetDescriptor {
element_ref: Some("e1".into()),
role: Some("button".into()),
name: Some("OK".into()),
ctx: None,
unmatched: false,
},
};
let v = serde_json::to_value(&step).unwrap();
assert!(v.get("key").is_none());
assert!(v.get("unmatched").is_none());
assert!(v["target"].get("ctx").is_none());
assert!(v["target"].get("unmatched").is_none());
}
#[test]
fn unmatched_target_serializes_flag() {
let step = Step::Click {
common: sample_common(1, "s1", "s2"),
target: TargetDescriptor {
element_ref: None,
role: Some("button".into()),
name: Some("发布".into()),
ctx: None,
unmatched: true,
},
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["target"]["unmatched"], true);
assert!(v["target"].get("ref").is_none());
}
#[test]
fn extension_trace_deserializes() {
let v = json!({
"version": 3,
"recorded_at": "2026-07-21T08:00:00Z",
"started_at": "2026-07-21T07:59:00Z",
"purpose": "demo",
"stopped_by": "user_finish",
"entry": { "start_url": "https://example.com/editor" },
"pages": [
{ "id": "p1", "url": "https://example.com/editor" },
{ "id": "p2", "url": "https://example.com/p/99" }
"recorder": { "bsk": "0.1.10", "vom": 1 },
"states": [
{ "id": "s1", "url": "https://example.com/editor", "page": "s1.vom.txt" },
{ "id": "s2", "url": "https://example.com/p/99", "page": "s2.vom.txt" }
],
"steps": [
{
"op": "fill",
"id": 1,
"page": "p1",
"target": { "tag": "input", "role": "textbox", "name": "标题" },
"value": "hello"
"state": "s1",
"result": { "state": "s1" },
"target": { "ref": "e1", "role": "textbox", "name": "标题" },
"value": "hello",
"commit": "blur"
},
{
"op": "click",
"id": 2,
"page": "p1",
"target": { "tag": "button", "role": "button", "name": "发布" },
"effect": { "navigated_to": "p2" }
"state": "s1",
"result": { "state": "s2" },
"target": { "ref": "e2", "role": "button", "name": "发布" }
}
]
});
let trace: Trace = serde_json::from_value(v).unwrap();
assert_eq!(trace.entry.start_url, "https://example.com/editor");
assert_eq!(trace.pages.len(), 2);
assert_eq!(trace.version, 3);
assert_eq!(trace.states.len(), 2);
assert_eq!(trace.steps.len(), 2);
}
#[test]
fn trace_deserializes_optional_frame_capture_metadata() {
let v = json!({
"version": 3,
"recorded_at": "2026-07-21T08:00:00Z",
"stopped_by": "cli_stop",
"entry": { "start_url": "https://example.com/editor" },
"recorder": { "bsk": "0.1.10", "vom": 1 },
"frame_capture": {
"status": "partial",
"failures": [
{
"reason": "not_injectable",
"frame_id": 2,
"url": "chrome://settings",
"detail": "restricted frame url"
}
]
},
"states": [],
"steps": []
});
let trace: Trace = serde_json::from_value(v).unwrap();
let frame_capture = trace.frame_capture.expect("frame_capture");
assert_eq!(frame_capture.status, FrameCaptureStatus::Partial);
let failures = frame_capture.failures.expect("failures");
assert_eq!(failures.len(), 1);
assert_eq!(
failures[0].reason,
FrameCaptureFailureReason::NotInjectable
);
assert_eq!(failures[0].frame_id, 2);
assert_eq!(failures[0].url.as_deref(), Some("chrome://settings"));
}
}
@@ -6,20 +6,20 @@
"disconnected": "Disconnected",
"connecting": "Connecting…",
"connected": "Connected",
"version_skew": "Protocol mismatch",
"version_skew": "Version mismatch",
"disabled": "Connection off"
},
"stateDetail": {
"disconnected": "Open BrowserSkill to connect.",
"connecting": "Connecting to BrowserSkill…",
"connected": "Ready for automation tasks.",
"version_skew": "Some features may be unavailable."
"version_skew": "bsk and the browser extension differ in version. You can continue using them, but updating is recommended."
},
"stateBadge": {
"disconnected": "Standby",
"connecting": "Igniting",
"connected": "Ready",
"version_skew": "Action needed",
"version_skew": "Update recommended",
"disabled": "Off"
},
"connectionToggleTitle": "BrowserSkill connection",
@@ -27,6 +27,7 @@
"controlHintsToggleHint": "Show the status pill and orange glow while the Agent controls a page.",
"controlHintsInfoLabel": "About control hints",
"versionSkewWarning": "Extension protocol v{{extensionProtocol}}, daemon v{{daemonProtocol}}.",
"versionIncompatibleError": "bsk and the BrowserSkill extension are incompatible. Update both, reload the extension, then run bsk daemon restart.",
"upgradeAvailable": "Upgradable",
"launcher": {
"title": "Quick actions"
@@ -48,7 +49,7 @@
"record": {
"sectionTitle": "Action recording",
"cardDesc": "Record your actions for Agent reference",
"sectionHint": "Generates a recording instruction — send it to your Agent to start recording your actions; the Agent can then use the recording to automate similar tasks.",
"sectionHint": "Generates a recording instruction — send it to your Agent to start recording your actions; the Agent can then use the resulting trace bundle to automate similar tasks.",
"topicLabel": "Topic (optional)",
"topicPlaceholder": "e.g. Publish an article",
"startUrlLabel": "Starting URL (optional)",
@@ -56,7 +57,7 @@
"copyButton": "Copy recording instructions",
"copied": "Copied",
"hintDisconnected": "Available when connected",
"promptTemplate": "Use BrowserSkill's `bsk` CLI to record the browser actions I'm about to demonstrate.\n\nSteps:\n1. Run: {{command}}\n2. Once you confirm it's started, I'll operate in the opened window myself.\n3. After I click Finish, read the returned trace.json and summarize the actions (pages + steps); don't re-run or \"verify\"."
"promptTemplate": "Use BrowserSkill's `bsk` CLI to record the browser actions I'm about to demonstrate.\n\nSteps:\n1. Run: {{command}}\n2. Once you confirm it's started, I'll operate in the opened window myself.\n3. After I click Finish, read the written `./trace` bundle (`trace.json` + `pages/`) and summarize what I did."
}
},
"controlOverlay": {
@@ -89,6 +90,7 @@
},
"recordOverlay": {
"recording": "Recording your actions",
"finish": "Finish"
"finish": "Finish",
"partialCapture": "Some embedded frames may not be recorded."
}
}
@@ -6,20 +6,20 @@
"disconnected": "未连接",
"connecting": "连接中…",
"connected": "已连接",
"version_skew": "协议不一致",
"version_skew": "版本不一致",
"disabled": "连接已关闭"
},
"stateDetail": {
"disconnected": "请先打开 BrowserSkill。",
"connecting": "正在连接 BrowserSkill…",
"connected": "可接收自动化任务。",
"version_skew": "部分能力可能不可用。"
"version_skew": "bsk 与浏览器扩展版本不同,当前仍可使用,建议更新至最新版本。"
},
"stateBadge": {
"disconnected": "Standby",
"connecting": "Igniting",
"connected": "Ready",
"version_skew": "Action needed",
"version_skew": "建议更新",
"disabled": "Off"
},
"connectionToggleTitle": "BrowserSkill 连接",
@@ -27,6 +27,7 @@
"controlHintsToggleHint": "Agent 控制页面时显示提示条和橙色闪光。",
"controlHintsInfoLabel": "控制提示说明",
"versionSkewWarning": "扩展协议 v{{extensionProtocol}},daemon 协议 v{{daemonProtocol}}。",
"versionIncompatibleError": "bsk 与 BrowserSkill 扩展版本不兼容。请将两者更新至最新版本,重新加载扩展后,再运行 bsk daemon restart。",
"upgradeAvailable": "可升级",
"launcher": {
"title": "快捷功能"
@@ -48,7 +49,7 @@
"record": {
"sectionTitle": "操作录制",
"cardDesc": "录制你的操作,供 Agent 参考",
"sectionHint": "生成一段录制指令,发给 Agent 后可以启动插件录制用户操作,之后 Agent 可以参考录制结果自动完成同类任务。",
"sectionHint": "生成一段录制指令,发给 Agent 后可以启动插件录制用户操作,之后 Agent 可以参考写出的 trace 目录包自动完成同类任务。",
"topicLabel": "操作主题(可选)",
"topicPlaceholder": "例如:发布一篇文章",
"startUrlLabel": "起始网址(可选)",
@@ -56,7 +57,7 @@
"copyButton": "复制录制指令",
"copied": "已复制",
"hintDisconnected": "连接后可用",
"promptTemplate": "用 BrowserSkill 的 `bsk` CLI 录制我接下来的浏览器操作。\n\n步骤:\n1. 执行:{{command}}\n2. 你确认已开始后,我会在打开的窗口里自己操作。\n3. 我点“结束”后,读取返回的 trace.json 并总结操作(pages + steps),不要重跑或“验证”。"
"promptTemplate": "用 BrowserSkill 的 `bsk` CLI 录制我接下来的浏览器操作。\n\n步骤:\n1. 执行:{{command}}\n2. 你确认已开始后,我会在打开的窗口里自己操作。\n3. 我点“结束”后,读取写出的 `./trace` 目录包(`trace.json` + `pages/`),总结我做了什么。"
}
},
"controlOverlay": {
@@ -89,6 +90,7 @@
},
"recordOverlay": {
"recording": "正在录制你的操作",
"finish": "结束"
"finish": "结束",
"partialCapture": "部分嵌入页面可能未被录制。"
}
}
+39 -14
View File
@@ -24,6 +24,11 @@ function scene(nodes: VomNode[]): VomScene {
};
}
/** Compare refs without recording-metadata fields (role/name/ctx/line). */
function coreRefs(refs: ReturnType<typeof renderVom>["refs"]) {
return refs.map(({ ref: id, backendNodeId }) => ({ ref: id, backendNodeId }));
}
describe("renderVom single-layer page", () => {
it("always emits a complete @vom single-layer document when there is no blocker", () => {
const out = renderVom(
@@ -51,7 +56,7 @@ describe("renderVom single-layer page", () => {
' @e2 button "加入购物车"',
].join("\n"),
);
expect(out.refs).toEqual([
expect(coreRefs(out.refs)).toEqual([
{ ref: "e1", backendNodeId: 3 },
{ ref: "e2", backendNodeId: 6 },
]);
@@ -72,7 +77,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).toContain(' heading "Title"');
expect(out.text).toContain(' img "Logo"');
expect(out.text).toContain(' @e1 button "Continue"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
});
it("assigns refs to listbox controls", () => {
@@ -84,7 +89,7 @@ describe("renderVom single-layer page", () => {
);
expect(out.text).toContain(' @e1 listbox "Choices"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
});
it("annotates external links regardless of role casing", () => {
@@ -121,7 +126,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).toContain(" alertdialog");
expect(out.text).toContain(" section");
expect(out.text).toContain(' @e1 button "Close"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 5 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 5 }]);
});
it("skips generic nodes without adding an extra indentation level", () => {
@@ -192,7 +197,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).toContain(" section");
expect(out.text).not.toContain("Too deep");
expect(out.text).toContain(' @e1 button "Later"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
});
it("truncates when maxTokens is exceeded while keeping refs in sync with rendered text", () => {
@@ -227,7 +232,7 @@ describe("renderVom single-layer page", () => {
);
expect(out.text).toContain('@e1 button "Open settings"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
});
it("keeps the deepest custom control when wrapper and child both look clickable", () => {
@@ -257,7 +262,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).not.toContain('@e1 button "Card"');
expect(out.text).toContain('@e1 button "Details"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]);
});
it("does not recover clickable wrappers around native controls", () => {
@@ -280,7 +285,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).not.toContain('@e1 button "Checkout"');
expect(out.text).toContain('@e1 button "Pay"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]);
});
it("adds context to duplicate weak action labels", () => {
@@ -343,7 +348,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).toContain("@layers 1 focus=L1");
expect(out.text).toContain('@e1 button "Background"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
});
it("can filter refs blocked by active positioned regions without changing layer output", () => {
@@ -386,7 +391,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).not.toContain("active-region");
expect(out.text).not.toContain("Background");
expect(out.text).toContain('@e1 button "Foreground"');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
});
it("renders conditional surface items inline on the trigger", () => {
@@ -405,7 +410,7 @@ describe("renderVom single-layer page", () => {
});
expect(out.text).toContain('@e1 button "Products" [hover first: Shoes | Bags | Accessories]');
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
});
it("injects active scope blocks immediately after their trigger without refs", () => {
@@ -431,7 +436,7 @@ describe("renderVom single-layer page", () => {
" Bob - great sound",
].join("\n"),
);
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
});
it("does not render redundant children inside named ref controls", () => {
@@ -445,7 +450,7 @@ describe("renderVom single-layer page", () => {
expect(out.text).toContain('@e1 button "Save"');
expect(out.text).not.toContain("disk icon");
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
});
it("does not render redundant children inside native text inputs", () => {
@@ -612,7 +617,7 @@ describe("renderVom double-layer page", () => {
].join("\n"),
);
expect(out.text).not.toContain("底层按钮");
expect(out.refs).toEqual([
expect(coreRefs(out.refs)).toEqual([
{ ref: "e1", backendNodeId: 4 },
{ ref: "e2", backendNodeId: 5 },
]);
@@ -664,4 +669,24 @@ describe("renderVom double-layer page", () => {
expect(out.text).toContain(' @e2 textbox "密码" [filled] ="•••"');
expect(out.refs.map((r) => r.backendNodeId)).toEqual([101, 102]);
});
it("redactValues omits literal form values", () => {
const out = renderVom(
scene([
node({ id: 1, role: "RootWebArea", name: "Form" }),
node({
id: 2,
parentId: 1,
role: "textbox",
name: "邮箱",
value: "user@example.com",
inputState: "filled",
tag: "input",
}),
]),
{ redactValues: true },
);
expect(out.text).toContain("[filled]");
expect(out.text).not.toContain("user@example.com");
});
});
+1
View File
@@ -5,6 +5,7 @@ export type {
CondSurface,
LayerKind,
Rect,
RenderedRef,
Viewport,
VomNode,
VomOptions,
+22 -8
View File
@@ -4,6 +4,7 @@ import type {
BlockingLayer,
CondSurface,
Rect,
RenderedRef,
VomNode,
VomOptions,
VomResult,
@@ -606,6 +607,7 @@ function renderNodeLine(
ref: string | undefined,
context: string[] = [],
surface: CondSurface | undefined = undefined,
redactValues = false,
): string {
let line = `${" ".repeat(depth)}${ref ? `@${ref} ` : ""}${node.role}`;
@@ -627,11 +629,12 @@ function renderNodeLine(
}
const placeholder = cleaned(node.placeholder);
if (placeholder) line += ` placeholder=${JSON.stringify(placeholder)}`;
const value = node.sensitive
? rawValue !== undefined
? SENSITIVE_MASK
: undefined
: rawValue?.slice(0, MAX_VALUE_LENGTH);
const value =
node.sensitive || redactValues
? rawValue !== undefined
? SENSITIVE_MASK
: undefined
: rawValue?.slice(0, MAX_VALUE_LENGTH);
if (value !== undefined) line += ` =${JSON.stringify(value)}`;
if (surface && surface.subItems.length > 0) {
@@ -684,11 +687,12 @@ function shouldSkipRedundantChildren(node: VomNode, state: RenderState): boolean
interface RenderState {
lines: string[];
refs: Array<{ ref: string; backendNodeId: number }>;
refs: RenderedRef[];
nextRef: number;
tokens: number;
maxDepth: number;
maxTokens: number;
redactValues: boolean;
truncated: boolean;
stopped: boolean;
children: Map<number | null, VomNode[]>;
@@ -753,7 +757,7 @@ function renderTree(
const ref = isVomReferenceNode(node) ? `e${state.nextRef}` : undefined;
const context = ref ? handleContext(node, state) : [];
const surface = state.surfaceMap.get(node.id);
const line = renderNodeLine(node, depth, ref, context, surface);
const line = renderNodeLine(node, depth, ref, context, surface, state.redactValues);
const nextTokens = state.tokens + estimateTokens(line);
if (nextTokens > state.maxTokens) {
state.truncated = true;
@@ -764,7 +768,16 @@ function renderTree(
state.lines.push(line);
state.tokens = nextTokens;
if (ref) {
state.refs.push({ ref, backendNodeId: node.id });
const lineIndex = state.lines.length - 1;
const refName = cleaned(node.name);
state.refs.push({
ref,
backendNodeId: node.id,
role: node.role,
...(refName ? { name: refName } : {}),
...(context.length > 0 ? { ctx: context.join(" > ") } : {}),
line: lineIndex,
});
state.nextRef += 1;
}
@@ -793,6 +806,7 @@ function renderNodes(
tokens: estimateTokens(initialLines.join("\n")),
maxDepth: options.maxDepth ?? DEFAULT_MAX_DEPTH,
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
redactValues: options.redactValues ?? false,
truncated: false,
stopped: false,
children,
+15 -1
View File
@@ -59,6 +59,10 @@ export interface VomScene {
export interface VomOptions {
maxDepth?: number;
maxTokens?: number;
/**
* When true, all form values render as [filled]/[empty] without literal values.
*/
redactValues?: boolean;
/**
* Experimental: filter refs that are geometrically blocked by foreground
* fixed/absolute/sticky regions. Disabled by default and does not alter the
@@ -79,9 +83,19 @@ export interface ActiveScopeBlock {
lines: string[];
}
export interface RenderedRef {
ref: string;
backendNodeId: number;
role?: string;
name?: string;
ctx?: string;
/** Zero-based line index in `VomResult.text`. */
line: number;
}
export interface VomResult {
text: string;
refs: Array<{ ref: string; backendNodeId: number }>;
refs: RenderedRef[];
truncated: boolean;
}
+14 -6
View File
@@ -265,19 +265,27 @@ retry — complete the task autonomously or stop gracefully.
### Recording — `bsk record`
Capture the user's own actions in the Agent Window to a `trace.json`, for later LLM-driven automation:
Capture the user's own actions in the Agent Window to a **trace v3 bundle**, for later LLM-driven automation:
```bash
bsk record start --browser <instance-id-or-label> [--url https://…] [--purpose "publish a wiki doc"] [--output trace.json]
bsk record start --browser <instance-id-or-label> \
[--url https://…] [--purpose "publish a wiki doc"] \
[--max-page-tokens 3000] [--redact-values] \
[--output trace]
# `--url` is optional; default https://example.com/ when omitted (must be http(s)).
# Blocks until the user clicks Finish in the recording panel, then writes ./trace.json and closes the window.
# Blocks until the user clicks Finish in the recording panel, then writes:
# trace/trace.json — action chain + state index (version 3)
# trace/pages/ — one `.vom.txt` observe snapshot per settled page state
bsk record stop [--output trace.json] # terminal fallback if the browser panel is unavailable
bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable
```
- The trace is a **record-only action log** (a `pages[]` dictionary + `navigate`/`click`/`fill`/`select`/`press` steps with `target` descriptors). It records *what the user did*; deciding which inputs are variable is left to the executing agent.
- **Bundle layout:** `--output` is a directory (default `./trace`) containing `trace.json` and `pages/`. `trace.json` lists `states[]` (page observation ids) and `steps[]` (each step binds `state` = observe snapshot *before* the action and `result.state` = snapshot *after* settle). Page bodies live in `pages/sN.vom.txt` using the same VOM format as `bsk observe`.
- Each `states[]` entry is one **settled page observation** (captured after recording start, navigation landing, or action settle — not on a timer).
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
- `--purpose` is optional context metadata; it does **not** change what gets captured.
- There is **no** `bsk replay` — to redo a flow, read the trace and reuse the existing `session` / `snapshot` / `@eN` / `click` / `fill` tools. Follow **Stop when the goal is met**.
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
- Recording captures interactions in the **top frame and injectable child frames** (same-origin and cross-origin where the extension can attach). Embedded frames that cannot be injected are omitted; the trace may include `frame_capture.status: "partial"` with failure details.
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
## Error handling