diff --git a/Cargo.lock b/Cargo.lock
index dc8b442..a41ef9d 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -134,7 +134,7 @@ dependencies = [
[[package]]
name = "bsk"
-version = "0.1.7"
+version = "0.1.8"
dependencies = [
"anyhow",
"bsk-protocol",
@@ -168,7 +168,7 @@ dependencies = [
[[package]]
name = "bsk-protocol"
-version = "0.1.7"
+version = "0.1.8"
dependencies = [
"schemars",
"semver",
diff --git a/Cargo.toml b/Cargo.toml
index 6e41be9..b09d719 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -7,7 +7,7 @@ members = [
default-members = ["crates/bsk-cli"]
[workspace.package]
-version = "0.1.7"
+version = "0.1.8"
edition = "2024"
rust-version = "1.85"
license = "MIT"
diff --git a/apps/extension/package.json b/apps/extension/package.json
index d651876..87a92db 100644
--- a/apps/extension/package.json
+++ b/apps/extension/package.json
@@ -1,6 +1,6 @@
{
"name": "@browser-skill/extension",
- "version": "0.1.4",
+ "version": "0.1.5",
"private": true,
"type": "module",
"scripts": {
diff --git a/apps/extension/src/content/ControlOverlay.tsx b/apps/extension/src/content/ControlOverlay.tsx
index 51ed2f2..c6afc5b 100644
--- a/apps/extension/src/content/ControlOverlay.tsx
+++ b/apps/extension/src/content/ControlOverlay.tsx
@@ -58,6 +58,7 @@ export function ControlOverlay({
/>
{
if (automationBypass) return;
event.preventDefault();
@@ -80,13 +81,16 @@ export function ControlOverlay({
/>
void;
+}
+
+type Props = {
+ request: RecordRequestData | null;
+};
+
+export function RecordOverlay({ request }: Props) {
+ const { t } = useTranslation("extension");
+ const [show, setShow] = useState(false);
+
+ useEffect(() => {
+ if (request) {
+ const raf = requestAnimationFrame(() => setShow(true));
+ return () => cancelAnimationFrame(raf);
+ }
+ setShow(false);
+ }, [request]);
+
+ if (!request) return null;
+
+ return (
+ <>
+
+
+
+
+
+ {t("recordOverlay.recording")}
+
+
+
+ >
+ );
+}
diff --git a/apps/extension/src/content/__tests__/ControlOverlay.test.tsx b/apps/extension/src/content/__tests__/ControlOverlay.test.tsx
index b7aae0c..d35c609 100644
--- a/apps/extension/src/content/__tests__/ControlOverlay.test.tsx
+++ b/apps/extension/src/content/__tests__/ControlOverlay.test.tsx
@@ -7,7 +7,7 @@ describe("ControlOverlay", () => {
cleanup();
});
- it("sets pointer-events none on blocker and pill when automationBypass is true", () => {
+ it("keeps page blocker none under automationBypass but Interrupt stays clickable", () => {
const { container } = render(
{
/>,
);
- const blocker = container.querySelector("[data-slot='control-overlay']")?.nextElementSibling;
+ const blocker = container.querySelector("[data-slot='control-overlay-blocker']");
expect(blocker).toBeTruthy();
expect((blocker as HTMLElement).style.pointerEvents).toBe("none");
+ const pill = container.querySelector("[data-slot='control-overlay-pill']");
+ expect(pill).toBeTruthy();
+ expect((pill as HTMLElement).style.pointerEvents).toBe("auto");
+
const stopBtn = container.querySelector("[data-slot='control-overlay-stop-all']");
expect(stopBtn).toBeTruthy();
- expect((stopBtn as HTMLElement).style.pointerEvents).toBe("none");
+ expect((stopBtn as HTMLElement).style.pointerEvents).toBe("auto");
});
it("uses pointer-events auto on blocker when automationBypass is false", () => {
@@ -36,7 +40,7 @@ describe("ControlOverlay", () => {
/>,
);
- const blocker = container.querySelector("[data-slot='control-overlay']")?.nextElementSibling;
+ const blocker = container.querySelector("[data-slot='control-overlay-blocker']");
expect(blocker).toBeTruthy();
expect((blocker as HTMLElement).style.pointerEvents).toBe("auto");
});
diff --git a/apps/extension/src/content/__tests__/RecordOverlay.test.tsx b/apps/extension/src/content/__tests__/RecordOverlay.test.tsx
new file mode 100644
index 0000000..9ed5e5a
--- /dev/null
+++ b/apps/extension/src/content/__tests__/RecordOverlay.test.tsx
@@ -0,0 +1,37 @@
+import { cleanup, fireEvent, render } from "@testing-library/react";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { RecordOverlay } from "../RecordOverlay";
+
+describe("RecordOverlay", () => {
+ afterEach(() => {
+ cleanup();
+ });
+
+ it("renders the bottom pill with orange finish CTA and no logo", () => {
+ const onFinish = vi.fn();
+ const { container } = render();
+
+ const pill = container.querySelector("[data-slot='record-overlay-pill']");
+ expect(pill).toBeTruthy();
+ expect(pill?.querySelector("img")).toBeNull();
+ expect((pill as HTMLElement).style.borderRadius).toBe("9999px");
+ expect((pill as HTMLElement).style.backgroundColor).toBe("#fff");
+
+ const finish = container.querySelector("[data-slot='record-overlay-finish']");
+ expect(finish).toBeTruthy();
+ expect((finish as HTMLElement).style.backgroundColor).toBe("#f97316");
+ expect((finish as HTMLElement).style.borderRadius).toBe("9999px");
+
+ fireEvent.click(finish!);
+ expect(onFinish).toHaveBeenCalledTimes(1);
+ });
+
+ it("shows a pulsing recording indicator and no full-screen glow layer", () => {
+ const { container } = render();
+ expect(container.querySelector("[data-slot='record-overlay']")).toBeNull();
+
+ const indicator = container.querySelector("[data-slot='record-overlay-indicator']");
+ expect(indicator).toBeTruthy();
+ expect((indicator as HTMLElement).style.animation).toContain("bsk-rec-pulse");
+ });
+});
diff --git a/apps/extension/src/content/__tests__/overlay-controller.test.ts b/apps/extension/src/content/__tests__/overlay-controller.test.ts
index c0b397a..7e8fd93 100644
--- a/apps/extension/src/content/__tests__/overlay-controller.test.ts
+++ b/apps/extension/src/content/__tests__/overlay-controller.test.ts
@@ -1,5 +1,5 @@
import { describe, expect, it, vi } from "vitest";
-import { OverlayController } from "../overlay-controller";
+import { OverlayController, shouldShowAgentControlOverlay } from "../overlay-controller";
describe("OverlayController", () => {
it("resets agent overlays without clearing user-tab borrow requests", () => {
@@ -31,6 +31,7 @@ describe("OverlayController", () => {
expect(state.controlVisible).toBe(false);
expect(state.activeSessionId).toBeNull();
expect(state.activeHelp).toBeNull();
+ expect(state.activeRecord).toBeNull();
expect(state.automationBypassCount).toBe(0);
});
@@ -69,4 +70,50 @@ describe("OverlayController", () => {
expect(previous?.id).toBe("help-1");
expect(controller.snapshot().activeHelp?.id).toBe("help-2");
});
+
+ it("tracks active record request state", () => {
+ const controller = new OverlayController();
+
+ controller.activateAgentSession("sess-1");
+ controller.setAgentRecordRequest({
+ id: "rec-1",
+ onFinish: vi.fn(),
+ });
+
+ expect(controller.snapshot().activeRecord?.id).toBe("rec-1");
+ });
+
+ it("hides agent control overlay while recording is active", () => {
+ const controller = new OverlayController();
+ controller.activateAgentSession("sess-1");
+ expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(true);
+
+ controller.setAgentRecordRequest({
+ id: "rec-1",
+ onFinish: vi.fn(),
+ });
+ expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(false);
+ expect(controller.snapshot().controlVisible).toBe(true);
+ });
+
+ it("keeps control hidden after record ends until session overlays reset", () => {
+ const controller = new OverlayController();
+ controller.activateAgentSession("sess-1");
+ controller.setAgentRecordRequest({
+ id: "rec-1",
+ onFinish: vi.fn(),
+ });
+ controller.setAutomationBypass(true);
+ controller.setAutomationBypass(true);
+
+ controller.clearAgentRecordRequest("rec-1");
+ expect(controller.snapshot().activeRecord).toBeNull();
+ expect(controller.snapshot().suppressControlAfterRecord).toBe(true);
+ expect(controller.snapshot().automationBypassCount).toBe(0);
+ expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(false);
+
+ controller.resetAgentOverlays("sess-1");
+ expect(controller.snapshot().suppressControlAfterRecord).toBe(false);
+ expect(controller.snapshot().controlVisible).toBe(false);
+ });
});
diff --git a/apps/extension/src/content/__tests__/record-capture.test.ts b/apps/extension/src/content/__tests__/record-capture.test.ts
new file mode 100644
index 0000000..38ac5d7
--- /dev/null
+++ b/apps/extension/src/content/__tests__/record-capture.test.ts
@@ -0,0 +1,159 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+import { RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
+import { handleRecordContentMessage, startRecordCapture } from "../record-capture";
+
+vi.stubGlobal("chrome", {
+ runtime: {
+ sendMessage: vi.fn(() => Promise.resolve()),
+ },
+});
+
+describe("handleRecordContentMessage stop/cancel", () => {
+ it("ignores STOP when no recording is active", () => {
+ const dispose = vi.fn();
+ const onStop = vi.fn();
+ const setActiveRequestId = vi.fn();
+ const setCapture = vi.fn();
+ const sendResponse = vi.fn();
+
+ const needsAsync = handleRecordContentMessage(
+ { type: RECORD_STOP, requestId: "rec-stale" },
+ {
+ activeRequestId: null,
+ capture: { dispose },
+ setActiveRequestId,
+ setCapture,
+ onStart: vi.fn(),
+ onStop,
+ },
+ sendResponse,
+ );
+
+ expect(needsAsync).toBe(false);
+ expect(dispose).not.toHaveBeenCalled();
+ expect(onStop).not.toHaveBeenCalled();
+ expect(setActiveRequestId).not.toHaveBeenCalled();
+ expect(setCapture).not.toHaveBeenCalled();
+ expect(sendResponse).not.toHaveBeenCalled();
+ });
+
+ it("ignores STOP for a mismatched requestId", () => {
+ const dispose = vi.fn();
+ const onStop = vi.fn();
+
+ const needsAsync = handleRecordContentMessage(
+ { type: RECORD_STOP, requestId: "rec-other" },
+ {
+ activeRequestId: "rec-1",
+ capture: { dispose },
+ setActiveRequestId: vi.fn(),
+ setCapture: vi.fn(),
+ onStart: vi.fn(),
+ onStop,
+ },
+ vi.fn(),
+ );
+
+ expect(needsAsync).toBe(false);
+ expect(dispose).not.toHaveBeenCalled();
+ expect(onStop).not.toHaveBeenCalled();
+ });
+});
+
+describe("record-capture semantic", () => {
+ let steps: RecordStepPayload[];
+
+ beforeEach(() => {
+ steps = [];
+ document.body.innerHTML = `
+
+
+
+
+ `;
+ });
+
+ it("commits final fill value and records semantic click", () => {
+ const capture = startRecordCapture("rec-1", (step) => steps.push(step));
+ const input = document.querySelector("input")!;
+ const button = document.querySelector("button")!;
+
+ input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
+ input.value = "hello";
+ input.dispatchEvent(new Event("input", { bubbles: true }));
+ input.dispatchEvent(new FocusEvent("focusout", { bubbles: true }));
+
+ button.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
+
+ capture.dispose();
+
+ expect(steps).toEqual(
+ expect.arrayContaining([
+ expect.objectContaining({
+ op: "fill",
+ value: "hello",
+ target: expect.objectContaining({ name: "查询", tag: "input" }),
+ }),
+ expect.objectContaining({
+ op: "click",
+ target: expect.objectContaining({ name: "搜索", role: "button" }),
+ }),
+ ]),
+ );
+ expect(JSON.stringify(steps)).not.toMatch(/@e\d+/);
+ expect(steps.some((s) => "selector" in s)).toBe(false);
+ });
+
+ it("ignores autocomplete suggestion clicks while a fill session is open", () => {
+ const capture = startRecordCapture("rec-2", (step) => steps.push(step));
+ const input = document.querySelector("input")!;
+ const option = document.querySelector('[role="option"]')!;
+
+ input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
+ input.value = "hel";
+ input.dispatchEvent(new Event("input", { bubbles: true }));
+ option.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
+ input.value = "hello";
+ input.dispatchEvent(new Event("input", { bubbles: true }));
+ input.dispatchEvent(new FocusEvent("focusout", { bubbles: true }));
+ capture.dispose();
+
+ expect(steps.filter((s) => s.op === "click")).toHaveLength(0);
+ expect(steps.some((s) => s.op === "fill" && s.value === "hello")).toBe(true);
+ });
+
+ it("records Enter press but not bare typing keys", () => {
+ const capture = startRecordCapture("rec-press", (step) => steps.push(step));
+ const input = document.querySelector("input")!;
+ input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
+ input.dispatchEvent(new KeyboardEvent("keydown", { key: "a", bubbles: true }));
+ input.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", bubbles: true }));
+ capture.dispose();
+ expect(steps.filter((s) => s.op === "press")).toEqual([
+ expect.objectContaining({ op: "press", key: "Enter" }),
+ ]);
+ });
+
+ it("does not record clicks on anonymous layout divs", () => {
+ document.body.innerHTML = `
+ page chrome
+
+ `;
+ const capture = startRecordCapture("rec-3", (step) => steps.push(step));
+ document
+ .querySelector("#chrome")!
+ .dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
+ document
+ .querySelector("button")!
+ .dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
+ capture.dispose();
+
+ expect(steps).toHaveLength(1);
+ expect(steps[0]).toMatchObject({
+ op: "click",
+ target: { name: "下一步", role: "button" },
+ });
+ });
+});
diff --git a/apps/extension/src/content/overlay-controller.ts b/apps/extension/src/content/overlay-controller.ts
index 15077a6..a0f6560 100644
--- a/apps/extension/src/content/overlay-controller.ts
+++ b/apps/extension/src/content/overlay-controller.ts
@@ -1,13 +1,21 @@
import type { BorrowRequestData } from "./BorrowConfirmationOverlay";
import type { HelpRequestData } from "./HelpRequestOverlay";
+import type { RecordRequestData } from "./RecordOverlay";
export interface OverlayState {
borrowRequests: BorrowRequestData[];
activeHelp: HelpRequestData | null;
+ activeRecord: RecordRequestData | null;
controlVisible: boolean;
interrupting: boolean;
activeSessionId: string | null;
automationBypassCount: number;
+ /**
+ * After record Finish/Stop clears `activeRecord`, the session is still
+ * alive until CLI `session.stop`. Suppress the control mask for that gap
+ * so 「Agent 正在控制」does not flash between RecordOverlay and teardown.
+ */
+ suppressControlAfterRecord: boolean;
}
type MutableOverlayState = Omit;
@@ -20,9 +28,11 @@ export class OverlayController {
private state: MutableOverlayState = {
borrowRequests: [],
activeHelp: null,
+ activeRecord: null,
interrupting: false,
activeSessionId: null,
automationBypassCount: 0,
+ suppressControlAfterRecord: false,
};
snapshot(): OverlayState {
@@ -70,6 +80,25 @@ export class OverlayController {
this.state.activeHelp = null;
}
+ setAgentRecordRequest(request: RecordRequestData): RecordRequestData | null {
+ const previous = this.state.activeRecord;
+ this.state.activeRecord = request;
+ this.state.suppressControlAfterRecord = false;
+ return previous;
+ }
+
+ clearAgentRecordRequest(requestId?: string): void {
+ if (!this.state.activeRecord) return;
+ if (requestId && this.state.activeRecord.id !== requestId) return;
+ this.state.activeRecord = null;
+ // Hide control until session teardown — see OverlayState.suppressControlAfterRecord.
+ this.state.suppressControlAfterRecord = true;
+ // Record start/rearm may have stacked automation-bypass refs; drop them so a
+ // stray ControlOverlay cannot sit with pointer-events:none (page usable,
+ // Interrupt unclickable).
+ this.state.automationBypassCount = 0;
+ }
+
setAutomationBypass(enabled: boolean): void {
if (enabled) {
this.state.automationBypassCount += 1;
@@ -86,10 +115,22 @@ export class OverlayController {
this.state = {
...this.state,
activeHelp: null,
+ activeRecord: null,
interrupting: false,
activeSessionId: null,
automationBypassCount: 0,
+ suppressControlAfterRecord: false,
};
return previousHelp;
}
}
+
+/** Control mask ("Agent 正在控制") must hide while help/record overlays own the chrome. */
+export function shouldShowAgentControlOverlay(state: OverlayState): boolean {
+ return (
+ state.controlVisible &&
+ !state.suppressControlAfterRecord &&
+ state.activeHelp === null &&
+ state.activeRecord === null
+ );
+}
diff --git a/apps/extension/src/content/record-capture.ts b/apps/extension/src/content/record-capture.ts
new file mode 100644
index 0000000..12bf6af
--- /dev/null
+++ b/apps/extension/src/content/record-capture.ts
@@ -0,0 +1,537 @@
+import { describeEventTarget, describeTarget, type TargetDescriptor } from "@/lib/describe-target";
+import {
+ isRecordCancelMessage,
+ isRecordStartMessage,
+ isRecordStopMessage,
+ RECORD_CANCEL,
+ RECORD_START,
+ RECORD_STEP,
+ RECORD_STOP,
+ type RecordCancelMessage,
+ type RecordStartAck,
+ type RecordStartMessage,
+ type RecordStepPayload,
+ type RecordStopAck,
+ type RecordStopMessage,
+} from "@/lib/record-bridge";
+import { shouldRecordPress } from "@/lib/trace-reducer";
+
+const pendingStepSends = new Map>>();
+const failedStepDeliveries = new Set();
+const knownRecordRequests = new Set();
+
+function sendRecordStep(requestId: string, step: RecordStepPayload): void {
+ const payload = { type: RECORD_STEP, requestId, step };
+ const pending = Promise.resolve(chrome.runtime.sendMessage(payload)).then(
+ () => true,
+ () => {
+ failedStepDeliveries.add(requestId);
+ return false;
+ },
+ );
+ const sends = pendingStepSends.get(requestId) ?? new Set>();
+ sends.add(pending);
+ pendingStepSends.set(requestId, sends);
+ void pending.then(() => {
+ sends.delete(pending);
+ if (sends.size === 0) pendingStepSends.delete(requestId);
+ });
+}
+
+export interface RecordCaptureController {
+ dispose(): void;
+}
+
+interface FillSession {
+ element: FillableElement;
+ target: TargetDescriptor;
+ baselineValue: string;
+ lastValue: string;
+}
+
+type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement;
+
+function eventTarget(event: Event): EventTarget | null {
+ return event.composedPath()[0] ?? event.target;
+}
+
+function isOverlayTarget(target: EventTarget | null): boolean {
+ if (!(target instanceof Node)) return false;
+ const root = document.documentElement;
+ let node: Node | null = target;
+ while (node && node !== root) {
+ if (node instanceof Element && node.hasAttribute("data-bsk-overlay")) {
+ return true;
+ }
+ const rootNode: Node | Document | ShadowRoot = node.getRootNode();
+ if (rootNode instanceof ShadowRoot) {
+ const host: Element = rootNode.host;
+ if (host.hasAttribute("data-bsk-overlay")) {
+ return true;
+ }
+ node = host;
+ } else {
+ node = node.parentNode;
+ }
+ }
+ return false;
+}
+
+function isTextFillable(el: Element): el is FillableElement {
+ if (el instanceof HTMLElement && (el.isContentEditable || el.contentEditable === "true")) {
+ return true;
+ }
+ if (el instanceof HTMLTextAreaElement) return true;
+ if (!(el instanceof HTMLInputElement)) return false;
+ const type = el.type.toLowerCase();
+ return type !== "checkbox" && type !== "radio" && type !== "file" && type !== "button";
+}
+
+function fillableFromTarget(target: EventTarget | null): FillableElement | null {
+ if (!(target instanceof Element)) return null;
+ if (isTextFillable(target)) return target;
+ const el = target.closest('input,textarea,[contenteditable]:not([contenteditable="false"])');
+ if (el && isTextFillable(el)) return el;
+ return null;
+}
+
+/** Clicks on search chrome that only focus the nearby input should not become steps. */
+function nearbyFillableFromSearchChrome(target: Element): FillableElement | null {
+ const container = target.closest(
+ '[id*="chat-input"], [id*="search"], [class*="search"], form, [role="search"]',
+ );
+ if (!container) return null;
+ const fillable = container.querySelector(
+ 'textarea, input[type="search"], input[name="q"], #chat-textarea',
+ );
+ if (
+ fillable instanceof HTMLElement &&
+ isTextFillable(fillable) &&
+ fillable !== target &&
+ !fillable.contains(target)
+ ) {
+ return fillable;
+ }
+ return null;
+}
+
+function fillableValue(el: FillableElement): string {
+ if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
+ return el.value;
+ }
+ return el.textContent ?? "";
+}
+
+/** Clicks that only pick an autocomplete/suggestion value — not a semantic submit action. */
+function isInputCompletionClick(target: EventTarget | null, session: FillSession | null): boolean {
+ if (!session || !(target instanceof Element)) return false;
+ if (session.element.contains(target)) return false;
+
+ const option = target.closest('[role="option"]');
+ if (option?.closest('[role="listbox"]')) return true;
+
+ const suggestionRoot = target.closest(
+ [
+ '[id*="Sug"]',
+ '[id*="sug"]',
+ '[class*="suggest"]',
+ '[class*="autocomplete"]',
+ '[class*="typeahead"]',
+ "[data-autocomplete]",
+ ].join(", "),
+ );
+ if (suggestionRoot && !suggestionRoot.contains(session.element)) return true;
+
+ return false;
+}
+
+function scheduleInputCompletionCommit(
+ sessionElement: FillableElement,
+ syncFillSessionValue: (el: FillableElement) => void,
+ commitFillSession: () => void,
+ hasSessionFor: (el: FillableElement) => boolean,
+): void {
+ const syncAndCommit = () => {
+ if (!hasSessionFor(sessionElement)) return;
+ syncFillSessionValue(sessionElement);
+ commitFillSession();
+ };
+ syncFillSessionValue(sessionElement);
+ queueMicrotask(syncAndCommit);
+ setTimeout(syncAndCommit, 0);
+}
+
+export function startRecordCapture(
+ _requestId: string,
+ sendStep: (step: RecordStepPayload) => void,
+): RecordCaptureController {
+ const emitStep = (step: RecordStepPayload) => {
+ sendStep({ page_url: location.href, ...step });
+ };
+ let fillSession: FillSession | null = null;
+ let composing = false;
+ let lastUrl = location.href;
+ let keyboardActivation: { target: EventTarget | null; recordedAt: number } | undefined;
+ let generatedControlClick: Element | null = null;
+ let navigationActionPending = false;
+ let navigationActionVersion = 0;
+ const committedValues = new WeakMap();
+
+ const markNavigationAction = () => {
+ navigationActionPending = true;
+ const version = ++navigationActionVersion;
+ setTimeout(() => {
+ if (navigationActionVersion === version) {
+ navigationActionPending = false;
+ }
+ }, 0);
+ };
+
+ const emitFill = (session: FillSession) => {
+ if (session.lastValue === session.baselineValue) return;
+ const isPassword =
+ session.element instanceof HTMLInputElement && session.element.type === "password";
+ const value = isPassword ? "***" : session.lastValue;
+ emitStep({
+ op: "fill",
+ target: session.target,
+ value,
+ ...(isPassword ? { redacted: true } : {}),
+ });
+ };
+
+ const commitFillSession = () => {
+ if (!fillSession || composing) return;
+ const session = fillSession;
+ fillSession = null;
+ emitFill(session);
+ committedValues.set(session.element, session.lastValue);
+ };
+
+ const syncFillSessionValue = (el: FillableElement) => {
+ if (!fillSession || fillSession.element !== el) return;
+ fillSession.lastValue = fillableValue(el);
+ };
+
+ const ensureFillSession = (el: FillableElement) => {
+ if (fillSession?.element === el) return;
+ if (fillSession) commitFillSession();
+ const currentValue = fillableValue(el);
+ const baselineValue = committedValues.get(el) ?? currentValue;
+ fillSession = {
+ element: el,
+ target: describeTarget(el),
+ baselineValue,
+ lastValue: currentValue,
+ };
+ };
+
+ const emitNavigateIfChanged = (causedByAction?: boolean) => {
+ if (location.href === lastUrl) return;
+ commitFillSession();
+ lastUrl = location.href;
+ emitStep({
+ op: "navigate",
+ url: location.href,
+ ...(causedByAction !== undefined ? { navigation_caused_by_action: causedByAction } : {}),
+ });
+ };
+
+ const emitClick = (event: MouseEvent) => {
+ // Only record clicks an LLM can re-identify (named interactive controls).
+ const target = describeEventTarget(eventTarget(event));
+ if (!target) return;
+ markNavigationAction();
+ emitStep({
+ op: "click",
+ target,
+ expects_navigation: true,
+ });
+ };
+
+ const onClick = (event: MouseEvent) => {
+ if (event.button !== 0) return;
+ const target = eventTarget(event);
+ if (isOverlayTarget(target)) return;
+ if (event.detail === 0 && generatedControlClick !== null && target === generatedControlClick) {
+ generatedControlClick = null;
+ return;
+ }
+ if (
+ event.detail === 0 &&
+ keyboardActivation?.target === target &&
+ Date.now() - keyboardActivation.recordedAt < 500
+ ) {
+ keyboardActivation = undefined;
+ return;
+ }
+
+ const label = target instanceof Element ? target.closest("label") : null;
+ const nestedInteractive =
+ target instanceof Element ? target.closest("a,button,input,select,textarea") : null;
+ if (label instanceof HTMLLabelElement && !nestedInteractive) {
+ commitFillSession();
+ generatedControlClick = label.control;
+ emitClick(event);
+ return;
+ }
+
+ const fillable = fillableFromTarget(target);
+ if (fillable) {
+ ensureFillSession(fillable);
+ return;
+ }
+
+ if (target instanceof Element) {
+ const nearbyFillable = nearbyFillableFromSearchChrome(target);
+ if (nearbyFillable) {
+ ensureFillSession(nearbyFillable);
+ return;
+ }
+ }
+
+ if (isInputCompletionClick(target, fillSession)) {
+ markNavigationAction();
+ const sessionElement = fillSession!.element;
+ scheduleInputCompletionCommit(
+ sessionElement,
+ syncFillSessionValue,
+ commitFillSession,
+ (el) => fillSession?.element === el,
+ );
+ return;
+ }
+
+ commitFillSession();
+ if (target instanceof Element && target.closest("select")) return;
+ emitClick(event);
+ };
+
+ const onFocusIn = (event: FocusEvent) => {
+ const target = fillableFromTarget(eventTarget(event));
+ if (target) ensureFillSession(target);
+ };
+
+ const onFocusOut = (event: FocusEvent) => {
+ const target = fillableFromTarget(eventTarget(event));
+ if (!target) return;
+ if (!fillSession || fillSession.element !== target) return;
+ syncFillSessionValue(target);
+ commitFillSession();
+ };
+
+ const onInput = (event: Event) => {
+ const target = fillableFromTarget(eventTarget(event));
+ if (!target) return;
+ if (composing) return;
+ ensureFillSession(target);
+ syncFillSessionValue(target);
+ };
+
+ const onCompositionStart = () => {
+ composing = true;
+ };
+
+ const onCompositionEnd = (event: CompositionEvent) => {
+ composing = false;
+ const target = fillableFromTarget(eventTarget(event));
+ if (target) {
+ ensureFillSession(target);
+ syncFillSessionValue(target);
+ }
+ };
+
+ const onChange = (event: Event) => {
+ commitFillSession();
+ const target = eventTarget(event);
+ if (target instanceof HTMLSelectElement) {
+ const values = Array.from(target.selectedOptions).map((opt) => opt.value);
+ const labels = Array.from(target.selectedOptions).map((opt) =>
+ (opt.label || opt.textContent || opt.value).trim(),
+ );
+ const desc = describeTarget(target);
+ markNavigationAction();
+ emitStep({
+ op: "select",
+ target: desc,
+ values,
+ labels,
+ expects_navigation: true,
+ });
+ }
+ };
+
+ const onKeyDown = (event: KeyboardEvent) => {
+ if (isOverlayTarget(eventTarget(event))) return;
+ const target = eventTarget(event);
+ const fillable = fillableFromTarget(target);
+ if (fillable) {
+ ensureFillSession(fillable);
+ syncFillSessionValue(fillable);
+ }
+ const modifiers = [
+ ...(event.altKey ? (["alt"] as const) : []),
+ ...(event.ctrlKey ? (["ctrl"] as const) : []),
+ ...(event.metaKey ? (["meta"] as const) : []),
+ ...(event.shiftKey ? (["shift"] as const) : []),
+ ];
+ if (!shouldRecordPress(event.key, modifiers)) {
+ return;
+ }
+ markNavigationAction();
+ if (event.key === "Enter" || event.key === " ") {
+ const submitTarget =
+ event.key === "Enter" && fillable
+ ? fillable
+ .closest("form")
+ ?.querySelector(
+ 'button:not([type]),button[type="submit"],input[type="submit"],input[type="image"]',
+ )
+ : null;
+ keyboardActivation = {
+ target: submitTarget ?? target,
+ recordedAt: Date.now(),
+ };
+ }
+ if (fillable) {
+ commitFillSession();
+ }
+ const desc = describeEventTarget(target);
+ if (!desc && !event.key) return;
+ emitStep({
+ op: "press",
+ key: event.key,
+ ...(desc ? { target: desc } : {}),
+ ...(modifiers.length ? { modifiers } : {}),
+ expects_navigation: event.key === "Enter",
+ });
+ };
+
+ document.addEventListener("click", onClick, true);
+ document.addEventListener("focusin", onFocusIn, true);
+ document.addEventListener("focusout", onFocusOut, true);
+ document.addEventListener("input", onInput, true);
+ document.addEventListener("compositionstart", onCompositionStart, true);
+ document.addEventListener("compositionend", onCompositionEnd, true);
+ document.addEventListener("change", onChange, true);
+ document.addEventListener("keydown", onKeyDown, true);
+
+ const urlObserver = new MutationObserver(() => emitNavigateIfChanged());
+ urlObserver.observe(document, { subtree: true, childList: true });
+ const onUrlEvent = () => emitNavigateIfChanged();
+ window.addEventListener("hashchange", onUrlEvent);
+ window.addEventListener("popstate", onUrlEvent);
+ window.addEventListener("pagehide", commitFillSession);
+
+ const originalPushState = history.pushState;
+ const originalReplaceState = history.replaceState;
+ history.pushState = function (...args: Parameters) {
+ originalPushState.apply(this, args);
+ emitNavigateIfChanged(navigationActionPending ? true : undefined);
+ };
+ history.replaceState = function (...args: Parameters) {
+ originalReplaceState.apply(this, args);
+ emitNavigateIfChanged(navigationActionPending ? true : undefined);
+ };
+
+ return {
+ dispose() {
+ commitFillSession();
+ document.removeEventListener("click", onClick, true);
+ document.removeEventListener("focusin", onFocusIn, true);
+ document.removeEventListener("focusout", onFocusOut, true);
+ document.removeEventListener("input", onInput, true);
+ document.removeEventListener("compositionstart", onCompositionStart, true);
+ document.removeEventListener("compositionend", onCompositionEnd, true);
+ document.removeEventListener("change", onChange, true);
+ document.removeEventListener("keydown", onKeyDown, true);
+ urlObserver.disconnect();
+ window.removeEventListener("hashchange", onUrlEvent);
+ window.removeEventListener("popstate", onUrlEvent);
+ window.removeEventListener("pagehide", commitFillSession);
+ history.pushState = originalPushState;
+ history.replaceState = originalReplaceState;
+ },
+ };
+}
+
+export type RecordContentMessage = RecordStartMessage | RecordStopMessage | RecordCancelMessage;
+
+export function isRecordContentMessage(msg: unknown): msg is RecordContentMessage {
+ return isRecordStartMessage(msg) || isRecordStopMessage(msg) || isRecordCancelMessage(msg);
+}
+
+export function handleRecordContentMessage(
+ message: RecordContentMessage,
+ state: {
+ activeRequestId: string | null;
+ capture: RecordCaptureController | null;
+ setActiveRequestId(id: string | null): void;
+ setCapture(capture: RecordCaptureController | null): void;
+ onStart(requestId: string): void;
+ onStop(): void;
+ },
+ sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
+): boolean {
+ if (isRecordStartMessage(message)) {
+ if (!knownRecordRequests.has(message.requestId)) {
+ knownRecordRequests.add(message.requestId);
+ failedStepDeliveries.delete(message.requestId);
+ }
+ state.capture?.dispose();
+ state.setCapture(
+ startRecordCapture(message.requestId, (step) => {
+ sendRecordStep(message.requestId, step);
+ }),
+ );
+ state.setActiveRequestId(message.requestId);
+ state.onStart(message.requestId);
+ sendResponse?.({ ok: true });
+ return sendResponse !== undefined;
+ }
+
+ if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
+ // Require an active recording that matches this requestId — otherwise a
+ // stray STOP/CANCEL (e.g. after teardown) would still run finishStop and
+ // clear overlay state even though nothing was capturing.
+ if (!state.activeRequestId || state.activeRequestId !== message.requestId) {
+ return false;
+ }
+ state.capture?.dispose();
+ state.setCapture(null);
+ const finishStop = () => {
+ state.onStop();
+ state.setActiveRequestId(null);
+ };
+ if (isRecordStopMessage(message) && sendResponse) {
+ const pending = [...(pendingStepSends.get(message.requestId) ?? [])];
+ void Promise.all(pending).then((delivered) => {
+ finishStop();
+ const succeeded = delivered.every(Boolean) && !failedStepDeliveries.has(message.requestId);
+ if (succeeded) {
+ failedStepDeliveries.delete(message.requestId);
+ knownRecordRequests.delete(message.requestId);
+ }
+ sendResponse(
+ succeeded
+ ? { ok: true }
+ : {
+ ok: false,
+ error: "failed to deliver one or more recorded steps",
+ },
+ );
+ });
+ return true;
+ }
+ if (isRecordCancelMessage(message)) {
+ failedStepDeliveries.delete(message.requestId);
+ knownRecordRequests.delete(message.requestId);
+ }
+ finishStop();
+ return false;
+ }
+
+ return false;
+}
+
+export { RECORD_CANCEL, RECORD_START, RECORD_STEP, RECORD_STOP };
diff --git a/apps/extension/src/entrypoints/background.ts b/apps/extension/src/entrypoints/background.ts
index ac81ab7..6192811 100644
--- a/apps/extension/src/entrypoints/background.ts
+++ b/apps/extension/src/entrypoints/background.ts
@@ -9,6 +9,7 @@ import {
} from "@/lib/instance-id";
import { startKeepalive } from "@/lib/keepalive";
import {
+ OVERLAY_AUTOMATION_BYPASS,
OVERLAY_MSG_INTERRUPT,
OVERLAY_MSG_WHO_AM_I,
type OverlayInterruptRequest,
@@ -28,6 +29,11 @@ import {
requestBorrowConfirmation,
} from "@/tools/borrow-confirmation";
import { ToolDispatcher } from "@/tools/dispatcher";
+import {
+ attachRecordFinishListener,
+ attachRecordQueryListener,
+ attachRecordStepListener,
+} from "@/tools/record";
import { detectBrowserMeta } from "@/transport/handshake";
import type { Transport } from "@/transport/transport";
import { WSTransport } from "@/transport/ws-transport";
@@ -68,6 +74,27 @@ export default defineBackground(() => {
}),
});
dispatcher.start();
+ const recordDeps = {
+ tabsApi: chrome.tabs,
+ sendToTab: (tabId: number, msg: Parameters[1]) =>
+ chrome.tabs.sendMessage(tabId, msg),
+ bypassOverlay: async (tabId: number, enabled: boolean) => {
+ try {
+ await chrome.tabs.sendMessage(tabId, {
+ type: OVERLAY_AUTOMATION_BYPASS,
+ enabled,
+ });
+ } catch {
+ // Content script may be unavailable on restricted pages.
+ }
+ },
+ };
+ // Message listeners stay up (cheap; fire only for record message types).
+ // Tab / webNavigation observation attaches lazily while a recording is
+ // active — see ensureBrowserObservationListeners in tools/record.ts.
+ attachRecordStepListener();
+ attachRecordFinishListener(recordDeps);
+ attachRecordQueryListener(recordDeps);
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
attachBorrowNotificationClickHandler({
onClicked: chrome.notifications.onClicked,
diff --git a/apps/extension/src/entrypoints/content.ts b/apps/extension/src/entrypoints/content.ts
index b9e9dee..d721d70 100644
--- a/apps/extension/src/entrypoints/content.ts
+++ b/apps/extension/src/entrypoints/content.ts
@@ -6,7 +6,13 @@ import { BorrowConfirmationOverlay } from "@/content/BorrowConfirmationOverlay";
import { ControlOverlay } from "@/content/ControlOverlay";
import { HelpRequestOverlay } from "@/content/HelpRequestOverlay";
import overlayCss from "@/content/overlay.css?inline";
-import { OverlayController } from "@/content/overlay-controller";
+import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
+import { RecordOverlay } from "@/content/RecordOverlay";
+import {
+ handleRecordContentMessage,
+ isRecordContentMessage,
+ type RecordCaptureController,
+} from "@/content/record-capture";
import {
HELP_RESPONSE,
type HelpCancelMessage,
@@ -24,6 +30,12 @@ import {
type OverlayWhoAmIResponse,
} from "@/lib/overlay-bridge";
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
+import {
+ RECORD_FINISH,
+ RECORD_QUERY,
+ type RecordStartAck,
+ type RecordStopAck,
+} from "@/lib/record-bridge";
import { SESSIONS_LIVE_FLAG_KEY } from "@/lib/sessions-live-flag";
import type {
BorrowCancelMessage,
@@ -45,6 +57,8 @@ export default defineContentScript({
const overlays = new OverlayController();
let activeHelpRespond: ((outcome: "continued" | "cancelled", note?: string) => void) | null =
null;
+ let recordCapture: RecordCaptureController | null = null;
+ let activeRecordRequestId: string | null = null;
let reactRoot: ReactDOM.Root | null = null;
let overlayHost: HTMLElement | null = null;
let hostLossReported = false;
@@ -87,12 +101,13 @@ export default defineContentScript({
requests: overlayState.borrowRequests,
}),
React.createElement(ControlOverlay, {
- visible: overlayState.controlVisible && overlayState.activeHelp === null,
+ visible: shouldShowAgentControlOverlay(overlayState),
interrupting: overlayState.interrupting,
automationBypass: overlayState.automationBypassCount > 0,
onInterrupt: handleInterrupt,
}),
React.createElement(HelpRequestOverlay, { request: overlayState.activeHelp }),
+ React.createElement(RecordOverlay, { request: overlayState.activeRecord }),
),
),
);
@@ -104,6 +119,9 @@ export default defineContentScript({
activeHelpRespond?.("cancelled");
activeHelpRespond = null;
}
+ recordCapture?.dispose();
+ recordCapture = null;
+ activeRecordRequestId = null;
renderOverlay();
}
@@ -140,6 +158,42 @@ export default defineContentScript({
_sender: chrome.runtime.MessageSender,
sendResponse: (response: BorrowResponseMessage | HelpResponseMessage) => void,
) => {
+ if (isRecordContentMessage(message)) {
+ const needsAsync = handleRecordContentMessage(
+ message,
+ {
+ activeRequestId: activeRecordRequestId,
+ capture: recordCapture,
+ setActiveRequestId: (id) => {
+ activeRecordRequestId = id;
+ },
+ setCapture: (capture) => {
+ recordCapture = capture;
+ },
+ onStart: (requestId) => {
+ overlays.setAgentRecordRequest({
+ id: requestId,
+ onFinish: () => {
+ void chrome.runtime.sendMessage({
+ type: RECORD_FINISH,
+ requestId,
+ });
+ },
+ });
+ renderOverlay();
+ },
+ onStop: () => {
+ overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
+ renderOverlay();
+ },
+ },
+ sendResponse as unknown as
+ | ((response: RecordStartAck | RecordStopAck) => void)
+ | undefined,
+ );
+ return needsAsync;
+ }
+
if (
message &&
typeof message === "object" &&
@@ -228,23 +282,59 @@ export default defineContentScript({
return false;
};
- async function mountOverlayIfAgent(): Promise {
+ async function syncAgentOverlay(): Promise {
if (!(await anySessionLive())) return;
try {
const reply = (await chrome.runtime.sendMessage({
kind: OVERLAY_MSG_WHO_AM_I,
})) as OverlayWhoAmIResponse | undefined;
if (!reply?.sessionId) return;
+
+ // Query / re-arm recording *before* activating the control overlay so
+ // record start (navigate → content load) does not flash
+ // 「Agent 正在控制」before RecordOverlay mounts.
+ let recordQuery: { active?: boolean; requestId?: string } | undefined;
+ try {
+ recordQuery = (await chrome.runtime.sendMessage({
+ type: RECORD_QUERY,
+ })) as { active?: boolean; requestId?: string } | undefined;
+ } catch (err) {
+ console.debug("[bsk overlay] record query failed", err);
+ }
+
+ if (
+ recordQuery?.active &&
+ typeof recordQuery.requestId === "string" &&
+ overlays.snapshot().activeRecord === null
+ ) {
+ const requestId = recordQuery.requestId;
+ overlays.setAgentRecordRequest({
+ id: requestId,
+ onFinish: () => {
+ void chrome.runtime.sendMessage({
+ type: RECORD_FINISH,
+ requestId,
+ });
+ },
+ });
+ }
+
overlays.activateAgentSession(reply.sessionId);
renderOverlay();
} catch (err) {
- console.debug("[bsk overlay] who_am_i failed", err);
+ console.debug("[bsk overlay] syncAgentOverlay failed", err);
}
}
+ const onPageShow = (event: PageTransitionEvent) => {
+ if (event.persisted) void syncAgentOverlay();
+ };
+
ui.mount();
chrome.runtime.onMessage.addListener(onMessage);
- void mountOverlayIfAgent();
+ void syncAgentOverlay();
+
+ window.addEventListener("pageshow", onPageShow);
const hostObserver = new MutationObserver(() => {
const connected = overlayHost?.isConnected ?? false;
@@ -254,6 +344,7 @@ export default defineContentScript({
remountInProgress = true;
try {
ui.mount();
+ void syncAgentOverlay();
} finally {
remountInProgress = false;
}
@@ -268,6 +359,11 @@ export default defineContentScript({
ctx.onInvalidated(() => {
hostObserver.disconnect();
chrome.runtime.onMessage.removeListener(onMessage);
+ window.removeEventListener("pageshow", onPageShow);
+ // Restore history hooks / remove capture listeners before the CS unloads.
+ recordCapture?.dispose();
+ recordCapture = null;
+ activeRecordRequestId = null;
});
},
});
diff --git a/apps/extension/src/entrypoints/popup/App.test.tsx b/apps/extension/src/entrypoints/popup/App.test.tsx
index 0f59e46..155e7b6 100644
--- a/apps/extension/src/entrypoints/popup/App.test.tsx
+++ b/apps/extension/src/entrypoints/popup/App.test.tsx
@@ -24,6 +24,11 @@ const baseSnapshot: SnapshotInfo = {
connectionEnabled: true,
};
+function openRecordView() {
+ fireEvent.click(screen.getByRole("button", { name: "快捷功能" }));
+ fireEvent.click(screen.getByRole("button", { name: /操作录制/ }));
+}
+
describe("App", () => {
const setLabel = vi.fn();
const setConnectionEnabled = vi.fn();
@@ -63,13 +68,35 @@ describe("App", () => {
expect(screen.queryByText("请先打开 BrowserSkill。")).toBeNull();
});
- it("uses flex gap for label field spacing", () => {
- const { container } = render();
+ it("does not render record UI on the main view", () => {
+ render();
- const labelField = container.querySelector("[data-slot='popup-label-field']");
- expect(labelField?.className).toContain("flex");
- expect(labelField?.className).toContain("flex-col");
- expect(labelField?.className).toContain("gap-3");
+ expect(screen.queryByRole("button", { name: "复制录制指令" })).toBeNull();
+ expect(screen.queryByRole("button", { name: "操作录制" })).toBeNull();
+ });
+
+ it("opens the feature list from the launcher and navigates to record", () => {
+ render();
+
+ fireEvent.click(screen.getByRole("button", { name: "快捷功能" }));
+
+ expect(screen.getByText("操作录制")).toBeTruthy();
+ expect(screen.getByText("录制你的操作,供 Agent 参考")).toBeTruthy();
+ expect(screen.queryByText("未连接")).toBeNull();
+
+ fireEvent.click(screen.getByRole("button", { name: /操作录制/ }));
+
+ expect(screen.getByRole("button", { name: "复制录制指令" })).toBeTruthy();
+ });
+
+ it("returns from the feature list to the main view", () => {
+ render();
+
+ fireEvent.click(screen.getByRole("button", { name: "快捷功能" }));
+ fireEvent.click(screen.getByRole("button", { name: "返回" }));
+
+ expect(screen.getByText("未连接")).toBeTruthy();
+ expect(screen.queryByText("录制你的操作,供 Agent 参考")).toBeNull();
});
it("shows single-line compact metadata and copies the instance id", async () => {
@@ -136,4 +163,84 @@ describe("App", () => {
screen.getByRole("switch", { name: "BrowserSkill 连接" }).getAttribute("aria-checked"),
).toBe("false");
});
+
+ it("copies the record prompt with instance id and --browser when connected", async () => {
+ mockUseConnectionState.mockReturnValue({
+ snapshot: {
+ ...baseSnapshot,
+ state: "connected",
+ instanceId: "03c3e47f",
+ handshake: {
+ server: "bh",
+ version: mockDaemonVersion,
+ protocol_version: "1.0",
+ },
+ },
+ statusState: "connected",
+ setLabel,
+ setConnectionEnabled,
+ });
+
+ render();
+ openRecordView();
+
+ fireEvent.change(screen.getByPlaceholderText("例如:发布一篇文章"), {
+ target: { value: "发布 wiki 文档" },
+ });
+
+ const copyButton = screen.getByRole("button", { name: "复制录制指令" });
+ expect(copyButton.getAttribute("disabled")).toBeNull();
+
+ fireEvent.click(copyButton);
+
+ expect(navigator.clipboard.writeText).toHaveBeenCalledTimes(1);
+ const copied = vi.mocked(navigator.clipboard.writeText).mock.calls[0]?.[0] ?? "";
+ expect(copied).toContain("03c3e47f");
+ expect(copied).toContain("--browser 03c3e47f");
+ expect(copied).toContain('--purpose "发布 wiki 文档"');
+ expect(copied).not.toMatch(/bsk record start[^\n]*--url/);
+ // Button label stays static; a transient toast confirms the copy.
+ expect(copyButton.textContent).toContain("复制录制指令");
+ await waitFor(() => expect(screen.getByRole("status").textContent).toContain("已复制"));
+ });
+
+ it("includes --url in record prompt when a start URL is provided", async () => {
+ mockUseConnectionState.mockReturnValue({
+ snapshot: {
+ ...baseSnapshot,
+ state: "connected",
+ instanceId: "03c3e47f",
+ handshake: {
+ server: "bh",
+ version: mockDaemonVersion,
+ protocol_version: "1.0",
+ },
+ },
+ statusState: "connected",
+ setLabel,
+ setConnectionEnabled,
+ });
+
+ render();
+ openRecordView();
+
+ fireEvent.change(screen.getByPlaceholderText("https://…"), {
+ target: { value: "https://example.com/" },
+ });
+
+ fireEvent.click(screen.getByRole("button", { name: "复制录制指令" }));
+
+ const copied = vi.mocked(navigator.clipboard.writeText).mock.calls[0]?.[0] ?? "";
+ expect(copied).toContain("--url https://example.com/");
+ expect(copied).not.toContain("--purpose");
+ });
+
+ it("disables record copy when disconnected", () => {
+ render();
+ openRecordView();
+
+ const copyButton = screen.getByRole("button", { name: "复制录制指令" });
+ expect(copyButton.getAttribute("disabled")).not.toBeNull();
+ expect(screen.getByText("连接后可用")).toBeTruthy();
+ });
});
diff --git a/apps/extension/src/entrypoints/popup/App.tsx b/apps/extension/src/entrypoints/popup/App.tsx
index 09b0c3c..6aee921 100644
--- a/apps/extension/src/entrypoints/popup/App.tsx
+++ b/apps/extension/src/entrypoints/popup/App.tsx
@@ -1,9 +1,16 @@
import { useTranslation } from "@browser-skill/i18n/react";
import { Badge, Button, cn, Input, Label } from "@browser-skill/ui";
-import { RiCheckLine, RiFileCopyLine } from "@remixicon/react";
+import {
+ RiApps2Line,
+ RiArrowLeftLine,
+ RiArrowRightSLine,
+ RiCheckLine,
+ RiFileCopyLine,
+} from "@remixicon/react";
import { type ChangeEvent, useEffect, useState } from "react";
import { PROTOCOL_VERSION } from "@/transport/handshake";
import { ConnectionStatusIndicator } from "./connection-status-indicator";
+import { POPUP_FEATURES, type PopupView } from "./features";
import { type PopupStatusState, useConnectionState } from "./use-connection-state";
const STATE_LABEL_KEYS = {
@@ -29,9 +36,15 @@ function getLogoSrc() {
export function App() {
const { t } = useTranslation("extension");
- const { snapshot, statusState, setLabel, setConnectionEnabled } = useConnectionState();
- const [labelDraft, setLabelDraft] = useState(snapshot.label);
+ const { snapshot, statusState, setConnectionEnabled } = useConnectionState();
+ const [view, setView] = useState("main");
const [copiedInstanceId, setCopiedInstanceId] = useState(false);
+ const [purposeDraft, setPurposeDraft] = useState("");
+ const [startUrlDraft, setStartUrlDraft] = useState("");
+ // Bumped on every successful copy so the "copied" toast re-shows (and its
+ // auto-hide timer restarts) even when the copied content is unchanged.
+ const [copiedTick, setCopiedTick] = useState(0);
+ const showCopiedToast = copiedTick > 0;
useEffect(() => {
const query = window.matchMedia("(prefers-color-scheme: dark)");
@@ -44,17 +57,22 @@ export function App() {
return () => query.removeEventListener("change", onChange);
}, []);
- useEffect(() => {
- setLabelDraft(snapshot.label);
- }, [snapshot.label]);
-
useEffect(() => {
setCopiedInstanceId(false);
+ setCopiedTick(0);
}, [snapshot.instanceId]);
- const onLabelBlur = () => {
- if (labelDraft !== snapshot.label) setLabel(labelDraft);
- };
+ // Hide the copied toast when the command changes (topic / start URL edits).
+ useEffect(() => {
+ setCopiedTick(0);
+ }, [purposeDraft, startUrlDraft]);
+
+ // Auto-hide the copied toast shortly after it appears.
+ useEffect(() => {
+ if (copiedTick === 0) return;
+ const timer = window.setTimeout(() => setCopiedTick(0), 1500);
+ return () => window.clearTimeout(timer);
+ }, [copiedTick]);
const isSkewed = statusState === "version_skew";
const daemonVersion = snapshot.handshake?.version ?? "—";
@@ -68,134 +86,302 @@ export function App() {
setCopiedInstanceId(true);
};
+ const recordReady = statusState === "connected" && Boolean(snapshot.instanceId);
+ const recordPurpose = purposeDraft.trim();
+ const recordStartUrl = startUrlDraft.trim();
+ const recordCommand = snapshot.instanceId
+ ? [
+ `bsk record start --browser ${snapshot.instanceId}`,
+ recordStartUrl ? `--url ${recordStartUrl}` : "",
+ recordPurpose ? `--purpose "${recordPurpose}"` : "",
+ ]
+ .filter(Boolean)
+ .join(" ")
+ : "";
+ const recordPrompt = snapshot.instanceId
+ ? t("popup.record.promptTemplate", {
+ command: recordCommand,
+ })
+ : "";
+
+ const copyRecordPrompt = async () => {
+ if (!recordReady || !navigator.clipboard?.writeText) return;
+ await navigator.clipboard.writeText(recordPrompt);
+ setCopiedTick((tick) => tick + 1);
+ };
+
+ const headerTitle =
+ view === "features"
+ ? t("popup.launcher.title")
+ : view === "record"
+ ? t("popup.record.sectionTitle")
+ : t("popup.brandName");
+
return (
-
-
- {t("popup.brandName")}
-
-
-
-
-
-
-
- {t(STATE_LABEL_KEYS[statusState])}
-
-
-
-
- {t(STATE_BADGE_KEYS[statusState])}
-
-
-
-
- {isSkewed && (
-
- {t("popup.versionSkewWarning", {
- extensionProtocol: PROTOCOL_VERSION,
- daemonProtocol,
- })}
-
- )}
-
-
- {snapshot.lastError && (
-
- {snapshot.lastError}
-
- )}
-
-
-
- ) => setLabelDraft(event.target.value)}
- onBlur={onLabelBlur}
- placeholder={t("popup.labelPlaceholder")}
- className="mt-0 h-8 text-sm"
- data-slot="popup-label-input"
- />
-
-
-
-
- {extensionVersion}
- /
- {daemonVersion}
-
-
- {t("popup.instanceTitle")}
-
- {instanceId}
-
+ {view !== "main" ? (
-
-
+ ) : (
+
+ )}
+
+ {headerTitle}
+
+ {view === "main" && (
+
+ )}
+
+
+ {view === "main" && (
+ <>
+
+
+
+
+
+ {t(STATE_LABEL_KEYS[statusState])}
+
+
+
+
+ {t(STATE_BADGE_KEYS[statusState])}
+
+
+
+
+ {isSkewed && (
+
+ {t("popup.versionSkewWarning", {
+ extensionProtocol: PROTOCOL_VERSION,
+ daemonProtocol,
+ })}
+
+ )}
+
+
+ {snapshot.lastError && (
+
+ {snapshot.lastError}
+
+ )}
+
+
+
+ {extensionVersion}
+ /
+ {daemonVersion}
+
+
+ {t("popup.instanceTitle")}
+
+ {instanceId}
+
+
+
+
+ >
+ )}
+
+ {view === "features" && (
+
+ {POPUP_FEATURES.map((feature) => {
+ const Icon = feature.icon;
+ return (
+
+ );
+ })}
+
+ )}
+
+ {view === "record" && (
+
+
+ {t("popup.record.sectionHint")}
+
+
+
+ ) =>
+ setPurposeDraft(event.target.value)
+ }
+ placeholder={t("popup.record.topicPlaceholder")}
+ className="mt-0 h-8 text-sm"
+ data-slot="popup-record-purpose-input"
+ />
+
+
+
+ ) =>
+ setStartUrlDraft(event.target.value)
+ }
+ placeholder={t("popup.record.startUrlPlaceholder")}
+ className="mt-0 h-8 text-sm"
+ data-slot="popup-record-start-url-input"
+ />
+
+
+
+ {!recordReady && (
+
+ {t("popup.record.hintDisconnected")}
+
+ )}
+
+ {showCopiedToast && (
+
+
+ {t("popup.record.copied")}
+
+ )}
+
+
+
+
+ )}
);
}
diff --git a/apps/extension/src/entrypoints/popup/features.ts b/apps/extension/src/entrypoints/popup/features.ts
new file mode 100644
index 0000000..84b2e86
--- /dev/null
+++ b/apps/extension/src/entrypoints/popup/features.ts
@@ -0,0 +1,22 @@
+import type { RemixiconComponentType } from "@remixicon/react";
+import { RiRecordCircleLine } from "@remixicon/react";
+
+export type PopupFeatureId = "record";
+
+export type PopupView = "main" | "features" | PopupFeatureId;
+
+export type PopupFeature = {
+ id: PopupFeatureId;
+ icon: RemixiconComponentType;
+ titleKey: "popup.record.sectionTitle";
+ descKey: "popup.record.cardDesc";
+};
+
+export const POPUP_FEATURES: PopupFeature[] = [
+ {
+ id: "record",
+ icon: RiRecordCircleLine,
+ titleKey: "popup.record.sectionTitle",
+ descKey: "popup.record.cardDesc",
+ },
+];
diff --git a/apps/extension/src/entrypoints/popup/style.css b/apps/extension/src/entrypoints/popup/style.css
index 1d1365a..20ed097 100644
--- a/apps/extension/src/entrypoints/popup/style.css
+++ b/apps/extension/src/entrypoints/popup/style.css
@@ -1,3 +1,13 @@
@import "@browser-skill/ui/styles";
@source "../../";
@source "../../../../../packages/ui/src";
+
+/* The shared base sets html/body/#root to height:100%, which is meant for
+ full-page surfaces. In a popup that pins the window to the tallest content
+ it ever showed, so collapsing the record section leaves blank space. Size
+ to content instead so Chrome can grow and shrink the popup to fit. */
+html,
+body,
+#root {
+ height: auto;
+}
diff --git a/apps/extension/src/lib/__tests__/describe-target.test.ts b/apps/extension/src/lib/__tests__/describe-target.test.ts
new file mode 100644
index 0000000..b265b16
--- /dev/null
+++ b/apps/extension/src/lib/__tests__/describe-target.test.ts
@@ -0,0 +1,113 @@
+import { beforeEach, describe, expect, it } from "vitest";
+import { describeEventTarget, describeTarget, isMeaningfulClickTarget } from "../describe-target";
+
+describe("describeTarget", () => {
+ beforeEach(() => {
+ document.body.innerHTML = "";
+ });
+
+ it("prefers aria-label and button role", () => {
+ document.body.innerHTML = ``;
+ const el = document.querySelector("button")!;
+ expect(describeTarget(el)).toEqual({
+ tag: "button",
+ role: "button",
+ name: "发布",
+ });
+ });
+
+ it("uses label text and name_attr for inputs", () => {
+ document.body.innerHTML = `
+
+
+ `;
+ const el = document.querySelector("input")!;
+ expect(describeTarget(el)).toEqual({
+ tag: "input",
+ role: "textbox",
+ name: "服务名称",
+ name_attr: "serviceName",
+ });
+ });
+
+ it("never invents CSS selectors or @eN", () => {
+ document.body.innerHTML = `详情`;
+ const desc = describeTarget(document.querySelector("a")!);
+ expect(JSON.stringify(desc)).not.toMatch(/@e\d+|nth-of-type|#/);
+ expect(desc).toEqual({ tag: "a", role: "link", name: "详情" });
+ });
+
+ it("prefers heading text inside SERP-style links", () => {
+ document.body.innerHTML = `
+
+ Tencent 腾讯
+ https://www.tencent.com
+
+ `;
+ expect(describeTarget(document.querySelector("a")!).name).toBe("Tencent 腾讯");
+ });
+});
+
+describe("LLM textbook click targets", () => {
+ beforeEach(() => {
+ document.body.innerHTML = "";
+ });
+
+ it("rejects tag-only targets the LLM cannot find", () => {
+ expect(isMeaningfulClickTarget({ tag: "div" })).toBe(false);
+ });
+
+ it("treats a short visible name as the teaching signal", () => {
+ expect(isMeaningfulClickTarget({ tag: "div", name: "发布" })).toBe(true);
+ });
+
+ it("ignores anonymous layout chrome and lifts child clicks to the named button", () => {
+ document.body.innerHTML = `
+ background
+
+ `;
+ expect(describeEventTarget(document.querySelector("#noise"))).toBeNull();
+ expect(describeEventTarget(document.querySelector("#icon"))).toEqual({
+ tag: "button",
+ role: "button",
+ name: "发布",
+ });
+ });
+
+ it("does not attach layout sibling text as nearby_label on buttons", () => {
+ document.body.innerHTML = `
+ background
+
+ `;
+ expect(describeTarget(document.querySelector("#pub")!)).toEqual({
+ tag: "button",
+ role: "button",
+ name: "发布",
+ });
+ });
+
+ it("keeps nearby_label for form fields from a preceding label-like sibling", () => {
+ document.body.innerHTML = `
+ 服务名称
+
+ `;
+ expect(describeTarget(document.querySelector("#svc")!)).toEqual(
+ expect.objectContaining({
+ tag: "input",
+ nearby_label: "服务名称",
+ }),
+ );
+ });
+
+ it("does not treat plain text blocks as clicks", () => {
+ document.body.innerHTML = `说明文字
`;
+ expect(describeEventTarget(document.querySelector("#copy"))).toBeNull();
+ });
+
+ it("records custom button-like controls by their visible label", () => {
+ document.body.innerHTML = `发布
`;
+ expect(describeEventTarget(document.querySelector("#pub"))).toEqual(
+ expect.objectContaining({ tag: "div", name: "发布" }),
+ );
+ });
+});
diff --git a/apps/extension/src/lib/__tests__/recording-step-buffer.test.ts b/apps/extension/src/lib/__tests__/recording-step-buffer.test.ts
new file mode 100644
index 0000000..396253a
--- /dev/null
+++ b/apps/extension/src/lib/__tests__/recording-step-buffer.test.ts
@@ -0,0 +1,80 @@
+import { describe, expect, it } from "vitest";
+import { appendRecordedPayload, observeRecordedNavigation } from "../recording-step-buffer";
+
+describe("recording-step-buffer", () => {
+ it("stores semantic click without summary", () => {
+ const buffer = { steps: [], pendingNavigation: false };
+ appendRecordedPayload(buffer, {
+ op: "click",
+ target: { tag: "button", role: "button", name: "发布" },
+ expects_navigation: true,
+ });
+ expect(buffer.steps).toEqual([
+ {
+ op: "click",
+ target: { tag: "button", role: "button", name: "发布" },
+ },
+ ]);
+ expect(buffer.pendingNavigation).toBe(true);
+ });
+
+ it("annotates navigated_to on action-caused navigation instead of wait_for_navigation", () => {
+ const buffer = {
+ steps: [
+ {
+ op: "click" as const,
+ target: { tag: "button", role: "button", name: "发布" },
+ },
+ ],
+ currentUrl: "https://example.com/a",
+ pendingNavigation: true,
+ pendingNavigationDeadline: Date.now() + 5_000,
+ };
+ observeRecordedNavigation(buffer, "https://example.com/b", true);
+ expect(buffer.steps).toEqual([
+ {
+ op: "click",
+ target: { tag: "button", role: "button", name: "发布" },
+ navigated_to: "https://example.com/b",
+ },
+ ]);
+ expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation");
+ });
+
+ it("annotates navigated_to onto a select that auto-submits a navigation", () => {
+ const buffer = {
+ steps: [
+ {
+ op: "select" as const,
+ target: { tag: "select", role: "combobox", name: "分类" },
+ values: ["tech"],
+ },
+ ],
+ currentUrl: "https://example.com/list",
+ pendingNavigation: true,
+ pendingNavigationDeadline: Date.now() + 5_000,
+ };
+ observeRecordedNavigation(buffer, "https://example.com/list?cat=tech", true);
+ expect(buffer.steps).toEqual([
+ {
+ op: "select",
+ target: { tag: "select", role: "combobox", name: "分类" },
+ values: ["tech"],
+ navigated_to: "https://example.com/list?cat=tech",
+ },
+ ]);
+ });
+
+ it("emits navigate for uncaused URL changes", () => {
+ const buffer = {
+ steps: [],
+ currentUrl: "https://example.com/a",
+ pendingNavigation: false,
+ };
+ observeRecordedNavigation(buffer, "https://example.com/b", false);
+ expect(buffer.steps[0]).toMatchObject({
+ op: "navigate",
+ url: "https://example.com/b",
+ });
+ });
+});
diff --git a/apps/extension/src/lib/__tests__/trace-reducer.test.ts b/apps/extension/src/lib/__tests__/trace-reducer.test.ts
new file mode 100644
index 0000000..777a86c
--- /dev/null
+++ b/apps/extension/src/lib/__tests__/trace-reducer.test.ts
@@ -0,0 +1,141 @@
+import { describe, expect, it } from "vitest";
+import type { DraftTraceStep } from "@/transport/types";
+import { reduceTraceSteps, resolveTraceStartUrl, shouldRecordPress } from "../trace-reducer";
+
+describe("shouldRecordPress", () => {
+ it("keeps Enter and Escape", () => {
+ expect(shouldRecordPress("Enter")).toBe(true);
+ expect(shouldRecordPress("Escape")).toBe(true);
+ });
+
+ it("drops modifiers, clipboard shortcuts, and bare typing", () => {
+ expect(shouldRecordPress("Meta")).toBe(false);
+ expect(shouldRecordPress("c", ["meta"])).toBe(false);
+ expect(shouldRecordPress("a", ["ctrl"])).toBe(false);
+ expect(shouldRecordPress("x")).toBe(false);
+ expect(shouldRecordPress("中")).toBe(false);
+ });
+});
+
+describe("reduceTraceSteps", () => {
+ it("builds steps with pages dictionary and page id references", () => {
+ const drafts: DraftTraceStep[] = [
+ {
+ op: "navigate",
+ url: "https://example.com/search?q=hello&utm_source=x",
+ page_url: "https://example.com/search?q=hello&utm_source=x",
+ },
+ {
+ op: "fill",
+ target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
+ value: "browser skill",
+ page_url: "https://example.com/search?q=hello&utm_source=x",
+ },
+ {
+ op: "press",
+ key: "Enter",
+ target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
+ navigated_to: "https://example.com/results/42",
+ page_url: "https://example.com/search?q=hello&utm_source=x",
+ },
+ {
+ op: "click",
+ target: { tag: "button", role: "button", name: "发布" },
+ navigated_to: "https://example.com/p/99",
+ page_url: "https://example.com/results/42",
+ },
+ {
+ op: "press",
+ key: "a",
+ page_url: "https://example.com/p/99",
+ },
+ ];
+
+ const { pages, steps } = reduceTraceSteps(
+ drafts,
+ "https://example.com/search?q=hello&utm_source=x",
+ );
+ expect(JSON.stringify(steps)).not.toContain("parameters");
+ expect(JSON.stringify(steps)).not.toContain("intent");
+ expect(JSON.stringify(steps)).not.toContain("summary");
+ expect(steps.map((s) => s.op)).toEqual(["navigate", "fill", "press", "click"]);
+ expect(pages.map((p) => p.url)).toEqual([
+ "https://example.com/search?q=hello&utm_source=x",
+ "https://example.com/results/42",
+ "https://example.com/p/99",
+ ]);
+
+ expect(steps[0]).toMatchObject({
+ id: 1,
+ op: "navigate",
+ page: "p1",
+ to: "https://example.com/search?q=hello&utm_source=x",
+ });
+
+ expect(steps[1]).toMatchObject({
+ op: "fill",
+ page: "p1",
+ value: "browser skill",
+ });
+
+ expect(steps[2]).toMatchObject({
+ op: "press",
+ key: "Enter",
+ page: "p1",
+ effect: { navigated_to: "p2" },
+ });
+
+ expect(steps[3]).toMatchObject({
+ op: "click",
+ page: "p2",
+ effect: { navigated_to: "p3" },
+ });
+ });
+
+ it("collapses consecutive navigations", () => {
+ const { steps } = reduceTraceSteps([
+ { op: "navigate", url: "https://a.example/redirect1" },
+ { op: "navigate", url: "https://a.example/final" },
+ ]);
+ expect(steps).toHaveLength(1);
+ expect(steps[0]).toMatchObject({
+ op: "navigate",
+ to: "https://a.example/final",
+ });
+ });
+
+ it("maps select navigated_to onto effect.navigated_to (page id)", () => {
+ const { pages, steps } = reduceTraceSteps(
+ [
+ {
+ op: "select",
+ target: { tag: "select", role: "combobox", name: "分类" },
+ values: ["tech"],
+ labels: ["技术"],
+ navigated_to: "https://example.com/list?cat=tech",
+ page_url: "https://example.com/list",
+ },
+ ],
+ "https://example.com/list",
+ );
+ expect(pages.map((p) => p.url)).toEqual([
+ "https://example.com/list",
+ "https://example.com/list?cat=tech",
+ ]);
+ expect(steps[0]).toMatchObject({
+ op: "select",
+ page: "p1",
+ selection: [{ value: "tech", label: "技术" }],
+ effect: { navigated_to: "p2" },
+ });
+ });
+
+ it("resolveTraceStartUrl prefers explicit start URL", () => {
+ expect(
+ resolveTraceStartUrl(
+ [{ op: "navigate", url: "https://example.com/other" }],
+ "https://example.com/start",
+ ),
+ ).toBe("https://example.com/start");
+ });
+});
diff --git a/apps/extension/src/lib/describe-target.ts b/apps/extension/src/lib/describe-target.ts
new file mode 100644
index 0000000..7d3898a
--- /dev/null
+++ b/apps/extension/src/lib/describe-target.ts
@@ -0,0 +1,345 @@
+/**
+ * Build a semantic TargetDescriptor for an interacted element.
+ *
+ * Trace steps are an LLM *textbook*: each click must say what to look for
+ * on screen (usually a short visible name). Tag-only noise like
+ * `{ "tag": "div" }` / “点击div” is useless and must not be recorded.
+ */
+
+export interface TargetDescriptor {
+ role?: string;
+ name?: string;
+ tag: string;
+ name_attr?: string;
+ placeholder?: string;
+ nearby_label?: string;
+}
+
+/** Max length for a label that is still a useful “find this on screen” hint. */
+const ACTIONABLE_LABEL_MAX = 48;
+
+function normalizeWhitespace(value: string): string {
+ return value.replace(/\s+/g, " ").trim();
+}
+
+function truncate(value: string, max = 80): string {
+ const trimmed = normalizeWhitespace(value);
+ if (trimmed.length <= max) return trimmed;
+ return `${trimmed.slice(0, max - 1)}…`;
+}
+
+/** Short, human label an LLM can search for in a snapshot. */
+export function isActionableLabel(name: string): boolean {
+ const trimmed = normalizeWhitespace(name);
+ if (!trimmed) return false;
+ if (trimmed.length > ACTIONABLE_LABEL_MAX) return false;
+ if (/^https?:\/\//i.test(trimmed)) return false;
+ return true;
+}
+
+function labelledByText(el: Element): string | null {
+ const ids = el.getAttribute("aria-labelledby");
+ if (!ids) return null;
+ const parts = ids
+ .split(/\s+/)
+ .map((id) => document.getElementById(id)?.textContent ?? "")
+ .map(normalizeWhitespace)
+ .filter(Boolean);
+ return parts.length ? truncate(parts.join(" ")) : null;
+}
+
+function associatedLabelText(el: Element): string | null {
+ if (!(el instanceof HTMLElement) || !el.id) return null;
+ const label = document.querySelector(`label[for="${CSS.escape(el.id)}"]`);
+ if (!label) return null;
+ const text = normalizeWhitespace(label.textContent ?? "");
+ return text ? truncate(text) : null;
+}
+
+function wrappingLabelText(el: Element): string | null {
+ const label = el.closest("label");
+ if (!label) return null;
+ const clone = label.cloneNode(true) as HTMLElement;
+ for (const control of clone.querySelectorAll("input,textarea,select,button")) {
+ control.remove();
+ }
+ const text = normalizeWhitespace(clone.textContent ?? "");
+ return text ? truncate(text) : null;
+}
+
+function placeholderOrTitle(el: Element): string | null {
+ if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
+ const ph = normalizeWhitespace(el.placeholder);
+ if (ph) return truncate(ph);
+ }
+ const title = normalizeWhitespace(el.getAttribute("title") ?? "");
+ return title ? truncate(title) : null;
+}
+
+function ownText(el: Element): string | null {
+ if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
+ return null;
+ }
+ const text = normalizeWhitespace(el.textContent ?? "");
+ return text ? truncate(text) : null;
+}
+
+/** Prefer a compact title inside links (e.g. SERP result ``), not the whole blob. */
+function compactLinkName(el: Element): string | null {
+ const heading = el.querySelector("h1, h2, h3, h4");
+ if (heading) {
+ const text = normalizeWhitespace(heading.textContent ?? "");
+ if (text) return truncate(text);
+ }
+ const img = el.querySelector("img[alt]");
+ if (img instanceof HTMLImageElement) {
+ const alt = normalizeWhitespace(img.alt);
+ if (alt) return truncate(alt);
+ }
+ const clone = el.cloneNode(true) as HTMLElement;
+ for (const junk of clone.querySelectorAll("cite, script, style, svg, noscript")) {
+ junk.remove();
+ }
+ const text = normalizeWhitespace(clone.textContent ?? "");
+ if (!text) return null;
+ const withoutUrl = text.replace(/https?:\/\/\S+/g, "").trim();
+ return truncate(withoutUrl || text);
+}
+
+/** Visible accessible name preference order for LLM textbooks. */
+export function accessibleName(el: Element): string | undefined {
+ const ariaLabel = normalizeWhitespace(el.getAttribute("aria-label") ?? "");
+ if (ariaLabel) return truncate(ariaLabel);
+
+ const labelledBy = labelledByText(el);
+ if (labelledBy) return labelledBy;
+
+ const forLabel = associatedLabelText(el);
+ if (forLabel) return forLabel;
+
+ const wrapLabel = wrappingLabelText(el);
+ if (wrapLabel) return wrapLabel;
+
+ const alt =
+ el instanceof HTMLImageElement
+ ? normalizeWhitespace(el.alt)
+ : normalizeWhitespace(el.getAttribute("alt") ?? "");
+ if (alt) return truncate(alt);
+
+ const ph = placeholderOrTitle(el);
+ if (ph) return ph;
+
+ const tag = el.tagName.toLowerCase();
+ if (tag === "a" || el.getAttribute("role") === "link") {
+ const linkName = compactLinkName(el);
+ if (linkName) return linkName;
+ }
+
+ const text = ownText(el);
+ if (text) return text;
+
+ if (el instanceof HTMLInputElement || el instanceof HTMLButtonElement) {
+ const value = normalizeWhitespace(el.value);
+ if (
+ value &&
+ (el instanceof HTMLButtonElement || el.type === "submit" || el.type === "button")
+ ) {
+ return truncate(value);
+ }
+ }
+
+ return undefined;
+}
+
+const INTERACTIVE_SELECTOR = [
+ "a[href]",
+ "button",
+ "summary",
+ "select",
+ 'input:not([type="hidden"])',
+ "textarea",
+ '[role="button"]',
+ '[role="link"]',
+ '[role="tab"]',
+ '[role="menuitem"]',
+ '[role="menuitemcheckbox"]',
+ '[role="menuitemradio"]',
+ '[role="option"]',
+ '[role="checkbox"]',
+ '[role="radio"]',
+ '[role="switch"]',
+ '[role="treeitem"]',
+ '[role="combobox"]',
+ '[contenteditable]:not([contenteditable="false"])',
+].join(",");
+
+const INTERACTIVE_ROLES = new Set([
+ "button",
+ "link",
+ "tab",
+ "menuitem",
+ "menuitemcheckbox",
+ "menuitemradio",
+ "option",
+ "checkbox",
+ "radio",
+ "switch",
+ "treeitem",
+ "combobox",
+ "textbox",
+ "searchbox",
+ "slider",
+]);
+
+function looksClickable(el: Element): boolean {
+ if (!(el instanceof HTMLElement)) return false;
+ if (el.matches(INTERACTIVE_SELECTOR)) return true;
+ const role = normalizeWhitespace(el.getAttribute("role") ?? "").toLowerCase();
+ if (role && INTERACTIVE_ROLES.has(role)) return true;
+ if (el.tabIndex >= 0) return true;
+ try {
+ if (getComputedStyle(el).cursor === "pointer") return true;
+ } catch {
+ // happy-dom / detached nodes
+ }
+ const cls = typeof el.className === "string" ? el.className : "";
+ if (/\b(btn|button|link|clickable|action)\b/i.test(cls)) return true;
+ return false;
+}
+
+/**
+ * Resolve the element a click should describe for the LLM textbook.
+ * Never falls back to an anonymous layout `div`.
+ */
+export function resolveClickableElement(target: Element): Element | null {
+ const interactive = target.closest(INTERACTIVE_SELECTOR);
+ if (interactive instanceof Element) return interactive;
+
+ let node: Element | null = target;
+ let depth = 0;
+ while (node && node !== document.body && node !== document.documentElement && depth < 8) {
+ const name = accessibleName(node);
+ if (name && isActionableLabel(name) && looksClickable(node)) {
+ return node;
+ }
+ node = node.parentElement;
+ depth += 1;
+ }
+ return null;
+}
+
+/**
+ * Teachable clicks: the LLM must get a label it can later find via snapshot
+ * (visible name), or at least a form `name_attr` for checkbox/radio.
+ * Recording “点击div” with no name fails this bar.
+ */
+export function isMeaningfulClickTarget(target: TargetDescriptor): boolean {
+ const name = target.name?.trim();
+ if (name && isActionableLabel(name)) return true;
+ if (
+ (target.role === "checkbox" || target.role === "radio" || target.tag === "input") &&
+ target.name_attr
+ ) {
+ return true;
+ }
+ return false;
+}
+
+export function inferRole(el: Element): string | undefined {
+ const explicit = normalizeWhitespace(el.getAttribute("role") ?? "").toLowerCase();
+ if (explicit) return explicit;
+
+ const tag = el.tagName.toLowerCase();
+ if (tag === "button") return "button";
+ if (tag === "a" && el.hasAttribute("href")) return "link";
+ if (tag === "select") return "combobox";
+ if (tag === "textarea") return "textbox";
+ if (tag === "img") return "img";
+ if (tag === "summary") return "button";
+ if (el instanceof HTMLElement && (el.isContentEditable || el.contentEditable === "true")) {
+ return "textbox";
+ }
+ if (el instanceof HTMLInputElement) {
+ const type = el.type.toLowerCase();
+ if (type === "checkbox") return "checkbox";
+ if (type === "radio") return "radio";
+ if (type === "submit" || type === "button" || type === "image" || type === "reset") {
+ return "button";
+ }
+ if (type === "range") return "slider";
+ return "textbox";
+ }
+ return undefined;
+}
+
+/**
+ * Nearby labels help the LLM name form fields. Only apply to fillable
+ * controls — never to buttons/links, where a previous layout sibling
+ * (e.g. a chrome ``) would pollute the textbook target.
+ */
+function nearbyLabelText(el: Element): string | undefined {
+ if (
+ !(
+ el instanceof HTMLInputElement ||
+ el instanceof HTMLTextAreaElement ||
+ el instanceof HTMLSelectElement
+ )
+ ) {
+ return undefined;
+ }
+ if (el.id) {
+ const byFor = document.querySelector(`label[for="${CSS.escape(el.id)}"]`);
+ const text = byFor ? normalizeWhitespace(byFor.textContent ?? "") : "";
+ if (text) return truncate(text, ACTIONABLE_LABEL_MAX);
+ }
+ const wrapped = el.closest("label");
+ if (wrapped) {
+ const clone = wrapped.cloneNode(true) as HTMLLabelElement;
+ for (const control of clone.querySelectorAll("input, textarea, select, button")) {
+ control.remove();
+ }
+ const text = normalizeWhitespace(clone.textContent ?? "");
+ if (text) return truncate(text, ACTIONABLE_LABEL_MAX);
+ }
+ const prev = el.previousElementSibling;
+ if (prev && /^(LABEL|SPAN|DIV|P|DT)$/i.test(prev.tagName)) {
+ const text = normalizeWhitespace(prev.textContent ?? "");
+ if (text && text.length <= ACTIONABLE_LABEL_MAX) return text;
+ }
+ return undefined;
+}
+
+export function describeTarget(el: Element): TargetDescriptor {
+ const tag = el.tagName.toLowerCase();
+ const role = inferRole(el);
+ const name = accessibleName(el);
+ const nameAttr =
+ el instanceof HTMLInputElement ||
+ el instanceof HTMLTextAreaElement ||
+ el instanceof HTMLSelectElement ||
+ el instanceof HTMLButtonElement
+ ? normalizeWhitespace(el.name) || undefined
+ : normalizeWhitespace(el.getAttribute("name") ?? "") || undefined;
+ const placeholder =
+ el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement
+ ? normalizeWhitespace(el.placeholder) || undefined
+ : normalizeWhitespace(el.getAttribute("placeholder") ?? "") || undefined;
+ const nearby = nearbyLabelText(el);
+
+ return {
+ tag,
+ ...(role ? { role } : {}),
+ ...(name ? { name } : {}),
+ ...(nameAttr ? { name_attr: nameAttr } : {}),
+ ...(placeholder ? { placeholder } : {}),
+ ...(nearby && nearby !== name ? { nearby_label: nearby } : {}),
+ };
+}
+
+export function describeEventTarget(target: EventTarget | null): TargetDescriptor | null {
+ if (!(target instanceof Element)) return null;
+ const clickable = resolveClickableElement(target);
+ if (!clickable) return null;
+ const desc = describeTarget(clickable);
+ return isMeaningfulClickTarget(desc) ? desc : null;
+}
diff --git a/apps/extension/src/lib/record-bridge.ts b/apps/extension/src/lib/record-bridge.ts
new file mode 100644
index 0000000..cb5d446
--- /dev/null
+++ b/apps/extension/src/lib/record-bridge.ts
@@ -0,0 +1,111 @@
+/**
+ * Wire protocol for user-action recording, sent between the background
+ * service worker and a tab's content script.
+ */
+
+import type { TargetDescriptor } from "./describe-target";
+
+export const RECORD_START = "bsk-record-start";
+export const RECORD_STEP = "bsk-record-step";
+export const RECORD_STOP = "bsk-record-stop";
+export const RECORD_CANCEL = "bsk-record-cancel";
+export const RECORD_FINISH = "bsk-record-finish";
+export const RECORD_QUERY = "bsk-record-query";
+
+export interface RecordStartAck {
+ ok: true;
+}
+
+export type RecordStopAck = { ok: true } | { ok: false; error: string };
+
+export interface RecordQueryMessage {
+ type: typeof RECORD_QUERY;
+}
+
+export interface RecordQueryResponse {
+ active: boolean;
+ requestId?: string;
+}
+
+export interface RecordStartMessage {
+ type: typeof RECORD_START;
+ requestId: string;
+}
+
+export interface RecordStepPayload {
+ op: "click" | "fill" | "press" | "select" | "navigate";
+ target?: TargetDescriptor;
+ value?: string;
+ key?: string;
+ modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">;
+ values?: string[];
+ labels?: string[];
+ url?: string;
+ redacted?: boolean;
+ /** Page URL when the step was captured. */
+ page_url?: string;
+ /** Capture-only hint; never persisted unless converted to navigated_to. */
+ expects_navigation?: boolean;
+ /** Whether an observed URL change was synchronously caused by the action. */
+ navigation_caused_by_action?: boolean;
+}
+
+export interface RecordStepMessage {
+ type: typeof RECORD_STEP;
+ requestId: string;
+ step: RecordStepPayload;
+}
+
+export interface RecordStopMessage {
+ type: typeof RECORD_STOP;
+ requestId: string;
+}
+
+export interface RecordCancelMessage {
+ type: typeof RECORD_CANCEL;
+ requestId: string;
+}
+
+export interface RecordFinishMessage {
+ type: typeof RECORD_FINISH;
+ requestId: string;
+}
+
+export function isRecordStartMessage(msg: unknown): msg is RecordStartMessage {
+ if (typeof msg !== "object" || msg === null) return false;
+ const m = msg as Record
;
+ return m.type === RECORD_START && typeof m.requestId === "string";
+}
+
+export function isRecordStopMessage(msg: unknown): msg is RecordStopMessage {
+ if (typeof msg !== "object" || msg === null) return false;
+ const m = msg as Record;
+ return m.type === RECORD_STOP && typeof m.requestId === "string";
+}
+
+export function isRecordCancelMessage(msg: unknown): msg is RecordCancelMessage {
+ if (typeof msg !== "object" || msg === null) return false;
+ const m = msg as Record;
+ return m.type === RECORD_CANCEL && typeof m.requestId === "string";
+}
+
+export function isRecordFinishMessage(msg: unknown): msg is RecordFinishMessage {
+ if (typeof msg !== "object" || msg === null) return false;
+ const m = msg as Record;
+ return m.type === RECORD_FINISH && typeof m.requestId === "string";
+}
+
+export function isRecordQueryMessage(msg: unknown): msg is RecordQueryMessage {
+ if (typeof msg !== "object" || msg === null) return false;
+ const m = msg as Record;
+ return m.type === RECORD_QUERY;
+}
+
+export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage {
+ if (typeof msg !== "object" || msg === null) return false;
+ const m = msg as Record;
+ if (m.type !== RECORD_STEP || typeof m.requestId !== "string") return false;
+ const step = m.step;
+ if (typeof step !== "object" || step === null) return false;
+ return typeof (step as RecordStepPayload).op === "string";
+}
diff --git a/apps/extension/src/lib/recording-step-buffer.ts b/apps/extension/src/lib/recording-step-buffer.ts
new file mode 100644
index 0000000..0c7c26e
--- /dev/null
+++ b/apps/extension/src/lib/recording-step-buffer.ts
@@ -0,0 +1,128 @@
+import type { DraftTraceStep } from "@/transport/types";
+import type { RecordStepPayload } from "./record-bridge";
+
+export interface RecordingStepBuffer {
+ steps: DraftTraceStep[];
+ currentUrl?: string;
+ pendingNavigation: boolean;
+ pendingNavigationDeadline?: number;
+}
+
+const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
+
+function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
+ const pageUrl = payload.page_url;
+ switch (payload.op) {
+ case "click":
+ return payload.target
+ ? {
+ op: "click",
+ target: payload.target,
+ ...(pageUrl ? { page_url: pageUrl } : {}),
+ }
+ : null;
+ case "fill":
+ return payload.target
+ ? {
+ op: "fill",
+ target: payload.target,
+ value: payload.value ?? "",
+ ...(payload.redacted ? { redacted: true } : {}),
+ ...(pageUrl ? { page_url: pageUrl } : {}),
+ }
+ : null;
+ case "press":
+ return payload.key
+ ? {
+ op: "press",
+ key: payload.key,
+ ...(payload.target ? { target: payload.target } : {}),
+ ...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
+ ...(pageUrl ? { page_url: pageUrl } : {}),
+ }
+ : null;
+ case "select":
+ return payload.target && payload.values
+ ? {
+ op: "select",
+ target: payload.target,
+ values: payload.values,
+ ...(payload.labels?.length ? { labels: payload.labels } : {}),
+ ...(pageUrl ? { page_url: pageUrl } : {}),
+ }
+ : null;
+ case "navigate":
+ return null;
+ }
+}
+
+function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): boolean {
+ for (let i = buffer.steps.length - 1; i >= 0; i -= 1) {
+ const step = buffer.steps[i];
+ if (!step) continue;
+ if (step.op === "click" || step.op === "press" || step.op === "select") {
+ buffer.steps[i] = { ...step, navigated_to: url };
+ return true;
+ }
+ break;
+ }
+ return false;
+}
+
+export function observeRecordedNavigation(
+ buffer: RecordingStepBuffer,
+ url: string,
+ causedByAction?: boolean,
+): void {
+ if (!url || url === buffer.currentUrl) return;
+ buffer.currentUrl = url;
+ const pendingIsCurrent =
+ buffer.pendingNavigation &&
+ (buffer.pendingNavigationDeadline === undefined ||
+ buffer.pendingNavigationDeadline >= Date.now());
+
+ if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
+ buffer.pendingNavigation = false;
+ buffer.pendingNavigationDeadline = undefined;
+ if (!annotateLastStepNavigation(buffer, url)) {
+ buffer.steps.push({
+ op: "navigate",
+ url,
+ page_url: url,
+ });
+ }
+ return;
+ }
+
+ buffer.pendingNavigation = false;
+ buffer.pendingNavigationDeadline = undefined;
+ buffer.steps.push({
+ op: "navigate",
+ url,
+ page_url: url,
+ });
+}
+
+export function appendRecordedPayload(
+ buffer: RecordingStepBuffer,
+ payload: RecordStepPayload,
+): void {
+ if (payload.op === "navigate") {
+ if (payload.url) {
+ observeRecordedNavigation(buffer, payload.url, payload.navigation_caused_by_action);
+ }
+ return;
+ }
+ const step = toDraftStep({
+ ...payload,
+ page_url: payload.page_url ?? buffer.currentUrl,
+ });
+ if (!step) return;
+ buffer.steps.push(step);
+ if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
+ buffer.pendingNavigation = payload.expects_navigation === true;
+ buffer.pendingNavigationDeadline = buffer.pendingNavigation
+ ? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
+ : undefined;
+ }
+}
diff --git a/apps/extension/src/lib/trace-reducer.ts b/apps/extension/src/lib/trace-reducer.ts
new file mode 100644
index 0000000..e3572ee
--- /dev/null
+++ b/apps/extension/src/lib/trace-reducer.ts
@@ -0,0 +1,218 @@
+import type { DraftTraceStep, PageRef, SelectedOption, Step } from "@/transport/types";
+
+const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
+const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
+
+export function shouldRecordPress(
+ key: string,
+ modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
+): boolean {
+ if (MODIFIER_ONLY_KEYS.has(key)) return false;
+ const mods = modifiers ?? [];
+ const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
+ if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
+ if (key === "Enter" || key === "Escape") return true;
+ // Drop bare character typing — FillSession already records the value.
+ if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
+ return false;
+}
+
+function shouldIncludeDraft(step: DraftTraceStep): boolean {
+ if (step.op === "fill" && !(step.value ?? "").trim() && !step.redacted) return false;
+ if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
+ return true;
+}
+
+/** Collapse consecutive navigations to the last hop. */
+function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
+ const out: DraftTraceStep[] = [];
+ for (const step of steps) {
+ const prev = out[out.length - 1];
+ if (step.op === "navigate" && prev?.op === "navigate") {
+ out[out.length - 1] = step;
+ continue;
+ }
+ out.push(step);
+ }
+ return out;
+}
+
+function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
+ const urls: string[] = [];
+ if (startUrl) urls.push(startUrl);
+ for (const step of steps) {
+ if (step.op === "navigate") {
+ urls.push(step.url);
+ continue;
+ }
+ if ("page_url" in step && step.page_url) urls.push(step.page_url);
+ if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to);
+ }
+ const seen = new Set();
+ const unique: string[] = [];
+ for (const url of urls) {
+ if (!url || seen.has(url)) continue;
+ seen.add(url);
+ unique.push(url);
+ }
+ return unique;
+}
+
+function buildPageRegistry(
+ steps: DraftTraceStep[],
+ startUrl?: string,
+): { pages: PageRef[]; urlToId: Map } {
+ const urls = collectUrls(steps, startUrl);
+ const urlToId = new Map();
+ const pages = urls.map((url, index) => {
+ const id = `p${index + 1}`;
+ urlToId.set(url, id);
+ return { id, url };
+ });
+ return { pages, urlToId };
+}
+
+function pageIdFor(
+ url: string | undefined,
+ urlToId: Map,
+ fallbackUrl?: string,
+): string {
+ if (url && urlToId.has(url)) return urlToId.get(url)!;
+ if (fallbackUrl && urlToId.has(fallbackUrl)) return urlToId.get(fallbackUrl)!;
+ return urlToId.values().next().value ?? "p1";
+}
+
+function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined {
+ if (step.op === "navigate") return step.page_url ?? step.url;
+ if ("page_url" in step && step.page_url) return step.page_url;
+ return fallbackUrl;
+}
+
+function effectForNavigation(
+ navigatedTo: string | undefined,
+ urlToId: Map,
+): Step["effect"] {
+ if (!navigatedTo) return undefined;
+ const pageId = urlToId.get(navigatedTo);
+ if (!pageId) return undefined;
+ return { navigated_to: pageId };
+}
+
+function withEffect(step: Step, effect: Step["effect"]): Step {
+ if (!effect) return step;
+ return { ...step, effect };
+}
+
+function toSelection(values: string[], labels?: string[]): SelectedOption[] {
+ return values.map((value, index) => ({
+ value,
+ ...(labels?.[index] ? { label: labels[index] } : {}),
+ }));
+}
+
+function toV2Step(
+ step: DraftTraceStep,
+ id: number,
+ urlToId: Map,
+ fallbackUrl?: string,
+): Step | null {
+ if (!shouldIncludeDraft(step)) return null;
+
+ const pageUrl = pageUrlForDraft(step, fallbackUrl);
+ const page = pageIdFor(pageUrl, urlToId, fallbackUrl);
+
+ switch (step.op) {
+ case "navigate":
+ return {
+ op: "navigate",
+ id,
+ page: pageIdFor(step.url, urlToId, fallbackUrl),
+ to: step.url,
+ };
+ case "click":
+ return withEffect(
+ {
+ op: "click",
+ id,
+ page,
+ target: step.target,
+ },
+ effectForNavigation(step.navigated_to, urlToId),
+ );
+ case "fill":
+ return {
+ op: "fill",
+ id,
+ page,
+ target: step.target,
+ value: step.value,
+ ...(step.redacted ? { redacted: true } : {}),
+ };
+ case "press":
+ return withEffect(
+ {
+ op: "press",
+ id,
+ page,
+ key: step.key,
+ ...(step.target ? { target: step.target } : {}),
+ ...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
+ },
+ effectForNavigation(step.navigated_to, urlToId),
+ );
+ case "select":
+ return withEffect(
+ {
+ op: "select",
+ id,
+ page,
+ target: step.target,
+ selection: toSelection(step.values, step.labels),
+ },
+ effectForNavigation(step.navigated_to, urlToId),
+ );
+ }
+}
+
+export interface ReducedTrace {
+ pages: PageRef[];
+ steps: Step[];
+}
+
+/**
+ * Compile capture drafts into record-only trace v2 steps.
+ * Variable inputs are NOT classified here — executing agents infer that at run time.
+ */
+export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): ReducedTrace {
+ const collapsed = collapseNavigations(steps);
+ const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
+ const out: Step[] = [];
+ let id = 1;
+ let lastUrl = startUrl;
+ for (const draft of collapsed) {
+ if (draft.op === "navigate") lastUrl = draft.url;
+ else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to;
+ else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url;
+ const step = toV2Step(draft, id, urlToId, lastUrl);
+ if (!step) continue;
+ out.push(step);
+ id += 1;
+ }
+ return { pages, steps: out };
+}
+
+export function resolveTraceStartUrl(
+ drafts: DraftTraceStep[],
+ startUrl?: string,
+ pages?: PageRef[],
+): string {
+ if (startUrl) return startUrl;
+ const navigate = drafts.find((step): step is Extract => {
+ return step.op === "navigate";
+ });
+ if (navigate) return navigate.url;
+ for (const draft of drafts) {
+ if ("page_url" in draft && draft.page_url) return draft.page_url;
+ }
+ return pages?.[0]?.url ?? "about:blank";
+}
diff --git a/apps/extension/src/tools/__tests__/record-observation.test.ts b/apps/extension/src/tools/__tests__/record-observation.test.ts
new file mode 100644
index 0000000..32b700f
--- /dev/null
+++ b/apps/extension/src/tools/__tests__/record-observation.test.ts
@@ -0,0 +1,59 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import {
+ ensureBrowserObservationListeners,
+ isBrowserObservationAttachedForTests,
+ RECORD_DEFAULT_START_URL,
+ releaseBrowserObservationListenersIfIdle,
+ resetBrowserObservationForTests,
+ setBrowserObservationAttachForTests,
+} from "../record";
+
+describe("record defaults", () => {
+ it("uses example.com as the default injectable start URL", () => {
+ expect(RECORD_DEFAULT_START_URL).toBe("https://example.com/");
+ });
+});
+
+describe("browser observation lifecycle", () => {
+ afterEach(() => {
+ resetBrowserObservationForTests();
+ });
+
+ it("does not attach listeners until ensure is called", () => {
+ const tab = vi.fn(() => () => undefined);
+ const nav = vi.fn(() => () => undefined);
+ setBrowserObservationAttachForTests(tab, nav);
+
+ expect(isBrowserObservationAttachedForTests()).toBe(false);
+ expect(tab).not.toHaveBeenCalled();
+ expect(nav).not.toHaveBeenCalled();
+ });
+
+ it("attaches once on ensure and detaches when idle", () => {
+ const detachTab = vi.fn();
+ const detachNav = vi.fn();
+ const tab = vi.fn(() => detachTab);
+ const nav = vi.fn(() => detachNav);
+ setBrowserObservationAttachForTests(tab, nav);
+
+ ensureBrowserObservationListeners({
+ tabsApi: { get: vi.fn(), query: vi.fn(), create: vi.fn() } as never,
+ sendToTab: vi.fn(),
+ });
+ expect(isBrowserObservationAttachedForTests()).toBe(true);
+ expect(tab).toHaveBeenCalledTimes(1);
+ expect(nav).toHaveBeenCalledTimes(1);
+
+ // Idempotent.
+ ensureBrowserObservationListeners({
+ tabsApi: { get: vi.fn(), query: vi.fn(), create: vi.fn() } as never,
+ sendToTab: vi.fn(),
+ });
+ expect(tab).toHaveBeenCalledTimes(1);
+
+ releaseBrowserObservationListenersIfIdle();
+ expect(isBrowserObservationAttachedForTests()).toBe(false);
+ expect(detachTab).toHaveBeenCalledTimes(1);
+ expect(detachNav).toHaveBeenCalledTimes(1);
+ });
+});
diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts
index 669727f..5091141 100644
--- a/apps/extension/src/tools/dispatcher.ts
+++ b/apps/extension/src/tools/dispatcher.ts
@@ -13,6 +13,9 @@ import type {
NetworkParams,
PressParams,
ProtocolFrame,
+ RecordAwaitParams,
+ RecordStartParams,
+ RecordStopParams,
ReloadParams,
RequestFrame,
RequestHelpParams,
@@ -42,6 +45,7 @@ import {
handleScreenshot,
handleSnapshot,
} from "./observation";
+import { handleRecordAwait, handleRecordStart, handleRecordStop } from "./record";
import {
handleSessionStart,
handleSessionStop,
@@ -396,6 +400,45 @@ export class ToolDispatcher {
notificationCopy: this.helpNotificationCopy?.(),
signal,
});
+ case "tool.record_start":
+ return handleRecordStart(this.sessions, req.params as RecordStartParams, {
+ tabsApi: chromeTabsApi,
+ sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
+ bypassOverlay: async (tabId, enabled) => {
+ try {
+ await chrome.tabs.sendMessage(tabId, {
+ type: OVERLAY_AUTOMATION_BYPASS,
+ enabled,
+ });
+ } catch {
+ // Content script may be unavailable on restricted pages.
+ }
+ },
+ ...(this.cdp ? { cdp: this.cdp } : {}),
+ signal,
+ });
+ case "tool.record_stop":
+ return handleRecordStop(this.sessions, req.params as RecordStopParams, {
+ tabsApi: chromeTabsApi,
+ sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
+ bypassOverlay: async (tabId, enabled) => {
+ try {
+ await chrome.tabs.sendMessage(tabId, {
+ type: OVERLAY_AUTOMATION_BYPASS,
+ enabled,
+ });
+ } catch {
+ // Content script may be unavailable on restricted pages.
+ }
+ },
+ signal,
+ });
+ case "tool.record_await":
+ return handleRecordAwait(this.sessions, req.params as RecordAwaitParams, {
+ tabsApi: chromeTabsApi,
+ sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
+ signal,
+ });
default:
return {
code: "unknown_method",
diff --git a/apps/extension/src/tools/record.ts b/apps/extension/src/tools/record.ts
new file mode 100644
index 0000000..4f3dde3
--- /dev/null
+++ b/apps/extension/src/tools/record.ts
@@ -0,0 +1,828 @@
+// `tool.record_start` / `tool.record_stop` / `tool.record_await` — capture
+// user actions in the Agent Window via the content script and return a
+// semantic (LLM textbook) trace.
+
+import {
+ isRecordFinishMessage,
+ isRecordQueryMessage,
+ isRecordStepMessage,
+ RECORD_CANCEL,
+ RECORD_START,
+ RECORD_STEP,
+ RECORD_STOP,
+ type RecordCancelMessage,
+ type RecordFinishMessage,
+ type RecordQueryResponse,
+ type RecordStartAck,
+ type RecordStartMessage,
+ type RecordStopMessage,
+} from "@/lib/record-bridge";
+import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording-step-buffer";
+import { reduceTraceSteps, resolveTraceStartUrl } from "@/lib/trace-reducer";
+import type { SessionManager } from "@/session-manager/manager";
+import type {
+ DraftTraceStep,
+ RecordAwaitParams,
+ RecordAwaitResult,
+ RecordStartParams,
+ RecordStartResult,
+ RecordStopParams,
+ RecordStopResult,
+ RpcError,
+ Trace,
+} from "@/transport/types";
+import { handleNavigate } from "./navigation";
+import {
+ type CdpRunner,
+ type ChromeTabsApi,
+ chromeTabsApi,
+ isRpcError,
+ lookupSession,
+ resolveTargetTab,
+} from "./shared";
+
+interface ActiveRecording {
+ requestId: string;
+ tabId: number;
+ agentWindowId: number;
+ startUrl?: string;
+ purpose?: string;
+ steps: DraftTraceStep[];
+ startedAt: string;
+ finishPromise: Promise;
+ resolveFinish: (trace: Trace) => void;
+ rejectFinish: (err: Error) => void;
+ settled: boolean;
+ finishing: boolean;
+ currentUrl?: string;
+ pendingNavigation: boolean;
+ pendingNavigationDeadline?: number;
+}
+
+const recordings = new Map();
+
+const RECORD_START_RETRIES = 3;
+const RECORD_START_RETRY_DELAY_MS = 500;
+const RECORD_REARM_DEBOUNCE_MS = 150;
+const RECORD_REARM_MAX_ATTEMPTS = 12;
+const RECORD_REARM_RETRY_DELAY_MS = 400;
+
+const rearmTimers = new Map>();
+
+function makeRequestId(tabId: number): string {
+ return `rec-${tabId}-${Date.now().toString(36)}`;
+}
+
+function sleep(ms: number): Promise {
+ return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+/** Injectable http(s) landing page when `tool.record_start` omits `url`. */
+export const RECORD_DEFAULT_START_URL = "https://example.com/";
+
+/** Pages where MV3 content scripts cannot attach (Agent Window boots here). */
+function isContentScriptRestrictedUrl(url: string | undefined): boolean {
+ if (!url) return true;
+ const lower = url.toLowerCase();
+ return (
+ lower === "about:blank" ||
+ lower.startsWith("about:") ||
+ lower.startsWith("chrome://") ||
+ lower.startsWith("chrome-extension://") ||
+ lower.startsWith("edge://") ||
+ lower.startsWith("devtools://") ||
+ lower.startsWith("devtools:") ||
+ lower.startsWith("https://chrome.google.com/webstore")
+ );
+}
+
+async function waitForTabReady(
+ tabId: number,
+ tabsApi: ChromeTabsApi,
+ timeoutMs = 10_000,
+): Promise {
+ try {
+ const tab = await tabsApi.get(tabId);
+ if (tab.status === "complete") return;
+ } catch {
+ return;
+ }
+
+ await new Promise((resolve, reject) => {
+ const timer = setTimeout(() => {
+ chrome.tabs.onUpdated.removeListener(listener);
+ reject(new Error("tab load timeout"));
+ }, timeoutMs);
+ const listener = (updatedTabId: number, info: chrome.tabs.TabChangeInfo) => {
+ if (updatedTabId !== tabId || info.status !== "complete") return;
+ clearTimeout(timer);
+ chrome.tabs.onUpdated.removeListener(listener);
+ resolve();
+ };
+ chrome.tabs.onUpdated.addListener(listener);
+ });
+}
+
+function isRecordStartAck(response: unknown): response is RecordStartAck {
+ return (
+ typeof response === "object" &&
+ response !== null &&
+ "ok" in response &&
+ (response as RecordStartAck).ok === true
+ );
+}
+
+async function sendRecordStartWithAck(
+ tabId: number,
+ msg: RecordStartMessage,
+ sendToTab: RecordDeps["sendToTab"],
+): Promise {
+ let lastError: unknown;
+ for (let attempt = 0; attempt < RECORD_START_RETRIES; attempt += 1) {
+ try {
+ const response = await sendToTab(tabId, msg);
+ if (isRecordStartAck(response)) return;
+ lastError = new Error("content script did not ack RECORD_START");
+ } catch (err) {
+ lastError = err;
+ }
+ if (attempt + 1 < RECORD_START_RETRIES) {
+ await sleep(RECORD_START_RETRY_DELAY_MS);
+ }
+ }
+ throw lastError ?? new Error("failed to start recording in content script");
+}
+
+function buildTrace(recording: ActiveRecording): Trace {
+ const { pages, steps } = reduceTraceSteps(recording.steps, recording.startUrl);
+ const startUrl = resolveTraceStartUrl(recording.steps, recording.startUrl, pages);
+ return {
+ ...(recording.purpose ? { purpose: recording.purpose } : {}),
+ recorded_at: new Date().toISOString(),
+ started_at: recording.startedAt,
+ entry: { start_url: startUrl },
+ pages,
+ steps,
+ };
+}
+
+export interface RecordDeps {
+ tabsApi: ChromeTabsApi;
+ sendToTab(
+ tabId: number,
+ msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
+ ): Promise;
+ bypassOverlay?: (tabId: number, enabled: boolean) => Promise;
+ cdp?: CdpRunner;
+ signal?: AbortSignal;
+}
+
+let defaultDeps: RecordDeps | null = null;
+function getDefaultDeps(): RecordDeps {
+ if (!defaultDeps) {
+ defaultDeps = {
+ tabsApi: chromeTabsApi,
+ sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
+ };
+ }
+ return defaultDeps;
+}
+
+/** Disposer for lazily attached tab / webNavigation observers. */
+let detachBrowserObservation: (() => void) | null = null;
+
+type AttachObservation = (deps: RecordDeps) => () => void;
+
+// Deferred wrappers so we do not capture attach* before their declarations.
+let attachTabObservation: AttachObservation = (deps) => attachRecordTabListener(deps);
+let attachNavObservation: AttachObservation = (deps) => attachRecordNavigationListener(deps);
+
+/** Test seam: swap real chrome listeners for fakes. */
+export function setBrowserObservationAttachForTests(
+ tab: AttachObservation | null,
+ nav: AttachObservation | null,
+): void {
+ attachTabObservation = tab ?? ((deps) => attachRecordTabListener(deps));
+ attachNavObservation = nav ?? ((deps) => attachRecordNavigationListener(deps));
+}
+
+export function isBrowserObservationAttachedForTests(): boolean {
+ return detachBrowserObservation !== null;
+}
+
+export function resetBrowserObservationForTests(): void {
+ detachBrowserObservation?.();
+ detachBrowserObservation = null;
+ recordings.clear();
+ attachTabObservation = (deps) => attachRecordTabListener(deps);
+ attachNavObservation = (deps) => attachRecordNavigationListener(deps);
+}
+
+/**
+ * Attach tab/webNavigation listeners while any recording is active.
+ * Must run before navigate-on-start so rearm observes the destination load.
+ */
+export function ensureBrowserObservationListeners(deps: RecordDeps = getDefaultDeps()): void {
+ if (detachBrowserObservation) return;
+ const detachTab = attachTabObservation(deps);
+ const detachNav = attachNavObservation(deps);
+ detachBrowserObservation = () => {
+ detachTab();
+ detachNav();
+ };
+}
+
+/** Detach when the recordings map is empty. */
+export function releaseBrowserObservationListenersIfIdle(): void {
+ if (recordings.size > 0) return;
+ if (!detachBrowserObservation) return;
+ detachBrowserObservation();
+ detachBrowserObservation = null;
+}
+
+export function attachRecordStepListener(): () => void {
+ const listener = (
+ message: unknown,
+ _sender: chrome.runtime.MessageSender,
+ _sendResponse: () => void,
+ ) => {
+ if (!isRecordStepMessage(message)) return false;
+ for (const recording of recordings.values()) {
+ if (recording.requestId !== message.requestId) continue;
+ appendRecordedPayload(recording, message.step);
+ return false;
+ }
+ return false;
+ };
+ chrome.runtime.onMessage.addListener(listener);
+ return () => chrome.runtime.onMessage.removeListener(listener);
+}
+
+export function attachRecordFinishListener(deps: RecordDeps = getDefaultDeps()): () => void {
+ const listener = (
+ message: unknown,
+ sender: chrome.runtime.MessageSender,
+ _sendResponse: () => void,
+ ) => {
+ if (!isRecordFinishMessage(message)) return false;
+ const tabId = sender.tab?.id;
+ if (tabId === undefined) return false;
+ void finishRecordingByRequest(message.requestId, tabId, deps);
+ return false;
+ };
+ chrome.runtime.onMessage.addListener(listener);
+ return () => chrome.runtime.onMessage.removeListener(listener);
+}
+
+function findRecordingByTabId(tabId: number): ActiveRecording | null {
+ for (const recording of recordings.values()) {
+ if (recording.tabId === tabId && !recording.settled) return recording;
+ }
+ return null;
+}
+
+async function findRecordingForTab(
+ tabId: number,
+ deps: RecordDeps,
+): Promise {
+ const direct = findRecordingByTabId(tabId);
+ if (direct) return direct;
+
+ try {
+ const tab = await deps.tabsApi.get(tabId);
+ const windowId = tab.windowId;
+ if (typeof windowId !== "number") return null;
+ for (const recording of recordings.values()) {
+ if (!recording.settled && recording.agentWindowId === windowId) return recording;
+ }
+ } catch {
+ return null;
+ }
+ return null;
+}
+
+function clearRearmTimer(tabId: number): void {
+ const timer = rearmTimers.get(tabId);
+ if (timer) {
+ clearTimeout(timer);
+ rearmTimers.delete(tabId);
+ }
+}
+
+async function clearRearmTimersForRecording(
+ recording: ActiveRecording,
+ deps: RecordDeps,
+): Promise {
+ clearRearmTimer(recording.tabId);
+ try {
+ const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
+ for (const tab of tabs) {
+ if (typeof tab.id === "number") clearRearmTimer(tab.id);
+ }
+ } catch {
+ // Best-effort cleanup.
+ }
+}
+
+async function stopRecordingOnAllAgentTabs(
+ recording: ActiveRecording,
+ deps: RecordDeps,
+): Promise {
+ const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
+ let tabIds = [recording.tabId];
+ try {
+ const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
+ tabIds = [
+ ...new Set([
+ recording.tabId,
+ ...tabs.flatMap((tab) => (typeof tab.id === "number" ? [tab.id] : [])),
+ ]),
+ ];
+ } catch {
+ // Fall back to the current recording tab.
+ }
+
+ for (const tabId of tabIds) {
+ try {
+ const response = await deps.sendToTab(tabId, stopMsg);
+ if (tabId === recording.tabId && !isRecordStartAck(response)) {
+ throw new Error("content script did not confirm recorded steps");
+ }
+ } catch {
+ if (tabId === recording.tabId) {
+ throw new Error("failed to flush recorded steps");
+ }
+ }
+ if (deps.bypassOverlay) {
+ try {
+ await deps.bypassOverlay(tabId, false);
+ } catch {
+ // Best-effort cleanup.
+ }
+ }
+ }
+}
+
+async function rearmRecording(
+ recording: ActiveRecording,
+ targetTabId: number,
+ deps: RecordDeps,
+): Promise {
+ // Do NOT toggle automation-bypass here: each retry used to increment the
+ // content-script counter, and a single stop decrement left the ControlOverlay
+ // stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
+ // already hides the control chrome while activeRecord is set.
+ for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
+ const startMsg: RecordStartMessage = { type: RECORD_START, requestId: recording.requestId };
+ try {
+ await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
+ recording.tabId = targetTabId;
+ return true;
+ } catch {
+ if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
+ await sleep(RECORD_REARM_RETRY_DELAY_MS);
+ }
+ }
+ }
+ return false;
+}
+
+function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
+ const existing = rearmTimers.get(tabId);
+ if (existing) clearTimeout(existing);
+ rearmTimers.set(
+ tabId,
+ setTimeout(() => {
+ rearmTimers.delete(tabId);
+ void (async () => {
+ const current = await findRecordingForTab(tabId, deps);
+ if (current) await rearmRecording(current, tabId, deps);
+ })();
+ }, RECORD_REARM_DEBOUNCE_MS),
+ );
+}
+
+export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): () => void {
+ const onCreated = (tab: chrome.tabs.Tab) => {
+ const tabId = tab.id;
+ const windowId = tab.windowId;
+ if (tabId === undefined || windowId === undefined) return;
+ for (const recording of recordings.values()) {
+ if (recording.settled || recording.agentWindowId !== windowId) continue;
+ scheduleRearmForTab(tabId, deps);
+ return;
+ }
+ };
+
+ const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => {
+ scheduleRearmForTab(activeInfo.tabId, deps);
+ };
+
+ chrome.tabs.onCreated.addListener(onCreated);
+ chrome.tabs.onActivated.addListener(onActivated);
+ return () => {
+ chrome.tabs.onCreated.removeListener(onCreated);
+ chrome.tabs.onActivated.removeListener(onActivated);
+ };
+}
+
+export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps()): () => void {
+ const observeMainFrame = (tabId: number, url?: string, causedByAction?: boolean) => {
+ void (async () => {
+ const recording = await findRecordingForTab(tabId, deps);
+ if (recording && url) {
+ observeRecordedNavigation(recording, url, causedByAction);
+ }
+ })();
+ };
+ const onMainFrameComplete = (tabId: number, url?: string) => {
+ void (async () => {
+ observeMainFrame(tabId, url);
+ scheduleRearmForTab(tabId, deps);
+ })();
+ };
+
+ if (chrome.webNavigation?.onCompleted) {
+ const completedListener = (
+ details: chrome.webNavigation.WebNavigationFramedCallbackDetails,
+ ) => {
+ if (details.frameId !== 0) return;
+ onMainFrameComplete(details.tabId, details.url);
+ };
+ const committedListener = (
+ details: chrome.webNavigation.WebNavigationTransitionCallbackDetails,
+ ) => {
+ if (details.frameId !== 0) return;
+ const causedByAction =
+ details.transitionType === "link" || details.transitionType === "form_submit"
+ ? true
+ : details.transitionType === "typed" ||
+ details.transitionType === "auto_bookmark" ||
+ details.transitionType === "generated" ||
+ details.transitionType === "keyword" ||
+ details.transitionType === "keyword_generated" ||
+ details.transitionType === "reload"
+ ? false
+ : undefined;
+ observeMainFrame(details.tabId, details.url, causedByAction);
+ };
+ chrome.webNavigation.onCompleted.addListener(completedListener);
+ chrome.webNavigation.onCommitted?.addListener(committedListener);
+ return () => {
+ chrome.webNavigation.onCompleted.removeListener(completedListener);
+ chrome.webNavigation.onCommitted?.removeListener(committedListener);
+ };
+ }
+
+ const listener = (tabId: number, info: chrome.tabs.TabChangeInfo) => {
+ if (info.status !== "complete") return;
+ onMainFrameComplete(tabId, info.url);
+ };
+ chrome.tabs.onUpdated.addListener(listener);
+ return () => chrome.tabs.onUpdated.removeListener(listener);
+}
+
+export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()): () => void {
+ const listener = (
+ message: unknown,
+ sender: chrome.runtime.MessageSender,
+ sendResponse: (response: RecordQueryResponse) => void,
+ ) => {
+ if (!isRecordQueryMessage(message)) return false;
+ const tabId = sender.tab?.id;
+ if (tabId === undefined) {
+ sendResponse({ active: false });
+ return false;
+ }
+ void (async () => {
+ const recording = await findRecordingForTab(tabId, deps);
+ if (!recording) {
+ sendResponse({ active: false });
+ return;
+ }
+ await rearmRecording(recording, tabId, deps);
+ sendResponse({ active: true, requestId: recording.requestId });
+ })();
+ return true;
+ };
+ chrome.runtime.onMessage.addListener(listener);
+ return () => chrome.runtime.onMessage.removeListener(listener);
+}
+
+async function finishRecordingByRequest(
+ requestId: string,
+ tabId: number,
+ deps: RecordDeps,
+): Promise {
+ for (const [sessionId, recording] of recordings) {
+ if (recording.requestId !== requestId || recording.settled) continue;
+ const match = await findRecordingForTab(tabId, deps);
+ if (match !== recording) continue;
+ await finishRecording(sessionId, deps);
+ return;
+ }
+}
+
+async function finishRecording(sessionId: string, deps: RecordDeps): Promise {
+ const recording = recordings.get(sessionId);
+ if (!recording || recording.settled || recording.finishing) return null;
+ recording.finishing = true;
+
+ await clearRearmTimersForRecording(recording, deps);
+ try {
+ await stopRecordingOnAllAgentTabs(recording, deps);
+ } catch {
+ recording.finishing = false;
+ return null;
+ }
+
+ recording.settled = true;
+ recordings.delete(sessionId);
+ releaseBrowserObservationListenersIfIdle();
+ const trace = buildTrace(recording);
+ recording.resolveFinish(trace);
+ return trace;
+}
+
+export async function handleRecordStart(
+ manager: SessionManager,
+ params: RecordStartParams,
+ deps: RecordDeps = getDefaultDeps(),
+): Promise {
+ const ctxOrErr = lookupSession(manager, params, "record_start");
+ if (isRpcError(ctxOrErr)) return ctxOrErr;
+ const ctx = ctxOrErr;
+ if (recordings.has(params.session_id)) {
+ return {
+ code: "protocol_error",
+ message: `session ${params.session_id} is already recording`,
+ };
+ }
+ const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
+ if (isRpcError(target)) return target;
+
+ // Register the recording *before* navigate so content-script syncAgentOverlay
+ // on the destination page can RECORD_QUERY → rearm → show RecordOverlay
+ // instead of flashing ControlOverlay ("Agent 正在控制").
+ const requestId = makeRequestId(target.tabId);
+ let resolveFinish!: (trace: Trace) => void;
+ let rejectFinish!: (err: Error) => void;
+ const finishPromise = new Promise((resolve, reject) => {
+ resolveFinish = resolve;
+ rejectFinish = reject;
+ });
+ const navigateUrl = params.url ?? RECORD_DEFAULT_START_URL;
+ recordings.set(params.session_id, {
+ requestId,
+ tabId: target.tabId,
+ agentWindowId: ctx.agentWindowId,
+ startUrl: navigateUrl,
+ ...(params.purpose ? { purpose: params.purpose } : {}),
+ steps: [],
+ startedAt: new Date().toISOString(),
+ finishPromise,
+ resolveFinish,
+ rejectFinish,
+ settled: false,
+ finishing: false,
+ currentUrl: navigateUrl,
+ pendingNavigation: false,
+ pendingNavigationDeadline: undefined,
+ });
+ // Observe navigations for the whole recording lifetime; attach before
+ // optional navigate so the destination load can rearm capture.
+ ensureBrowserObservationListeners(deps);
+
+ const abortPending = async (notifyContent: boolean) => {
+ recordings.delete(params.session_id);
+ releaseBrowserObservationListenersIfIdle();
+ if (notifyContent) {
+ try {
+ await deps.sendToTab(target.tabId, {
+ type: RECORD_CANCEL,
+ requestId,
+ });
+ } catch {
+ // Content may never have received RECORD_START.
+ }
+ }
+ if (deps.bypassOverlay) {
+ try {
+ await deps.bypassOverlay(target.tabId, false);
+ } catch {
+ // Ignore cleanup errors.
+ }
+ }
+ };
+
+ const cancelledError = (): RpcError => ({
+ code: "cancelled",
+ message: "record_start aborted",
+ });
+
+ /** Dispatcher may race-cancel while we still hold a provisional recording. */
+ const abortIfCancelled = async (notifyContent: boolean): Promise => {
+ if (!deps.signal?.aborted) return null;
+ await abortPending(notifyContent);
+ return cancelledError();
+ };
+
+ {
+ const cancelled = await abortIfCancelled(false);
+ if (cancelled) return cancelled;
+ }
+
+ if (deps.cdp) {
+ const nav = await handleNavigate(
+ manager,
+ {
+ session_id: params.session_id,
+ url: navigateUrl,
+ tab_id: target.tabId,
+ },
+ { cdp: deps.cdp, tabsApi: deps.tabsApi, signal: deps.signal },
+ );
+ if (isRpcError(nav)) {
+ await abortPending(false);
+ return nav;
+ }
+ {
+ const cancelled = await abortIfCancelled(false);
+ if (cancelled) return cancelled;
+ }
+ try {
+ await waitForTabReady(target.tabId, deps.tabsApi);
+ } catch {
+ // Proceed with retries even if the tab never reports complete.
+ }
+ {
+ const cancelled = await abortIfCancelled(false);
+ if (cancelled) return cancelled;
+ }
+ }
+
+ let startUrl: string | undefined;
+ try {
+ const tab = await deps.tabsApi.get(target.tabId);
+ startUrl = tab.url;
+ } catch {
+ startUrl = navigateUrl;
+ }
+
+ const active = recordings.get(params.session_id);
+ if (!active) {
+ // Cleared by a concurrent abort / session teardown.
+ return cancelledError();
+ }
+ active.startUrl = startUrl;
+ active.currentUrl = startUrl;
+
+ if (isContentScriptRestrictedUrl(startUrl)) {
+ await abortPending(false);
+ return {
+ code: "invalid_params",
+ message: params.url
+ ? `cannot record on restricted URL (${startUrl}); use an http(s) page`
+ : `cannot record on restricted URL (${startUrl}); default start page must be injectable http(s)`,
+ };
+ }
+
+ {
+ const cancelled = await abortIfCancelled(false);
+ if (cancelled) return cancelled;
+ }
+
+ if (deps.bypassOverlay) {
+ try {
+ // Single ref for the initial race before RecordOverlay mounts; rearm must
+ // not stack additional refs (see rearmRecording). Cleared on stop.
+ await deps.bypassOverlay(target.tabId, true);
+ } catch {
+ // Best-effort; activeRecord also hides the control overlay.
+ }
+ }
+
+ {
+ const cancelled = await abortIfCancelled(true);
+ if (cancelled) return cancelled;
+ }
+
+ const startMsg: RecordStartMessage = { type: RECORD_START, requestId };
+
+ try {
+ await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
+ } catch {
+ await abortPending(true);
+ return {
+ code: "protocol_error",
+ message:
+ "failed to start recording in content script — reload the BrowserSkill extension, then retry",
+ };
+ }
+
+ {
+ const cancelled = await abortIfCancelled(true);
+ if (cancelled) return cancelled;
+ }
+
+ return { tab_id: target.tabId, recording: true };
+}
+
+export async function handleRecordStop(
+ manager: SessionManager,
+ params: RecordStopParams,
+ deps: RecordDeps = getDefaultDeps(),
+): Promise {
+ const ctxOrErr = lookupSession(manager, params, "record_stop");
+ if (isRpcError(ctxOrErr)) return ctxOrErr;
+
+ const recording = recordings.get(params.session_id);
+ if (!recording) {
+ return {
+ code: "not_found",
+ message: `no active recording for session ${params.session_id}`,
+ };
+ }
+
+ const trace = await finishRecording(params.session_id, deps);
+ if (!trace) {
+ return {
+ code: "protocol_error",
+ message: `failed to flush recorded steps for session ${params.session_id}; the recording is still active — retry \`bsk record stop\``,
+ };
+ }
+ return { trace };
+}
+
+export async function handleRecordAwait(
+ manager: SessionManager,
+ params: RecordAwaitParams,
+ deps: RecordDeps = getDefaultDeps(),
+): Promise {
+ const ctxOrErr = lookupSession(manager, params, "record_await");
+ if (isRpcError(ctxOrErr)) return ctxOrErr;
+
+ const recording = recordings.get(params.session_id);
+ if (!recording) {
+ return {
+ code: "not_found",
+ message: `no active recording for session ${params.session_id}`,
+ };
+ }
+
+ if (deps.signal?.aborted) {
+ return { code: "cancelled", message: "record_await aborted" };
+ }
+
+ const outcome = await new Promise<{ trace: Trace } | { error: RpcError }>((resolve) => {
+ let settled = false;
+ const finish = (result: { trace: Trace } | { error: RpcError }) => {
+ if (settled) return;
+ settled = true;
+ if (timer) clearTimeout(timer);
+ deps.signal?.removeEventListener("abort", onAbort);
+ resolve(result);
+ };
+ const onAbort = () => finish({ error: { code: "cancelled", message: "record_await aborted" } });
+ const timer =
+ params.timeout_ms === undefined
+ ? undefined
+ : setTimeout(
+ () =>
+ finish({
+ error: {
+ code: "timeout",
+ message: `record_await timed out after ${params.timeout_ms}ms`,
+ },
+ }),
+ params.timeout_ms,
+ );
+ deps.signal?.addEventListener("abort", onAbort, { once: true });
+ void recording.finishPromise.then(
+ (trace) => finish({ trace }),
+ () =>
+ finish({
+ error: { code: "cancelled", message: "recording was cleared" },
+ }),
+ );
+ });
+ return "trace" in outcome ? { trace: outcome.trace } : outcome.error;
+}
+
+export function clearRecordingForSession(sessionId: string): void {
+ const recording = recordings.get(sessionId);
+ if (!recording) {
+ recordings.delete(sessionId);
+ releaseBrowserObservationListenersIfIdle();
+ return;
+ }
+ void clearRearmTimersForRecording(recording, getDefaultDeps());
+ if (!recording.settled) {
+ recording.settled = true;
+ recording.rejectFinish(new Error("recording cleared"));
+ }
+ recordings.delete(sessionId);
+ releaseBrowserObservationListenersIfIdle();
+}
+
+export type { RecordFinishMessage };
diff --git a/apps/extension/src/tools/session.ts b/apps/extension/src/tools/session.ts
index 6c14d7e..c83a2cd 100644
--- a/apps/extension/src/tools/session.ts
+++ b/apps/extension/src/tools/session.ts
@@ -1,5 +1,6 @@
import type { SessionManager } from "@/session-manager/manager";
import type { RpcError } from "@/transport/types";
+import { clearRecordingForSession } from "./record";
import { returnBorrowedTab, type TabManagementDeps } from "./tabs";
export interface SessionStartParams {
@@ -156,6 +157,8 @@ export async function handleSessionStop(
// Step 2: clear the per-session RefStore (review M6/M7 parity).
ctx.refStore.clear();
+ clearRecordingForSession(params.session_id);
+
// Step 3: detach CDP sessions this session opened (no-op if none).
await deps.cdp?.detachSession(params.session_id);
diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts
index 2727491..bcff0e3 100644
--- a/apps/extension/src/transport/types.ts
+++ b/apps/extension/src/transport/types.ts
@@ -537,3 +537,135 @@ export interface RequestHelpResult {
tab_id: number;
resolved_targets?: ResolvedTarget[];
}
+
+// --------------------------------------------------------------------------
+// Semantic record payloads — mirror bsk-protocol record.rs (Trace v2)
+// --------------------------------------------------------------------------
+
+export interface TargetDescriptor {
+ role?: string;
+ name?: string;
+ tag: string;
+ name_attr?: string;
+ placeholder?: string;
+ nearby_label?: string;
+}
+
+export interface TraceEntry {
+ start_url: string;
+}
+
+export interface PageRef {
+ id: string;
+ url: string;
+ title?: string;
+}
+
+export interface SelectedOption {
+ value: string;
+ label?: string;
+}
+
+export interface StepEffect {
+ navigated_to: string;
+}
+
+export interface StepCommon {
+ id: number;
+ page: string;
+ effect?: StepEffect;
+}
+
+/** Capture/buffer draft before v2 reduction. */
+export type DraftTraceStep =
+ | {
+ op: "click";
+ target: TargetDescriptor;
+ navigated_to?: string;
+ page_url?: string;
+ }
+ | {
+ op: "fill";
+ target: TargetDescriptor;
+ value: string;
+ redacted?: boolean;
+ page_url?: string;
+ }
+ | {
+ op: "press";
+ key: string;
+ target?: TargetDescriptor;
+ modifiers?: KeyModifier[];
+ navigated_to?: string;
+ page_url?: string;
+ }
+ | {
+ op: "select";
+ target: TargetDescriptor;
+ values: string[];
+ labels?: string[];
+ navigated_to?: string;
+ page_url?: string;
+ }
+ | {
+ op: "navigate";
+ url: string;
+ page_url?: string;
+ };
+
+/** Exported record-only step (trace v2). */
+export type Step =
+ | ({ op: "navigate" } & StepCommon & { to: string })
+ | ({ op: "click" } & StepCommon & { target: TargetDescriptor })
+ | ({ op: "fill" } & StepCommon & {
+ target: TargetDescriptor;
+ value: string;
+ redacted?: boolean;
+ })
+ | ({ op: "select" } & StepCommon & {
+ target: TargetDescriptor;
+ selection: SelectedOption[];
+ })
+ | ({ op: "press" } & StepCommon & {
+ key: string;
+ modifiers?: KeyModifier[];
+ target?: TargetDescriptor;
+ });
+
+export interface Trace {
+ recorded_at: string;
+ started_at?: string;
+ purpose?: string;
+ entry: TraceEntry;
+ pages: PageRef[];
+ steps: Step[];
+}
+
+export interface RecordStartParams {
+ session_id: string;
+ tab_id?: number;
+ url?: string;
+ purpose?: string;
+}
+
+export interface RecordStartResult {
+ tab_id: number;
+ recording: boolean;
+}
+
+export interface RecordStopParams {
+ session_id: string;
+}
+
+export interface RecordStopResult {
+ trace: Trace;
+}
+
+export interface RecordAwaitParams {
+ session_id: string;
+ timeout_ms?: number;
+}
+
+export interface RecordAwaitResult {
+ trace: Trace;
+}
diff --git a/apps/extension/wxt.config.ts b/apps/extension/wxt.config.ts
index bca4399..34c72b6 100644
--- a/apps/extension/wxt.config.ts
+++ b/apps/extension/wxt.config.ts
@@ -29,7 +29,16 @@ export default defineConfig({
// service-worker idle timer, which the 20s heartbeat relies on to
// keep the daemon link alive.
minimum_chrome_version: "116",
- permissions: ["alarms", "debugger", "idle", "notifications", "tabs", "storage", "windows"],
+ permissions: [
+ "alarms",
+ "debugger",
+ "idle",
+ "notifications",
+ "tabs",
+ "storage",
+ "webNavigation",
+ "windows",
+ ],
host_permissions: [""],
icons: {
16: "icon/logo.png",
diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md
index 5edf9ea..ab32ff7 100644
--- a/crates/bsk-cli/skill/SKILL.md
+++ b/crates/bsk-cli/skill/SKILL.md
@@ -46,6 +46,21 @@ Optional: `bsk session start --browser ` when multiple bro
Emergency cleanup: `bsk session stop --all` or the Agent Window overlay **Stop all**.
+## Stop when the goal is met
+
+Every task is a **bounded goal**, not open-ended browsing. The goal may come from the user's request, a recorded `trace.json`, or both.
+
+1. **Define success first** — one concrete, observable condition derived from the user's words, `purpose`, or the last meaningful step in a trace (e.g. "form submitted", "item added to cart", "playback started").
+2. **Take the shortest path** — snapshot → act → at most one check. Do not wander, re-try unrelated actions, or stack exploratory steps.
+3. **Stop as soon as success is reached** — run `bsk session stop ` immediately unless the user explicitly asked to keep the session open (e.g. "don't close yet", "keep browsing").
+4. **No post-success work** — once the goal is met, do not click, refresh, navigate, re-search, switch tabs, or "double-check" that it worked. Further verification is a new task.
+5. **When blocked, pause — do not brute-force** — if the page requires human input (login, captcha, OTP, payment confirmation) or an action fails twice with no progress, call `bsk request-help` instead of retrying blindly. See **Ask the human for help** below.
+6. **When unsure** — at most one extra `bsk snapshot`. If success looks met, stop. If not, ask the user; do not keep clicking.
+
+**With a trace:** replay steps in order using `target` role/name/tag and raw `value`/`selection` fields. After the last step (or when its `effect.navigated_to` / success hint is satisfied), apply rules 3–4 immediately. The trace guides execution; it does not extend control beyond the goal.
+
+**Without a trace:** the user's request *is* the success condition. Satisfying it ends the task — same stop rules apply.
+
## Core interaction loop
Write operations only affect tabs in the **Agent Window** (or tabs you **borrowed** into it).
@@ -191,6 +206,23 @@ acts. The result `outcome` is one of:
`note` carries any text the user typed back. `resolved_targets` reports
which refs/selectors matched a live element.
+### Recording — `bsk record`
+
+Capture the user's own actions in the Agent Window to a `trace.json`, for later LLM-driven automation:
+
+```bash
+bsk record start --browser [--url https://…] [--purpose "publish a wiki doc"] [--output trace.json]
+# `--url` is optional; default https://example.com/ when omitted (must be http(s)).
+# Blocks until the user clicks Finish in the recording panel, then writes ./trace.json and closes the window.
+
+bsk record stop [--output trace.json] # terminal fallback if the browser panel is unavailable
+```
+
+- The trace is a **record-only action log** (a `pages[]` dictionary + `navigate`/`click`/`fill`/`select`/`press` steps with `target` descriptors). It records *what the user did*; deciding which inputs are variable is left to the executing agent.
+- `--purpose` is optional context metadata; it does **not** change what gets captured.
+- There is **no** `bsk replay` — to redo a flow, read the trace and reuse the existing `session` / `snapshot` / `@eN` / `click` / `fill` tools. Follow **Stop when the goal is met**.
+- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
+
## Error handling
### Exit codes (`echo $?` after `bsk …`)
@@ -221,8 +253,9 @@ Always **`bsk session stop `** in a `finally`-style path so the Agent Window
1. **No token theft** — do not `bsk evaluate` on sensitive sites to read `localStorage`, cookies, or auth headers for exfiltration.
2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks.
3. **No skip stop** — always `bsk session stop `; never assume idle timeout will clean up.
-4. **No observe escalation before snapshot** — use `bsk snapshot` first; only use `bsk get-html` or `bsk screenshot` when the snapshot is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot ref — never skip snapshot just to grab a visual.
-5. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
+4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open.
+5. **No observe escalation before snapshot** — use `bsk snapshot` first; only use `bsk get-html` or `bsk screenshot` when the snapshot is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot ref — never skip snapshot just to grab a visual.
+6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
---
diff --git a/crates/bsk-cli/src/cli/mod.rs b/crates/bsk-cli/src/cli/mod.rs
index cada994..9e8e48a 100644
--- a/crates/bsk-cli/src/cli/mod.rs
+++ b/crates/bsk-cli/src/cli/mod.rs
@@ -19,6 +19,8 @@ pub mod interaction;
pub mod logs;
pub mod navigate;
pub mod network;
+pub mod record;
+pub mod record_state;
pub mod render_error;
pub mod screenshot;
pub mod session;
@@ -39,6 +41,7 @@ use crate::cli::install_skill::InstallSkillArgs;
use crate::cli::interaction::{ClickArgs, FillArgs, PressArgs, SelectArgs};
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
use crate::cli::network::NetworkArgs;
+use crate::cli::record::RecordCmd;
use crate::cli::screenshot::ScreenshotArgs;
use crate::cli::session::SessionCmd;
use crate::cli::snapshot::SnapshotArgs;
@@ -169,6 +172,9 @@ pub enum Command {
/// Ask the human to complete an in-page step (captcha / login / confirm).
#[command(name = "request-help")]
RequestHelp(RequestHelpArgs),
+
+ /// Record user actions into a semantic `trace.json` textbook for LLMs.
+ Record(RecordCmd),
}
#[derive(Debug, Clone, Args, Default)]
diff --git a/crates/bsk-cli/src/cli/record.rs b/crates/bsk-cli/src/cli/record.rs
new file mode 100644
index 0000000..dce12c1
--- /dev/null
+++ b/crates/bsk-cli/src/cli/record.rs
@@ -0,0 +1,261 @@
+//! `bsk record start|stop` — capture user actions in the Agent Window.
+
+use std::fs;
+use std::path::PathBuf;
+use std::time::Duration;
+
+use anyhow::Context;
+use bsk_protocol::Method;
+use bsk_protocol::tools::{
+ RecordAwaitParams, RecordAwaitResult, RecordStartParams, RecordStartResult, RecordStopParams,
+ RecordStopResult, Trace,
+};
+use clap::{Args, Subcommand};
+
+use crate::cli::TOOL_IPC_TIMEOUT;
+use crate::cli::business_rpc;
+use crate::cli::ensure_daemon::ensure_daemon;
+use crate::cli::error::{CliError, Format};
+use crate::cli::record_state;
+use crate::cli::session::{start_session, stop_session};
+
+/// Max wait for the user to click 结束 in the browser (24 hours).
+const RECORD_AWAIT_TIMEOUT_MS: u32 = 86_400_000;
+
+/// `bsk record …` subcommand tree.
+#[derive(Debug, Clone, Args)]
+pub struct RecordCmd {
+ #[command(subcommand)]
+ pub sub: RecordSub,
+}
+
+#[derive(Debug, Clone, Subcommand)]
+pub enum RecordSub {
+ /// Open the Agent Window, record user actions, and block until finished in the browser.
+ Start(RecordStartArgs),
+ /// Stop recording from the terminal (fallback), write trace JSON, and close the window.
+ /// Works even while `record start` is blocked in `record_await` (daemon forwards
+ /// `tool.record_stop` without the per-session busy gate).
+ Stop(RecordStopArgs),
+}
+
+#[derive(Debug, Clone, Args)]
+pub struct RecordStartArgs {
+ /// Target browser instance id or label. Required when multiple browsers
+ /// are connected; omit when only one is online.
+ #[arg(long)]
+ pub browser: Option,
+
+ #[arg(long = "tab-id")]
+ pub tab_id: Option,
+
+ /// Navigate to this http(s) URL before recording. When omitted, defaults
+ /// to `https://example.com/`.
+ #[arg(long)]
+ pub url: Option,
+
+ /// Optional goal text stored on the exported trace for LLM context.
+ #[arg(long)]
+ pub purpose: Option,
+
+ /// Output path for the trace JSON (default `./trace.json`).
+ #[arg(long, default_value = "trace.json")]
+ pub output: PathBuf,
+}
+
+#[derive(Debug, Clone, Args)]
+pub struct RecordStopArgs {
+ /// Output path for the trace JSON (default `./trace.json`).
+ #[arg(long, default_value = "trace.json")]
+ pub output: PathBuf,
+}
+
+pub fn dispatch(cmd: RecordCmd, format: Format) -> Result<(), CliError> {
+ match cmd.sub {
+ RecordSub::Start(args) => dispatch_start(args, format),
+ RecordSub::Stop(args) => dispatch_stop(args, format),
+ }
+}
+
+fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError> {
+ if record_state::read().is_ok() {
+ return Err(CliError::Local(anyhow::anyhow!(
+ "a recording is already in progress; run `bsk record stop` first"
+ )));
+ }
+
+ let info = ensure_daemon().context("ensure daemon is running")?;
+ let session = start_session(info.sock_path.clone(), args.browser)?;
+
+ let start_params = RecordStartParams {
+ session_id: session.session_id.clone(),
+ tab_id: args.tab_id,
+ url: args.url,
+ purpose: args.purpose.clone(),
+ };
+ let start_result = business_rpc::call::(
+ info.sock_path.clone(),
+ "record-start",
+ Method::ToolRecordStart,
+ Some(start_params),
+ TOOL_IPC_TIMEOUT,
+ );
+
+ let start_result = match start_result {
+ Ok(result) => result,
+ Err(err) => {
+ let _ = stop_session(info.sock_path, &session.session_id);
+ return Err(err);
+ }
+ };
+
+ if let Err(err) = record_state::write(&session.session_id) {
+ let _ = stop_session(info.sock_path.clone(), &session.session_id);
+ return Err(CliError::Local(err));
+ }
+
+ if format == Format::Human {
+ println!(
+ "recording on tab={} — click 结束 in the browser when done",
+ start_result.tab_id
+ );
+ }
+
+ let await_params = RecordAwaitParams {
+ session_id: session.session_id.clone(),
+ timeout_ms: Some(RECORD_AWAIT_TIMEOUT_MS),
+ };
+ let await_result = business_rpc::call::(
+ info.sock_path.clone(),
+ "record-await",
+ Method::ToolRecordAwait,
+ Some(await_params),
+ record_await_ipc_timeout(RECORD_AWAIT_TIMEOUT_MS),
+ );
+
+ // Keep write/render inside a Result so `?` cannot skip session teardown.
+ let run_result: Result<(), CliError> = match await_result {
+ Ok(result) => (|| {
+ write_trace_file(&args.output, &result.trace)?;
+ render_finish(&result.trace, &args.output, format)
+ })(),
+ Err(err) => Err(err),
+ };
+
+ let session_stop_result = stop_session(info.sock_path, &session.session_id);
+ record_state::clear();
+
+ run_result?;
+ session_stop_result?;
+ Ok(())
+}
+
+fn dispatch_stop(args: RecordStopArgs, format: Format) -> Result<(), CliError> {
+ let state = record_state::read().map_err(CliError::Local)?;
+ let info = ensure_daemon().context("ensure daemon is running")?;
+ let session_id = state.session_id.clone();
+
+ let run_result: Result<(), CliError> = (|| {
+ let params = RecordStopParams {
+ session_id: session_id.clone(),
+ };
+ let result = business_rpc::call::(
+ info.sock_path.clone(),
+ "record-stop",
+ Method::ToolRecordStop,
+ Some(params),
+ TOOL_IPC_TIMEOUT,
+ )?;
+ write_trace_file(&args.output, &result.trace)?;
+ render_stop(&result, &args.output, format)
+ })();
+
+ let session_stop_result = stop_session(info.sock_path, &session_id);
+ record_state::clear();
+
+ run_result?;
+ session_stop_result?;
+ Ok(())
+}
+
+fn record_await_ipc_timeout(timeout_ms: u32) -> Duration {
+ Duration::from_millis(u64::from(timeout_ms))
+ .checked_add(Duration::from_secs(15))
+ .unwrap_or(Duration::from_secs(u64::from(timeout_ms / 1_000) + 15))
+}
+
+fn write_trace_file(output: &PathBuf, trace: &Trace) -> Result<(), CliError> {
+ let json = serde_json::to_string_pretty(trace)
+ .context("serialize trace JSON")
+ .map_err(CliError::Local)?;
+ if let Some(parent) = output.parent() {
+ if !parent.as_os_str().is_empty() {
+ fs::create_dir_all(parent)
+ .with_context(|| format!("create output directory {}", parent.display()))
+ .map_err(CliError::Local)?;
+ }
+ }
+ fs::write(output, format!("{json}\n"))
+ .with_context(|| format!("write trace to {}", output.display()))
+ .map_err(CliError::Local)?;
+ Ok(())
+}
+
+fn render_finish(trace: &Trace, output: &PathBuf, format: Format) -> Result<(), CliError> {
+ match format {
+ Format::Json => {
+ println!(
+ "{}",
+ serde_json::to_string(&serde_json::json!({
+ "output": output,
+ "trace": trace,
+ "window_closed": true,
+ }))
+ .map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
+ );
+ }
+ Format::Human => {
+ println!("saved {} steps to {}", trace.steps.len(), output.display());
+ }
+ }
+ Ok(())
+}
+
+fn render_stop(
+ result: &RecordStopResult,
+ output: &PathBuf,
+ format: Format,
+) -> Result<(), CliError> {
+ render_finish(&result.trace, output, format)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn default_output_is_trace_json() {
+ let args = RecordStopArgs {
+ output: PathBuf::from("trace.json"),
+ };
+ assert_eq!(args.output, PathBuf::from("trace.json"));
+ }
+
+ #[test]
+ fn start_args_default_output_is_trace_json() {
+ let args = RecordStartArgs {
+ browser: None,
+ tab_id: None,
+ url: None,
+ purpose: None,
+ output: PathBuf::from("trace.json"),
+ };
+ assert_eq!(args.output, PathBuf::from("trace.json"));
+ }
+
+ #[test]
+ fn record_await_ipc_timeout_covers_long_wait() {
+ let got = record_await_ipc_timeout(RECORD_AWAIT_TIMEOUT_MS);
+ assert!(got >= Duration::from_secs(86_400));
+ }
+}
diff --git a/crates/bsk-cli/src/cli/record_state.rs b/crates/bsk-cli/src/cli/record_state.rs
new file mode 100644
index 0000000..c56a686
--- /dev/null
+++ b/crates/bsk-cli/src/cli/record_state.rs
@@ -0,0 +1,117 @@
+//! Persist the session id between `bsk record start` and `bsk record stop`.
+
+use std::fs;
+use std::time::{SystemTime, UNIX_EPOCH};
+
+use anyhow::{Context, Result};
+use serde::{Deserialize, Serialize};
+
+use crate::daemon::paths;
+
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct RecordSessionState {
+ pub session_id: String,
+ pub started_at: String,
+}
+
+pub fn write(session_id: &str) -> Result<()> {
+ let path = paths::record_session_path()?;
+ paths::ensure_bsk_home()?;
+ if path.exists() {
+ return Err(anyhow::anyhow!(
+ "a recording is already in progress; run `bsk record stop` first"
+ ));
+ }
+ let state = RecordSessionState {
+ session_id: session_id.to_string(),
+ started_at: started_at_unix_ms(),
+ };
+ let json = serde_json::to_string_pretty(&state).context("serialize record session state")?;
+ fs::write(&path, format!("{json}\n")).with_context(|| format!("write {}", path.display()))?;
+ Ok(())
+}
+
+pub fn read() -> Result {
+ let path = paths::record_session_path()?;
+ if !path.exists() {
+ return Err(anyhow::anyhow!(
+ "no recording in progress; run `bsk record start` first"
+ ));
+ }
+ let raw = fs::read_to_string(&path).with_context(|| format!("read {}", path.display()))?;
+ serde_json::from_str(&raw).with_context(|| format!("parse {}", path.display()))
+}
+
+pub fn clear() {
+ if let Ok(path) = paths::record_session_path() {
+ let _ = fs::remove_file(path);
+ }
+}
+
+fn started_at_unix_ms() -> String {
+ let ms = SystemTime::now()
+ .duration_since(UNIX_EPOCH)
+ .unwrap_or_default()
+ .as_millis();
+ format!("{ms}")
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use std::sync::Mutex;
+
+ fn env_guard() -> &'static Mutex<()> {
+ static GUARD: Mutex<()> = Mutex::new(());
+ &GUARD
+ }
+
+ fn with_temp_home(f: F) {
+ let _lock = env_guard().lock().unwrap_or_else(|e| e.into_inner());
+ let tmp = tempfile::tempdir().unwrap();
+ unsafe {
+ std::env::set_var(crate::daemon::paths::BSK_HOME_ENV, tmp.path());
+ }
+ f();
+ clear();
+ unsafe {
+ std::env::remove_var(crate::daemon::paths::BSK_HOME_ENV);
+ }
+ }
+
+ #[test]
+ fn write_read_round_trips() {
+ with_temp_home(|| {
+ write("abcd").unwrap();
+ let state = read().unwrap();
+ assert_eq!(state.session_id, "abcd");
+ assert!(!state.started_at.is_empty());
+ });
+ }
+
+ #[test]
+ fn write_rejects_duplicate() {
+ with_temp_home(|| {
+ write("abcd").unwrap();
+ let err = write("efgh").unwrap_err();
+ assert!(err.to_string().contains("already in progress"));
+ });
+ }
+
+ #[test]
+ fn read_errors_when_missing() {
+ with_temp_home(|| {
+ let err = read().unwrap_err();
+ assert!(err.to_string().contains("no recording in progress"));
+ });
+ }
+
+ #[test]
+ fn clear_removes_state_file() {
+ with_temp_home(|| {
+ write("abcd").unwrap();
+ clear();
+ assert!(read().is_err());
+ });
+ }
+}
diff --git a/crates/bsk-cli/src/cli/session.rs b/crates/bsk-cli/src/cli/session.rs
index 20b6764..198f568 100644
--- a/crates/bsk-cli/src/cli/session.rs
+++ b/crates/bsk-cli/src/cli/session.rs
@@ -74,11 +74,11 @@ struct StartParams {
}
#[derive(Debug, Deserialize)]
-struct StartReply {
- session_id: String,
- browser_instance_id: String,
+pub struct StartReply {
+ pub session_id: String,
+ pub browser_instance_id: String,
#[serde(default)]
- agent_window_id: Option,
+ pub agent_window_id: Option,
}
#[derive(Debug, Serialize)]
@@ -89,21 +89,21 @@ struct StopParams {
}
#[derive(Debug, Deserialize)]
-struct StopReply {
- stopped: Vec,
+pub struct StopReply {
+ pub stopped: Vec,
#[serde(default)]
- failed: Vec,
+ pub failed: Vec,
#[serde(default)]
- returned_tab_ids: Vec,
+ pub returned_tab_ids: Vec,
#[serde(default)]
- return_failures: Vec,
+ pub return_failures: Vec,
}
#[derive(Debug, Deserialize, Serialize)]
-struct StopFailure {
- session_id: String,
- code: ErrorCode,
- message: String,
+pub struct StopFailure {
+ pub session_id: String,
+ pub code: ErrorCode,
+ pub message: String,
}
#[derive(Debug, Deserialize)]
@@ -140,14 +140,7 @@ fn run_start(sock: PathBuf, args: SessionStartArgs, format: Format) -> Result<()
}
});
}
- let result: Result = call(
- sock,
- Method::SessionStart,
- Some(StartParams {
- browser_instance_id: args.browser,
- }),
- SESSION_START_IPC_TIMEOUT,
- );
+ let result = start_session(sock, args.browser);
waited.store(true, Ordering::SeqCst);
match result {
Ok(reply) => match format {
@@ -171,6 +164,31 @@ fn run_start(sock: PathBuf, args: SessionStartArgs, format: Format) -> Result<()
Ok(())
}
+/// Start a session and open the Agent Window. Used by `session start` and `record start`.
+pub fn start_session(sock: PathBuf, browser: Option) -> Result {
+ call(
+ sock,
+ Method::SessionStart,
+ Some(StartParams {
+ browser_instance_id: browser,
+ }),
+ SESSION_START_IPC_TIMEOUT,
+ )
+}
+
+/// Stop a single session by id.
+pub fn stop_session(sock: PathBuf, session_id: &str) -> Result {
+ call(
+ sock,
+ Method::SessionStop,
+ Some(StopParams {
+ session_id: Some(session_id.to_string()),
+ all: false,
+ }),
+ SESSION_STOP_IPC_TIMEOUT,
+ )
+}
+
/// Render a `session.start` failure (review I3).
///
/// The previous implementation hand-rolled `eprintln!("error: ...")`
diff --git a/crates/bsk-cli/src/daemon/ipc.rs b/crates/bsk-cli/src/daemon/ipc.rs
index 654d96c..d273bcf 100644
--- a/crates/bsk-cli/src/daemon/ipc.rs
+++ b/crates/bsk-cli/src/daemon/ipc.rs
@@ -242,7 +242,10 @@ pub fn full_handler(status: DaemonStatus, state: Arc) -> RpcHandler
| Method::ToolSelect
| Method::ToolEvaluate
| Method::ToolWaitForNavigation
- | Method::ToolRequestHelp => {
+ | Method::ToolRequestHelp
+ | Method::ToolRecordStart
+ | Method::ToolRecordStop
+ | Method::ToolRecordAwait => {
handle_tool_dispatch(&state, rpc_id, method, params).await
}
Method::ToolWaitMs => handle_wait_ms(&state.abort_registry, rpc_id, params).await,
@@ -319,10 +322,19 @@ async fn handle_tool_dispatch(
}
};
let entry = inflight_guard.entry();
- let outcome = state
- .tool_queues
- .dispatch(&session_id, method, params, timeout, Some(entry))
- .await;
+ // `record_stop` must reach the extension while `record_await` holds the
+ // serial busy lock — finishing the recording unblocks await.
+ let outcome = if method == Method::ToolRecordStop {
+ state
+ .tool_queues
+ .dispatch_unlocked(&session_id, method, params, timeout, Some(entry))
+ .await
+ } else {
+ state
+ .tool_queues
+ .dispatch(&session_id, method, params, timeout, Some(entry))
+ .await
+ };
drop(inflight_guard);
match outcome {
Ok(v) => ResponseBody::Ok(v),
diff --git a/crates/bsk-cli/src/daemon/paths.rs b/crates/bsk-cli/src/daemon/paths.rs
index 1cf2bb5..f3ec24d 100644
--- a/crates/bsk-cli/src/daemon/paths.rs
+++ b/crates/bsk-cli/src/daemon/paths.rs
@@ -95,6 +95,11 @@ pub fn sock_path() -> Result {
Ok(bsk_home()?.join("run").join("daemon.sock"))
}
+/// Path to the in-progress recording session state (`record-session.json`).
+pub fn record_session_path() -> Result {
+ Ok(bsk_home()?.join("record-session.json"))
+}
+
/// Windows named-pipe name. Include the resolved `BSK_HOME` path in the
/// token so test homes and custom installs do not share a predictable
/// per-username pipe.
@@ -163,6 +168,10 @@ mod tests {
assert_eq!(log_path().unwrap(), home.join("daemon.log"));
assert_eq!(update_check_path().unwrap(), home.join("update-check.json"));
assert_eq!(sock_path().unwrap(), home.join("run").join("daemon.sock"));
+ assert_eq!(
+ record_session_path().unwrap(),
+ home.join("record-session.json")
+ );
});
}
}
diff --git a/crates/bsk-cli/src/daemon/queue.rs b/crates/bsk-cli/src/daemon/queue.rs
index 919c207..3fac0eb 100644
--- a/crates/bsk-cli/src/daemon/queue.rs
+++ b/crates/bsk-cli/src/daemon/queue.rs
@@ -221,6 +221,13 @@ impl ToolQueueRegistry {
guard.get(sid).is_some_and(|entry| entry.accepting)
}
+ #[cfg(test)]
+ fn force_busy_for_tests(&self, sid: &SessionId, busy: bool) {
+ let guard = self.queues.lock().expect("tool queue registry poisoned");
+ let entry = guard.get(sid).expect("queue must exist");
+ entry.state.lock().expect("queue state poisoned").busy = busy;
+ }
+
/// Spawn the worker task for a session id. Idempotent: spawning the
/// same id twice replaces the previous queue (the previous worker
/// closes when its sender drops).
@@ -320,6 +327,42 @@ impl ToolQueueRegistry {
outcome
}
+ /// Forward a tool RPC to the extension without taking the
+ /// per-session busy lock. Used by `tool.record_stop` so a second
+ /// CLI can finish an in-flight `tool.record_await` (extension
+ /// dispatch already runs concurrent `void dispatch(msg)` handlers).
+ pub async fn dispatch_unlocked(
+ &self,
+ sid: &SessionId,
+ method: Method,
+ params: Value,
+ timeout: Duration,
+ inflight: Option>,
+ ) -> Result {
+ {
+ let guard = self.queues.lock().expect("tool queue registry poisoned");
+ let Some(entry) = guard.get(sid) else {
+ return Err(DispatchError::SessionNotFound);
+ };
+ if !entry.accepting {
+ return Err(DispatchError::SessionStopping);
+ }
+ }
+ let (respond_tx, _respond_rx) = oneshot::channel();
+ let job = ToolJob {
+ method,
+ params,
+ timeout,
+ respond: respond_tx,
+ inflight,
+ session_id: sid.clone(),
+ };
+ match forward_one(sid, &self.browsers, &self.sessions, &job).await {
+ Ok(v) => Ok(v),
+ Err(err) => Err(DispatchError::Rpc(err)),
+ }
+ }
+
/// Stop accepting new jobs for `sid`, enqueue one final control RPC,
/// and wait for it behind any already-queued tools. Used by
/// `session.stop` so extension teardown cannot race an in-flight
@@ -789,3 +832,50 @@ mod tool_job_session_id_tests {
assert_eq!(job.session_id.0, "sess-A");
}
}
+
+#[cfg(test)]
+mod dispatch_unlocked_tests {
+ use super::*;
+ use crate::daemon::browsers::BrowserRegistry;
+ use crate::daemon::sessions::{SessionId, SessionRegistry};
+ use bsk_protocol::Method;
+ use serde_json::json;
+ use std::sync::Arc;
+
+ #[tokio::test]
+ async fn unlocked_dispatch_skips_session_busy_gate() {
+ let browsers = Arc::new(BrowserRegistry::new());
+ let sessions = Arc::new(SessionRegistry::new());
+ let queues = ToolQueueRegistry::new(browsers, sessions);
+ let sid = SessionId("s-record".into());
+ queues.spawn(sid.clone());
+ queues.force_busy_for_tests(&sid, true);
+
+ let busy = queues
+ .dispatch(
+ &sid,
+ Method::ToolTabList,
+ json!({ "session_id": "s-record" }),
+ Duration::from_secs(1),
+ None,
+ )
+ .await;
+ assert!(matches!(busy, Err(DispatchError::SessionBusy)));
+
+ let unlocked = queues
+ .dispatch_unlocked(
+ &sid,
+ Method::ToolRecordStop,
+ json!({ "session_id": "s-record" }),
+ Duration::from_secs(1),
+ None,
+ )
+ .await;
+ assert!(
+ !matches!(unlocked, Err(DispatchError::SessionBusy)),
+ "record_stop must not be rejected as SessionBusy; got {unlocked:?}"
+ );
+ // No session row → forward_one returns NotFound as Rpc.
+ assert!(matches!(unlocked, Err(DispatchError::Rpc(_))));
+ }
+}
diff --git a/crates/bsk-cli/src/main.rs b/crates/bsk-cli/src/main.rs
index dd720b9..cdc9e73 100644
--- a/crates/bsk-cli/src/main.rs
+++ b/crates/bsk-cli/src/main.rs
@@ -97,6 +97,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
Command::WaitForNavigation(args) => cli::waits::dispatch_wait_for_navigation(args, format),
Command::WaitMs(args) => cli::waits::dispatch_wait_ms(args, format),
Command::RequestHelp(args) => cli::human_loop::dispatch(args, format),
+ Command::Record(cmd) => cli::record::dispatch(cmd, format),
}
}
diff --git a/crates/bsk-cli/tests/cli_parse.rs b/crates/bsk-cli/tests/cli_parse.rs
index f3fd37e..25c6739 100644
--- a/crates/bsk-cli/tests/cli_parse.rs
+++ b/crates/bsk-cli/tests/cli_parse.rs
@@ -4,6 +4,7 @@ use std::time::Duration;
use bsk::cli::daemon::{DaemonCmd, parse_duration};
use bsk::cli::navigate::NavigateCmd;
+use bsk::cli::record::{RecordCmd, RecordSub};
use bsk::{Cli, Command};
use clap::Parser;
@@ -192,3 +193,37 @@ fn rejects_zero_click_count() {
Cli::try_parse_from(["bsk", "click", "@e1", "--session", "s1", "--count", "0"]).is_err()
);
}
+
+#[test]
+fn parses_record_start_with_browser_and_url() {
+ let cli = parse(&[
+ "bsk",
+ "record",
+ "start",
+ "--browser",
+ "022ca8ac",
+ "--url",
+ "https://x",
+ ]);
+ let Command::Record(RecordCmd {
+ sub: RecordSub::Start(args),
+ }) = cli.command
+ else {
+ panic!("expected record start subcommand");
+ };
+ assert_eq!(args.browser.as_deref(), Some("022ca8ac"));
+ assert_eq!(args.url.as_deref(), Some("https://x"));
+}
+
+#[test]
+fn parses_record_start_without_url() {
+ let cli = parse(&["bsk", "record", "start", "--browser", "022ca8ac"]);
+ let Command::Record(RecordCmd {
+ sub: RecordSub::Start(args),
+ }) = cli.command
+ else {
+ panic!("expected record start subcommand");
+ };
+ assert_eq!(args.browser.as_deref(), Some("022ca8ac"));
+ assert!(args.url.is_none());
+}
diff --git a/crates/bsk-protocol/schema/tool_record_await_params.json b/crates/bsk-protocol/schema/tool_record_await_params.json
new file mode 100644
index 0000000..7a8350a
--- /dev/null
+++ b/crates/bsk-protocol/schema/tool_record_await_params.json
@@ -0,0 +1,21 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "RecordAwaitParams",
+ "type": "object",
+ "required": [
+ "session_id"
+ ],
+ "properties": {
+ "session_id": {
+ "type": "string"
+ },
+ "timeout_ms": {
+ "type": [
+ "integer",
+ "null"
+ ],
+ "format": "uint32",
+ "minimum": 0.0
+ }
+ }
+}
diff --git a/crates/bsk-protocol/schema/tool_record_await_result.json b/crates/bsk-protocol/schema/tool_record_await_result.json
new file mode 100644
index 0000000..dad5f66
--- /dev/null
+++ b/crates/bsk-protocol/schema/tool_record_await_result.json
@@ -0,0 +1,417 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "RecordAwaitResult",
+ "type": "object",
+ "required": [
+ "trace"
+ ],
+ "properties": {
+ "trace": {
+ "$ref": "#/definitions/Trace"
+ }
+ },
+ "definitions": {
+ "KeyModifier": {
+ "description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
+ "type": "string",
+ "enum": [
+ "alt",
+ "ctrl",
+ "meta",
+ "shift"
+ ]
+ },
+ "PageRef": {
+ "description": "Page context dictionary entry — referenced by steps via `page` id.",
+ "type": "object",
+ "required": [
+ "id",
+ "url"
+ ],
+ "properties": {
+ "id": {
+ "type": "string"
+ },
+ "title": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "url": {
+ "type": "string"
+ }
+ }
+ },
+ "SelectedOption": {
+ "description": "One selected option (`select` op).",
+ "type": "object",
+ "required": [
+ "value"
+ ],
+ "properties": {
+ "label": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "value": {
+ "type": "string"
+ }
+ }
+ },
+ "Step": {
+ "description": "One recorded user action — discriminated union by `op`.",
+ "oneOf": [
+ {
+ "description": "Fields shared by every step variant (flattened in JSON).",
+ "type": "object",
+ "required": [
+ "id",
+ "op",
+ "page",
+ "to"
+ ],
+ "properties": {
+ "effect": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/StepEffect"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "id": {
+ "type": "integer",
+ "format": "uint32",
+ "minimum": 0.0
+ },
+ "op": {
+ "type": "string",
+ "enum": [
+ "navigate"
+ ]
+ },
+ "page": {
+ "description": "Reference into `pages[]`.",
+ "type": "string"
+ },
+ "to": {
+ "type": "string"
+ }
+ }
+ },
+ {
+ "description": "Fields shared by every step variant (flattened in JSON).",
+ "type": "object",
+ "required": [
+ "id",
+ "op",
+ "page",
+ "target"
+ ],
+ "properties": {
+ "effect": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/StepEffect"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "id": {
+ "type": "integer",
+ "format": "uint32",
+ "minimum": 0.0
+ },
+ "op": {
+ "type": "string",
+ "enum": [
+ "click"
+ ]
+ },
+ "page": {
+ "description": "Reference into `pages[]`.",
+ "type": "string"
+ },
+ "target": {
+ "$ref": "#/definitions/TargetDescriptor"
+ }
+ }
+ },
+ {
+ "description": "Fields shared by every step variant (flattened in JSON).",
+ "type": "object",
+ "required": [
+ "id",
+ "op",
+ "page",
+ "target",
+ "value"
+ ],
+ "properties": {
+ "effect": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/StepEffect"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "id": {
+ "type": "integer",
+ "format": "uint32",
+ "minimum": 0.0
+ },
+ "op": {
+ "type": "string",
+ "enum": [
+ "fill"
+ ]
+ },
+ "page": {
+ "description": "Reference into `pages[]`.",
+ "type": "string"
+ },
+ "redacted": {
+ "type": [
+ "boolean",
+ "null"
+ ]
+ },
+ "target": {
+ "$ref": "#/definitions/TargetDescriptor"
+ },
+ "value": {
+ "type": "string"
+ }
+ }
+ },
+ {
+ "description": "Fields shared by every step variant (flattened in JSON).",
+ "type": "object",
+ "required": [
+ "id",
+ "op",
+ "page",
+ "selection",
+ "target"
+ ],
+ "properties": {
+ "effect": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/StepEffect"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "id": {
+ "type": "integer",
+ "format": "uint32",
+ "minimum": 0.0
+ },
+ "op": {
+ "type": "string",
+ "enum": [
+ "select"
+ ]
+ },
+ "page": {
+ "description": "Reference into `pages[]`.",
+ "type": "string"
+ },
+ "selection": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/SelectedOption"
+ }
+ },
+ "target": {
+ "$ref": "#/definitions/TargetDescriptor"
+ }
+ }
+ },
+ {
+ "description": "Fields shared by every step variant (flattened in JSON).",
+ "type": "object",
+ "required": [
+ "id",
+ "key",
+ "op",
+ "page"
+ ],
+ "properties": {
+ "effect": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/StepEffect"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "id": {
+ "type": "integer",
+ "format": "uint32",
+ "minimum": 0.0
+ },
+ "key": {
+ "type": "string"
+ },
+ "modifiers": {
+ "type": [
+ "array",
+ "null"
+ ],
+ "items": {
+ "$ref": "#/definitions/KeyModifier"
+ }
+ },
+ "op": {
+ "type": "string",
+ "enum": [
+ "press"
+ ]
+ },
+ "page": {
+ "description": "Reference into `pages[]`.",
+ "type": "string"
+ },
+ "target": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/TargetDescriptor"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ }
+ }
+ }
+ ]
+ },
+ "StepEffect": {
+ "description": "Observed navigation after a step (objective fact only).",
+ "type": "object",
+ "required": [
+ "navigated_to"
+ ],
+ "properties": {
+ "navigated_to": {
+ "description": "Reference into `pages[]` for the destination page.",
+ "type": "string"
+ }
+ }
+ },
+ "TargetDescriptor": {
+ "description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
+ "type": "object",
+ "required": [
+ "tag"
+ ],
+ "properties": {
+ "name": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "name_attr": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "nearby_label": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "placeholder": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "role": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "tag": {
+ "type": "string"
+ }
+ }
+ },
+ "Trace": {
+ "description": "Persisted user-action trace exported by `tool.record_stop` / `await`.",
+ "type": "object",
+ "required": [
+ "entry",
+ "pages",
+ "recorded_at",
+ "steps"
+ ],
+ "properties": {
+ "entry": {
+ "$ref": "#/definitions/TraceEntry"
+ },
+ "pages": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/PageRef"
+ }
+ },
+ "purpose": {
+ "description": "Optional user-provided goal from `--purpose` (metadata only).",
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "recorded_at": {
+ "description": "RFC 3339 timestamp when recording stopped.",
+ "type": "string"
+ },
+ "started_at": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "steps": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/Step"
+ }
+ }
+ }
+ },
+ "TraceEntry": {
+ "description": "Recording entry point — first URL the flow starts from.",
+ "type": "object",
+ "required": [
+ "start_url"
+ ],
+ "properties": {
+ "start_url": {
+ "type": "string"
+ }
+ }
+ }
+ }
+}
diff --git a/crates/bsk-protocol/schema/tool_record_start_params.json b/crates/bsk-protocol/schema/tool_record_start_params.json
new file mode 100644
index 0000000..859ebf8
--- /dev/null
+++ b/crates/bsk-protocol/schema/tool_record_start_params.json
@@ -0,0 +1,32 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "RecordStartParams",
+ "type": "object",
+ "required": [
+ "session_id"
+ ],
+ "properties": {
+ "purpose": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "session_id": {
+ "type": "string"
+ },
+ "tab_id": {
+ "type": [
+ "integer",
+ "null"
+ ],
+ "format": "int64"
+ },
+ "url": {
+ "type": [
+ "string",
+ "null"
+ ]
+ }
+ }
+}
diff --git a/crates/bsk-protocol/schema/tool_record_start_result.json b/crates/bsk-protocol/schema/tool_record_start_result.json
new file mode 100644
index 0000000..5c04ceb
--- /dev/null
+++ b/crates/bsk-protocol/schema/tool_record_start_result.json
@@ -0,0 +1,18 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "RecordStartResult",
+ "type": "object",
+ "required": [
+ "recording",
+ "tab_id"
+ ],
+ "properties": {
+ "recording": {
+ "type": "boolean"
+ },
+ "tab_id": {
+ "type": "integer",
+ "format": "int64"
+ }
+ }
+}
diff --git a/crates/bsk-protocol/schema/tool_record_stop_params.json b/crates/bsk-protocol/schema/tool_record_stop_params.json
new file mode 100644
index 0000000..2132963
--- /dev/null
+++ b/crates/bsk-protocol/schema/tool_record_stop_params.json
@@ -0,0 +1,13 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "RecordStopParams",
+ "type": "object",
+ "required": [
+ "session_id"
+ ],
+ "properties": {
+ "session_id": {
+ "type": "string"
+ }
+ }
+}
diff --git a/crates/bsk-protocol/schema/tool_record_stop_result.json b/crates/bsk-protocol/schema/tool_record_stop_result.json
new file mode 100644
index 0000000..e3da733
--- /dev/null
+++ b/crates/bsk-protocol/schema/tool_record_stop_result.json
@@ -0,0 +1,417 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "RecordStopResult",
+ "type": "object",
+ "required": [
+ "trace"
+ ],
+ "properties": {
+ "trace": {
+ "$ref": "#/definitions/Trace"
+ }
+ },
+ "definitions": {
+ "KeyModifier": {
+ "description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
+ "type": "string",
+ "enum": [
+ "alt",
+ "ctrl",
+ "meta",
+ "shift"
+ ]
+ },
+ "PageRef": {
+ "description": "Page context dictionary entry — referenced by steps via `page` id.",
+ "type": "object",
+ "required": [
+ "id",
+ "url"
+ ],
+ "properties": {
+ "id": {
+ "type": "string"
+ },
+ "title": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "url": {
+ "type": "string"
+ }
+ }
+ },
+ "SelectedOption": {
+ "description": "One selected option (`select` op).",
+ "type": "object",
+ "required": [
+ "value"
+ ],
+ "properties": {
+ "label": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "value": {
+ "type": "string"
+ }
+ }
+ },
+ "Step": {
+ "description": "One recorded user action — discriminated union by `op`.",
+ "oneOf": [
+ {
+ "description": "Fields shared by every step variant (flattened in JSON).",
+ "type": "object",
+ "required": [
+ "id",
+ "op",
+ "page",
+ "to"
+ ],
+ "properties": {
+ "effect": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/StepEffect"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "id": {
+ "type": "integer",
+ "format": "uint32",
+ "minimum": 0.0
+ },
+ "op": {
+ "type": "string",
+ "enum": [
+ "navigate"
+ ]
+ },
+ "page": {
+ "description": "Reference into `pages[]`.",
+ "type": "string"
+ },
+ "to": {
+ "type": "string"
+ }
+ }
+ },
+ {
+ "description": "Fields shared by every step variant (flattened in JSON).",
+ "type": "object",
+ "required": [
+ "id",
+ "op",
+ "page",
+ "target"
+ ],
+ "properties": {
+ "effect": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/StepEffect"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "id": {
+ "type": "integer",
+ "format": "uint32",
+ "minimum": 0.0
+ },
+ "op": {
+ "type": "string",
+ "enum": [
+ "click"
+ ]
+ },
+ "page": {
+ "description": "Reference into `pages[]`.",
+ "type": "string"
+ },
+ "target": {
+ "$ref": "#/definitions/TargetDescriptor"
+ }
+ }
+ },
+ {
+ "description": "Fields shared by every step variant (flattened in JSON).",
+ "type": "object",
+ "required": [
+ "id",
+ "op",
+ "page",
+ "target",
+ "value"
+ ],
+ "properties": {
+ "effect": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/StepEffect"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "id": {
+ "type": "integer",
+ "format": "uint32",
+ "minimum": 0.0
+ },
+ "op": {
+ "type": "string",
+ "enum": [
+ "fill"
+ ]
+ },
+ "page": {
+ "description": "Reference into `pages[]`.",
+ "type": "string"
+ },
+ "redacted": {
+ "type": [
+ "boolean",
+ "null"
+ ]
+ },
+ "target": {
+ "$ref": "#/definitions/TargetDescriptor"
+ },
+ "value": {
+ "type": "string"
+ }
+ }
+ },
+ {
+ "description": "Fields shared by every step variant (flattened in JSON).",
+ "type": "object",
+ "required": [
+ "id",
+ "op",
+ "page",
+ "selection",
+ "target"
+ ],
+ "properties": {
+ "effect": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/StepEffect"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "id": {
+ "type": "integer",
+ "format": "uint32",
+ "minimum": 0.0
+ },
+ "op": {
+ "type": "string",
+ "enum": [
+ "select"
+ ]
+ },
+ "page": {
+ "description": "Reference into `pages[]`.",
+ "type": "string"
+ },
+ "selection": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/SelectedOption"
+ }
+ },
+ "target": {
+ "$ref": "#/definitions/TargetDescriptor"
+ }
+ }
+ },
+ {
+ "description": "Fields shared by every step variant (flattened in JSON).",
+ "type": "object",
+ "required": [
+ "id",
+ "key",
+ "op",
+ "page"
+ ],
+ "properties": {
+ "effect": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/StepEffect"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "id": {
+ "type": "integer",
+ "format": "uint32",
+ "minimum": 0.0
+ },
+ "key": {
+ "type": "string"
+ },
+ "modifiers": {
+ "type": [
+ "array",
+ "null"
+ ],
+ "items": {
+ "$ref": "#/definitions/KeyModifier"
+ }
+ },
+ "op": {
+ "type": "string",
+ "enum": [
+ "press"
+ ]
+ },
+ "page": {
+ "description": "Reference into `pages[]`.",
+ "type": "string"
+ },
+ "target": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/TargetDescriptor"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ }
+ }
+ }
+ ]
+ },
+ "StepEffect": {
+ "description": "Observed navigation after a step (objective fact only).",
+ "type": "object",
+ "required": [
+ "navigated_to"
+ ],
+ "properties": {
+ "navigated_to": {
+ "description": "Reference into `pages[]` for the destination page.",
+ "type": "string"
+ }
+ }
+ },
+ "TargetDescriptor": {
+ "description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
+ "type": "object",
+ "required": [
+ "tag"
+ ],
+ "properties": {
+ "name": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "name_attr": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "nearby_label": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "placeholder": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "role": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "tag": {
+ "type": "string"
+ }
+ }
+ },
+ "Trace": {
+ "description": "Persisted user-action trace exported by `tool.record_stop` / `await`.",
+ "type": "object",
+ "required": [
+ "entry",
+ "pages",
+ "recorded_at",
+ "steps"
+ ],
+ "properties": {
+ "entry": {
+ "$ref": "#/definitions/TraceEntry"
+ },
+ "pages": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/PageRef"
+ }
+ },
+ "purpose": {
+ "description": "Optional user-provided goal from `--purpose` (metadata only).",
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "recorded_at": {
+ "description": "RFC 3339 timestamp when recording stopped.",
+ "type": "string"
+ },
+ "started_at": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "steps": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/Step"
+ }
+ }
+ }
+ },
+ "TraceEntry": {
+ "description": "Recording entry point — first URL the flow starts from.",
+ "type": "object",
+ "required": [
+ "start_url"
+ ],
+ "properties": {
+ "start_url": {
+ "type": "string"
+ }
+ }
+ }
+ }
+}
diff --git a/crates/bsk-protocol/schema/trace.json b/crates/bsk-protocol/schema/trace.json
new file mode 100644
index 0000000..a1898e6
--- /dev/null
+++ b/crates/bsk-protocol/schema/trace.json
@@ -0,0 +1,406 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "Trace",
+ "description": "Persisted user-action trace exported by `tool.record_stop` / `await`.",
+ "type": "object",
+ "required": [
+ "entry",
+ "pages",
+ "recorded_at",
+ "steps"
+ ],
+ "properties": {
+ "entry": {
+ "$ref": "#/definitions/TraceEntry"
+ },
+ "pages": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/PageRef"
+ }
+ },
+ "purpose": {
+ "description": "Optional user-provided goal from `--purpose` (metadata only).",
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "recorded_at": {
+ "description": "RFC 3339 timestamp when recording stopped.",
+ "type": "string"
+ },
+ "started_at": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "steps": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/Step"
+ }
+ }
+ },
+ "definitions": {
+ "KeyModifier": {
+ "description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
+ "type": "string",
+ "enum": [
+ "alt",
+ "ctrl",
+ "meta",
+ "shift"
+ ]
+ },
+ "PageRef": {
+ "description": "Page context dictionary entry — referenced by steps via `page` id.",
+ "type": "object",
+ "required": [
+ "id",
+ "url"
+ ],
+ "properties": {
+ "id": {
+ "type": "string"
+ },
+ "title": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "url": {
+ "type": "string"
+ }
+ }
+ },
+ "SelectedOption": {
+ "description": "One selected option (`select` op).",
+ "type": "object",
+ "required": [
+ "value"
+ ],
+ "properties": {
+ "label": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "value": {
+ "type": "string"
+ }
+ }
+ },
+ "Step": {
+ "description": "One recorded user action — discriminated union by `op`.",
+ "oneOf": [
+ {
+ "description": "Fields shared by every step variant (flattened in JSON).",
+ "type": "object",
+ "required": [
+ "id",
+ "op",
+ "page",
+ "to"
+ ],
+ "properties": {
+ "effect": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/StepEffect"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "id": {
+ "type": "integer",
+ "format": "uint32",
+ "minimum": 0.0
+ },
+ "op": {
+ "type": "string",
+ "enum": [
+ "navigate"
+ ]
+ },
+ "page": {
+ "description": "Reference into `pages[]`.",
+ "type": "string"
+ },
+ "to": {
+ "type": "string"
+ }
+ }
+ },
+ {
+ "description": "Fields shared by every step variant (flattened in JSON).",
+ "type": "object",
+ "required": [
+ "id",
+ "op",
+ "page",
+ "target"
+ ],
+ "properties": {
+ "effect": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/StepEffect"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "id": {
+ "type": "integer",
+ "format": "uint32",
+ "minimum": 0.0
+ },
+ "op": {
+ "type": "string",
+ "enum": [
+ "click"
+ ]
+ },
+ "page": {
+ "description": "Reference into `pages[]`.",
+ "type": "string"
+ },
+ "target": {
+ "$ref": "#/definitions/TargetDescriptor"
+ }
+ }
+ },
+ {
+ "description": "Fields shared by every step variant (flattened in JSON).",
+ "type": "object",
+ "required": [
+ "id",
+ "op",
+ "page",
+ "target",
+ "value"
+ ],
+ "properties": {
+ "effect": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/StepEffect"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "id": {
+ "type": "integer",
+ "format": "uint32",
+ "minimum": 0.0
+ },
+ "op": {
+ "type": "string",
+ "enum": [
+ "fill"
+ ]
+ },
+ "page": {
+ "description": "Reference into `pages[]`.",
+ "type": "string"
+ },
+ "redacted": {
+ "type": [
+ "boolean",
+ "null"
+ ]
+ },
+ "target": {
+ "$ref": "#/definitions/TargetDescriptor"
+ },
+ "value": {
+ "type": "string"
+ }
+ }
+ },
+ {
+ "description": "Fields shared by every step variant (flattened in JSON).",
+ "type": "object",
+ "required": [
+ "id",
+ "op",
+ "page",
+ "selection",
+ "target"
+ ],
+ "properties": {
+ "effect": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/StepEffect"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "id": {
+ "type": "integer",
+ "format": "uint32",
+ "minimum": 0.0
+ },
+ "op": {
+ "type": "string",
+ "enum": [
+ "select"
+ ]
+ },
+ "page": {
+ "description": "Reference into `pages[]`.",
+ "type": "string"
+ },
+ "selection": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/SelectedOption"
+ }
+ },
+ "target": {
+ "$ref": "#/definitions/TargetDescriptor"
+ }
+ }
+ },
+ {
+ "description": "Fields shared by every step variant (flattened in JSON).",
+ "type": "object",
+ "required": [
+ "id",
+ "key",
+ "op",
+ "page"
+ ],
+ "properties": {
+ "effect": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/StepEffect"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "id": {
+ "type": "integer",
+ "format": "uint32",
+ "minimum": 0.0
+ },
+ "key": {
+ "type": "string"
+ },
+ "modifiers": {
+ "type": [
+ "array",
+ "null"
+ ],
+ "items": {
+ "$ref": "#/definitions/KeyModifier"
+ }
+ },
+ "op": {
+ "type": "string",
+ "enum": [
+ "press"
+ ]
+ },
+ "page": {
+ "description": "Reference into `pages[]`.",
+ "type": "string"
+ },
+ "target": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/TargetDescriptor"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ }
+ }
+ }
+ ]
+ },
+ "StepEffect": {
+ "description": "Observed navigation after a step (objective fact only).",
+ "type": "object",
+ "required": [
+ "navigated_to"
+ ],
+ "properties": {
+ "navigated_to": {
+ "description": "Reference into `pages[]` for the destination page.",
+ "type": "string"
+ }
+ }
+ },
+ "TargetDescriptor": {
+ "description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
+ "type": "object",
+ "required": [
+ "tag"
+ ],
+ "properties": {
+ "name": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "name_attr": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "nearby_label": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "placeholder": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "role": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "tag": {
+ "type": "string"
+ }
+ }
+ },
+ "TraceEntry": {
+ "description": "Recording entry point — first URL the flow starts from.",
+ "type": "object",
+ "required": [
+ "start_url"
+ ],
+ "properties": {
+ "start_url": {
+ "type": "string"
+ }
+ }
+ }
+ }
+}
diff --git a/crates/bsk-protocol/schema/trace_step.json b/crates/bsk-protocol/schema/trace_step.json
new file mode 100644
index 0000000..046925e
--- /dev/null
+++ b/crates/bsk-protocol/schema/trace_step.json
@@ -0,0 +1,328 @@
+{
+ "$schema": "http://json-schema.org/draft-07/schema#",
+ "title": "Step",
+ "description": "One recorded user action — discriminated union by `op`.",
+ "oneOf": [
+ {
+ "description": "Fields shared by every step variant (flattened in JSON).",
+ "type": "object",
+ "required": [
+ "id",
+ "op",
+ "page",
+ "to"
+ ],
+ "properties": {
+ "effect": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/StepEffect"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "id": {
+ "type": "integer",
+ "format": "uint32",
+ "minimum": 0.0
+ },
+ "op": {
+ "type": "string",
+ "enum": [
+ "navigate"
+ ]
+ },
+ "page": {
+ "description": "Reference into `pages[]`.",
+ "type": "string"
+ },
+ "to": {
+ "type": "string"
+ }
+ }
+ },
+ {
+ "description": "Fields shared by every step variant (flattened in JSON).",
+ "type": "object",
+ "required": [
+ "id",
+ "op",
+ "page",
+ "target"
+ ],
+ "properties": {
+ "effect": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/StepEffect"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "id": {
+ "type": "integer",
+ "format": "uint32",
+ "minimum": 0.0
+ },
+ "op": {
+ "type": "string",
+ "enum": [
+ "click"
+ ]
+ },
+ "page": {
+ "description": "Reference into `pages[]`.",
+ "type": "string"
+ },
+ "target": {
+ "$ref": "#/definitions/TargetDescriptor"
+ }
+ }
+ },
+ {
+ "description": "Fields shared by every step variant (flattened in JSON).",
+ "type": "object",
+ "required": [
+ "id",
+ "op",
+ "page",
+ "target",
+ "value"
+ ],
+ "properties": {
+ "effect": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/StepEffect"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "id": {
+ "type": "integer",
+ "format": "uint32",
+ "minimum": 0.0
+ },
+ "op": {
+ "type": "string",
+ "enum": [
+ "fill"
+ ]
+ },
+ "page": {
+ "description": "Reference into `pages[]`.",
+ "type": "string"
+ },
+ "redacted": {
+ "type": [
+ "boolean",
+ "null"
+ ]
+ },
+ "target": {
+ "$ref": "#/definitions/TargetDescriptor"
+ },
+ "value": {
+ "type": "string"
+ }
+ }
+ },
+ {
+ "description": "Fields shared by every step variant (flattened in JSON).",
+ "type": "object",
+ "required": [
+ "id",
+ "op",
+ "page",
+ "selection",
+ "target"
+ ],
+ "properties": {
+ "effect": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/StepEffect"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "id": {
+ "type": "integer",
+ "format": "uint32",
+ "minimum": 0.0
+ },
+ "op": {
+ "type": "string",
+ "enum": [
+ "select"
+ ]
+ },
+ "page": {
+ "description": "Reference into `pages[]`.",
+ "type": "string"
+ },
+ "selection": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/SelectedOption"
+ }
+ },
+ "target": {
+ "$ref": "#/definitions/TargetDescriptor"
+ }
+ }
+ },
+ {
+ "description": "Fields shared by every step variant (flattened in JSON).",
+ "type": "object",
+ "required": [
+ "id",
+ "key",
+ "op",
+ "page"
+ ],
+ "properties": {
+ "effect": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/StepEffect"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ },
+ "id": {
+ "type": "integer",
+ "format": "uint32",
+ "minimum": 0.0
+ },
+ "key": {
+ "type": "string"
+ },
+ "modifiers": {
+ "type": [
+ "array",
+ "null"
+ ],
+ "items": {
+ "$ref": "#/definitions/KeyModifier"
+ }
+ },
+ "op": {
+ "type": "string",
+ "enum": [
+ "press"
+ ]
+ },
+ "page": {
+ "description": "Reference into `pages[]`.",
+ "type": "string"
+ },
+ "target": {
+ "anyOf": [
+ {
+ "$ref": "#/definitions/TargetDescriptor"
+ },
+ {
+ "type": "null"
+ }
+ ]
+ }
+ }
+ }
+ ],
+ "definitions": {
+ "KeyModifier": {
+ "description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
+ "type": "string",
+ "enum": [
+ "alt",
+ "ctrl",
+ "meta",
+ "shift"
+ ]
+ },
+ "SelectedOption": {
+ "description": "One selected option (`select` op).",
+ "type": "object",
+ "required": [
+ "value"
+ ],
+ "properties": {
+ "label": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "value": {
+ "type": "string"
+ }
+ }
+ },
+ "StepEffect": {
+ "description": "Observed navigation after a step (objective fact only).",
+ "type": "object",
+ "required": [
+ "navigated_to"
+ ],
+ "properties": {
+ "navigated_to": {
+ "description": "Reference into `pages[]` for the destination page.",
+ "type": "string"
+ }
+ }
+ },
+ "TargetDescriptor": {
+ "description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
+ "type": "object",
+ "required": [
+ "tag"
+ ],
+ "properties": {
+ "name": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "name_attr": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "nearby_label": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "placeholder": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "role": {
+ "type": [
+ "string",
+ "null"
+ ]
+ },
+ "tag": {
+ "type": "string"
+ }
+ }
+ }
+ }
+}
diff --git a/crates/bsk-protocol/src/bin/dump-schema.rs b/crates/bsk-protocol/src/bin/dump-schema.rs
index d20efa3..f226e44 100644
--- a/crates/bsk-protocol/src/bin/dump-schema.rs
+++ b/crates/bsk-protocol/src/bin/dump-schema.rs
@@ -100,4 +100,13 @@ fn main() {
dump!(WaitMsResult, "tool_wait_ms_result");
dump!(RequestHelpParams, "tool_request_help_params");
dump!(RequestHelpResult, "tool_request_help_result");
+
+ dump!(Trace, "trace");
+ dump!(Step, "trace_step");
+ dump!(RecordStartParams, "tool_record_start_params");
+ dump!(RecordStartResult, "tool_record_start_result");
+ dump!(RecordStopParams, "tool_record_stop_params");
+ dump!(RecordStopResult, "tool_record_stop_result");
+ dump!(RecordAwaitParams, "tool_record_await_params");
+ dump!(RecordAwaitResult, "tool_record_await_result");
}
diff --git a/crates/bsk-protocol/src/method.rs b/crates/bsk-protocol/src/method.rs
index e54a66a..d4148c7 100644
--- a/crates/bsk-protocol/src/method.rs
+++ b/crates/bsk-protocol/src/method.rs
@@ -74,6 +74,12 @@ pub enum Method {
ToolWaitMs,
#[serde(rename = "tool.request_help")]
ToolRequestHelp,
+ #[serde(rename = "tool.record_start")]
+ ToolRecordStart,
+ #[serde(rename = "tool.record_stop")]
+ ToolRecordStop,
+ #[serde(rename = "tool.record_await")]
+ ToolRecordAwait,
#[serde(rename = "cancel")]
Cancel,
@@ -121,9 +127,16 @@ impl Method {
| Method::ToolFill
| Method::ToolPress
| Method::ToolSelect
- | Method::ToolEvaluate => true,
+ | Method::ToolEvaluate
+ // May navigate via optional `url` and changes Agent Window
+ // chrome; gate behind pending-interrupt like other writes.
+ | Method::ToolRecordStart => true,
// Read-only tool calls — transparent.
+ //
+ // `record_stop` / `record_await` observe / finish a recording
+ // without driving new automation gestures, so they stay
+ // ungated (teardown after interrupt must still work).
Method::ToolTabList
| Method::ToolSnapshot
| Method::ToolGetHtml
@@ -132,7 +145,9 @@ impl Method {
| Method::ToolNetwork
| Method::ToolWaitForNavigation
| Method::ToolWaitMs
- | Method::ToolRequestHelp => false,
+ | Method::ToolRequestHelp
+ | Method::ToolRecordStop
+ | Method::ToolRecordAwait => false,
// Session lifecycle — not gated.
Method::SessionStart
@@ -218,6 +233,13 @@ mod tests {
assert!(Method::ToolPress.is_mutating());
assert!(Method::ToolSelect.is_mutating());
assert!(Method::ToolEvaluate.is_mutating());
+ assert!(Method::ToolRecordStart.is_mutating());
+ }
+
+ #[test]
+ fn is_mutating_classifies_record_stop_await_as_non_mutating() {
+ assert!(!Method::ToolRecordStop.is_mutating());
+ assert!(!Method::ToolRecordAwait.is_mutating());
}
#[test]
diff --git a/crates/bsk-protocol/src/tools/mod.rs b/crates/bsk-protocol/src/tools/mod.rs
index 1103cd0..8a0c319 100644
--- a/crates/bsk-protocol/src/tools/mod.rs
+++ b/crates/bsk-protocol/src/tools/mod.rs
@@ -1,4 +1,4 @@
-//! Typed params/results for the 21 `tool.*` RPC methods (§7).
+//! Typed params/results for the `tool.*` RPC methods (§7).
pub mod console;
pub mod dialog;
@@ -7,6 +7,7 @@ pub mod interaction;
pub mod navigation;
pub mod network;
pub mod observation;
+pub mod record;
pub mod script;
pub mod session;
pub mod tabs;
@@ -19,6 +20,7 @@ pub use interaction::*;
pub use navigation::*;
pub use network::*;
pub use observation::*;
+pub use record::*;
pub use script::*;
pub use session::*;
pub use tabs::*;
diff --git a/crates/bsk-protocol/src/tools/record.rs b/crates/bsk-protocol/src/tools/record.rs
new file mode 100644
index 0000000..21ee1e6
--- /dev/null
+++ b/crates/bsk-protocol/src/tools/record.rs
@@ -0,0 +1,458 @@
+//! Semantic user-action recording (`tool.record_start` / `stop` / `await`).
+//!
+//! Trace v2 is a **record-only** log of user actions: what was clicked, filled,
+//! selected, and where navigation occurred. No LLM runs during recording, so
+//! variable-vs-constant classification is **not** stored — executing agents
+//! infer that at run time from raw values and control names.
+
+use schemars::JsonSchema;
+use serde::{Deserialize, Serialize};
+
+use super::interaction::KeyModifier;
+
+// ---------------------------------------------------------------------------
+// Target
+// ---------------------------------------------------------------------------
+
+/// Stable semantic handle for an interacted element.
+///
+/// `name` and `nearby_label` are **untrusted page text**.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+pub struct TargetDescriptor {
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub role: Option,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub name: Option,
+ pub tag: String,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub name_attr: Option,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub placeholder: Option,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub nearby_label: Option,
+}
+
+// ---------------------------------------------------------------------------
+// Trace envelope
+// ---------------------------------------------------------------------------
+
+/// Recording entry point — first URL the flow starts from.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+pub struct TraceEntry {
+ pub start_url: String,
+}
+
+/// Page context dictionary entry — referenced by steps via `page` id.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+pub struct PageRef {
+ pub id: String,
+ pub url: String,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub title: Option,
+}
+
+/// One selected option (`select` op).
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+pub struct SelectedOption {
+ pub value: String,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub label: Option,
+}
+
+/// Observed navigation after a step (objective fact only).
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+pub struct StepEffect {
+ /// Reference into `pages[]` for the destination page.
+ pub navigated_to: String,
+}
+
+/// Fields shared by every step variant (flattened in JSON).
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+pub struct StepCommon {
+ pub id: u32,
+ /// Reference into `pages[]`.
+ pub page: String,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub effect: Option,
+}
+
+// ---------------------------------------------------------------------------
+// Step op-specific payloads
+// ---------------------------------------------------------------------------
+
+/// One recorded user action — discriminated union by `op`.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+#[serde(tag = "op", rename_all = "snake_case")]
+pub enum Step {
+ Navigate {
+ #[serde(flatten)]
+ common: StepCommon,
+ to: String,
+ },
+ Click {
+ #[serde(flatten)]
+ common: StepCommon,
+ target: TargetDescriptor,
+ },
+ Fill {
+ #[serde(flatten)]
+ common: StepCommon,
+ target: TargetDescriptor,
+ value: String,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ redacted: Option,
+ },
+ Select {
+ #[serde(flatten)]
+ common: StepCommon,
+ target: TargetDescriptor,
+ selection: Vec,
+ },
+ Press {
+ #[serde(flatten)]
+ common: StepCommon,
+ key: String,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ modifiers: Option>,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ target: Option,
+ },
+}
+
+// ---------------------------------------------------------------------------
+// Trace root
+// ---------------------------------------------------------------------------
+
+/// Persisted user-action trace exported by `tool.record_stop` / `await`.
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+pub struct Trace {
+ /// RFC 3339 timestamp when recording stopped.
+ pub recorded_at: String,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub started_at: Option,
+ /// Optional user-provided goal from `--purpose` (metadata only).
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub purpose: Option,
+ pub entry: TraceEntry,
+ pub pages: Vec,
+ pub steps: Vec,
+}
+
+// ---------------------------------------------------------------------------
+// RPC params / results
+// ---------------------------------------------------------------------------
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+pub struct RecordStartParams {
+ pub session_id: String,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub tab_id: Option,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub url: Option,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub purpose: Option,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+pub struct RecordStartResult {
+ pub tab_id: i64,
+ pub recording: bool,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+pub struct RecordStopParams {
+ pub session_id: String,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+pub struct RecordStopResult {
+ pub trace: Trace,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+pub struct RecordAwaitParams {
+ pub session_id: String,
+ #[serde(default, skip_serializing_if = "Option::is_none")]
+ pub timeout_ms: Option,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
+pub struct RecordAwaitResult {
+ pub trace: Trace,
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use serde_json::json;
+
+ fn sample_common(id: u32) -> StepCommon {
+ StepCommon {
+ id,
+ page: "p1".into(),
+ effect: None,
+ }
+ }
+
+ fn sample_target() -> TargetDescriptor {
+ TargetDescriptor {
+ role: Some("button".into()),
+ name: Some("发布".into()),
+ tag: "button".into(),
+ name_attr: None,
+ placeholder: None,
+ nearby_label: None,
+ }
+ }
+
+ fn sample_trace() -> Trace {
+ Trace {
+ recorded_at: "2026-07-17T09:01:10Z".into(),
+ started_at: Some("2026-07-17T09:00:00Z".into()),
+ purpose: Some("发布一篇文章".into()),
+ entry: TraceEntry {
+ start_url: "https://x.com/editor".into(),
+ },
+ pages: vec![
+ PageRef {
+ id: "p1".into(),
+ url: "https://x.com/editor".into(),
+ title: Some("写文章".into()),
+ },
+ PageRef {
+ id: "p2".into(),
+ url: "https://x.com/p/99".into(),
+ title: None,
+ },
+ ],
+ steps: vec![
+ Step::Fill {
+ common: sample_common(1),
+ target: TargetDescriptor {
+ role: Some("textbox".into()),
+ name: Some("标题".into()),
+ tag: "input".into(),
+ name_attr: None,
+ placeholder: None,
+ nearby_label: None,
+ },
+ value: "我的第一篇文章".into(),
+ redacted: None,
+ },
+ Step::Select {
+ common: sample_common(3),
+ target: TargetDescriptor {
+ role: Some("combobox".into()),
+ name: Some("分类".into()),
+ tag: "select".into(),
+ name_attr: None,
+ placeholder: None,
+ nearby_label: None,
+ },
+ selection: vec![SelectedOption {
+ value: "tech".into(),
+ label: Some("技术分享".into()),
+ }],
+ },
+ Step::Click {
+ common: StepCommon {
+ effect: Some(StepEffect {
+ navigated_to: "p2".into(),
+ }),
+ ..sample_common(4)
+ },
+ target: sample_target(),
+ },
+ ],
+ }
+ }
+
+ #[test]
+ fn step_click_round_trips() {
+ let step = Step::Click {
+ common: sample_common(1),
+ target: sample_target(),
+ };
+ let v = serde_json::to_value(&step).unwrap();
+ assert_eq!(v.get("op").and_then(|v| v.as_str()), Some("click"));
+ assert_eq!(v.get("page").and_then(|v| v.as_str()), Some("p1"));
+ assert!(v.get("key").is_none());
+ let round: Step = serde_json::from_value(v).unwrap();
+ assert_eq!(round, step);
+ }
+
+ #[test]
+ fn step_fill_with_raw_value_round_trips() {
+ let step = Step::Fill {
+ common: sample_common(2),
+ target: TargetDescriptor {
+ role: Some("textbox".into()),
+ name: Some("服务名称".into()),
+ tag: "input".into(),
+ name_attr: Some("serviceName".into()),
+ placeholder: None,
+ nearby_label: None,
+ },
+ value: "my-svc".into(),
+ redacted: None,
+ };
+ let v = serde_json::to_value(&step).unwrap();
+ assert_eq!(v["op"], "fill");
+ assert_eq!(v["value"], "my-svc");
+ let round: Step = serde_json::from_value(v).unwrap();
+ assert_eq!(round, step);
+ }
+
+ #[test]
+ fn step_fill_password_is_redacted() {
+ let step = Step::Fill {
+ common: sample_common(1),
+ target: TargetDescriptor {
+ role: Some("textbox".into()),
+ name: Some("密码".into()),
+ tag: "input".into(),
+ name_attr: None,
+ placeholder: None,
+ nearby_label: None,
+ },
+ value: "***".into(),
+ redacted: Some(true),
+ };
+ let v = serde_json::to_value(&step).unwrap();
+ assert_eq!(v["value"], "***");
+ assert_eq!(v["redacted"], true);
+ }
+
+ #[test]
+ fn step_select_uses_object_array() {
+ let step = Step::Select {
+ common: sample_common(3),
+ target: TargetDescriptor {
+ role: Some("combobox".into()),
+ name: Some("分类".into()),
+ tag: "select".into(),
+ name_attr: None,
+ placeholder: None,
+ nearby_label: None,
+ },
+ selection: vec![SelectedOption {
+ value: "tech".into(),
+ label: Some("技术分享".into()),
+ }],
+ };
+ let v = serde_json::to_value(&step).unwrap();
+ assert_eq!(v["selection"][0]["value"], "tech");
+ assert_eq!(v["selection"][0]["label"], "技术分享");
+ let round: Step = serde_json::from_value(v).unwrap();
+ assert_eq!(round, step);
+ }
+
+ #[test]
+ fn step_navigate_round_trips() {
+ let step = Step::Navigate {
+ common: sample_common(1),
+ to: "https://example.com".into(),
+ };
+ let v = serde_json::to_value(&step).unwrap();
+ assert_eq!(v["op"], "navigate");
+ assert_eq!(v["to"], "https://example.com");
+ let round: Step = serde_json::from_value(v).unwrap();
+ assert_eq!(round, step);
+ }
+
+ #[test]
+ fn step_press_with_modifiers_round_trips() {
+ let step = Step::Press {
+ common: StepCommon {
+ effect: Some(StepEffect {
+ navigated_to: "p2".into(),
+ }),
+ ..sample_common(2)
+ },
+ key: "Enter".into(),
+ modifiers: Some(vec![KeyModifier::Ctrl, KeyModifier::Shift]),
+ target: Some(sample_target()),
+ };
+ let v = serde_json::to_value(&step).unwrap();
+ assert_eq!(v["key"], "Enter");
+ assert_eq!(v["modifiers"], json!(["ctrl", "shift"]));
+ assert_eq!(v["effect"]["navigated_to"], "p2");
+ let round: Step = serde_json::from_value(v).unwrap();
+ assert_eq!(round, step);
+ }
+
+ #[test]
+ fn trace_record_only_round_trips() {
+ let trace = sample_trace();
+ let v = serde_json::to_value(&trace).unwrap();
+ assert!(v.get("version").is_none());
+ assert_eq!(
+ v.get("purpose").and_then(|v| v.as_str()),
+ Some("发布一篇文章")
+ );
+ assert!(v.get("parameters").is_none());
+ assert!(v.get("site").is_none());
+ assert!(v.get("goal").is_none());
+ assert_eq!(
+ v.get("entry")
+ .and_then(|e| e.get("start_url"))
+ .and_then(|u| u.as_str()),
+ Some("https://x.com/editor")
+ );
+ let round: Trace = serde_json::from_value(v).unwrap();
+ assert_eq!(round, trace);
+ }
+
+ #[test]
+ fn discriminated_union_click_ignores_extra_key_field() {
+ let bad = json!({
+ "op": "click",
+ "id": 1,
+ "page": "p1",
+ "target": { "tag": "button", "name": "OK" },
+ "key": "Enter"
+ });
+ let step: Step = serde_json::from_value(bad).unwrap();
+ assert!(matches!(step, Step::Click { .. }));
+ let v = serde_json::to_value(&step).unwrap();
+ assert!(v.get("key").is_none());
+ }
+
+ #[test]
+ fn extension_trace_deserializes() {
+ let v = json!({
+ "recorded_at": "2026-07-21T08:00:00Z",
+ "started_at": "2026-07-21T07:59:00Z",
+ "purpose": "demo",
+ "entry": { "start_url": "https://example.com/editor" },
+ "pages": [
+ { "id": "p1", "url": "https://example.com/editor" },
+ { "id": "p2", "url": "https://example.com/p/99" }
+ ],
+ "steps": [
+ {
+ "op": "fill",
+ "id": 1,
+ "page": "p1",
+ "target": { "tag": "input", "role": "textbox", "name": "标题" },
+ "value": "hello"
+ },
+ {
+ "op": "click",
+ "id": 2,
+ "page": "p1",
+ "target": { "tag": "button", "role": "button", "name": "发布" },
+ "effect": { "navigated_to": "p2" }
+ }
+ ]
+ });
+ let trace: Trace = serde_json::from_value(v).unwrap();
+ assert_eq!(trace.entry.start_url, "https://example.com/editor");
+ assert_eq!(trace.pages.len(), 2);
+ assert_eq!(trace.steps.len(), 2);
+ }
+}
diff --git a/packages/i18n/src/locales/en-US/extension.json b/packages/i18n/src/locales/en-US/extension.json
index aa275ae..c597f9a 100644
--- a/packages/i18n/src/locales/en-US/extension.json
+++ b/packages/i18n/src/locales/en-US/extension.json
@@ -25,9 +25,10 @@
"connectionToggleTitle": "BrowserSkill connection",
"versionSkewWarning": "Extension protocol v{{extensionProtocol}}, daemon v{{daemonProtocol}}.",
"upgradeAvailable": "Upgradable",
- "labelTitle": "Browser alias",
- "labelDescription": "Distinguish multiple browser instances.",
- "labelPlaceholder": "e.g. Personal Chrome",
+ "launcher": {
+ "title": "Quick actions"
+ },
+ "back": "Back",
"runtimeTitle": "Runtime",
"daemonVersion": "daemon v{{version}}",
"extensionVersion": "extension v{{version}}",
@@ -40,7 +41,20 @@
"instanceDescription": "Unique connection identifier for this browser",
"copy": "Copy",
"copied": "Copied",
- "copyInstanceId": "Copy instance ID"
+ "copyInstanceId": "Copy instance ID",
+ "record": {
+ "sectionTitle": "Action recording",
+ "cardDesc": "Record your actions for Agent reference",
+ "sectionHint": "Generates a recording instruction — send it to your Agent to start recording your actions; the Agent can then use the recording to automate similar tasks.",
+ "topicLabel": "Topic (optional)",
+ "topicPlaceholder": "e.g. Publish an article",
+ "startUrlLabel": "Starting URL (optional)",
+ "startUrlPlaceholder": "https://…",
+ "copyButton": "Copy recording instructions",
+ "copied": "Copied",
+ "hintDisconnected": "Available when connected",
+ "promptTemplate": "Use BrowserSkill's `bsk` CLI to record the browser actions I'm about to demonstrate.\n\nSteps:\n1. Run: {{command}}\n2. Once you confirm it's started, I'll operate in the opened window myself.\n3. After I click Finish, read the returned trace.json and summarize the actions (pages + steps); don't re-run or \"verify\"."
+ }
},
"controlOverlay": {
"status": "Agent controlling",
@@ -68,5 +82,9 @@
"expand": "Expand",
"dragHandle": "Drag to move",
"notificationTitle": "BrowserSkill: Agent needs your help"
+ },
+ "recordOverlay": {
+ "recording": "Recording your actions",
+ "finish": "Finish"
}
}
diff --git a/packages/i18n/src/locales/zh-CN/extension.json b/packages/i18n/src/locales/zh-CN/extension.json
index b9e6415..18e85c6 100644
--- a/packages/i18n/src/locales/zh-CN/extension.json
+++ b/packages/i18n/src/locales/zh-CN/extension.json
@@ -25,9 +25,10 @@
"connectionToggleTitle": "BrowserSkill 连接",
"versionSkewWarning": "扩展协议 v{{extensionProtocol}},daemon 协议 v{{daemonProtocol}}。",
"upgradeAvailable": "可升级",
- "labelTitle": "浏览器别名",
- "labelDescription": "用于区分多个浏览器实例。",
- "labelPlaceholder": "例如:个人 Chrome",
+ "launcher": {
+ "title": "快捷功能"
+ },
+ "back": "返回",
"runtimeTitle": "运行版本",
"daemonVersion": "daemon v{{version}}",
"extensionVersion": "扩展 v{{version}}",
@@ -40,7 +41,20 @@
"instanceDescription": "本浏览器的唯一连接标识",
"copy": "复制",
"copied": "已复制",
- "copyInstanceId": "复制实例 ID"
+ "copyInstanceId": "复制实例 ID",
+ "record": {
+ "sectionTitle": "操作录制",
+ "cardDesc": "录制你的操作,供 Agent 参考",
+ "sectionHint": "生成一段录制指令,发给 Agent 后可以启动插件录制用户操作,之后 Agent 可以参考录制结果自动完成同类任务。",
+ "topicLabel": "操作主题(可选)",
+ "topicPlaceholder": "例如:发布一篇文章",
+ "startUrlLabel": "起始网址(可选)",
+ "startUrlPlaceholder": "https://…",
+ "copyButton": "复制录制指令",
+ "copied": "已复制",
+ "hintDisconnected": "连接后可用",
+ "promptTemplate": "用 BrowserSkill 的 `bsk` CLI 录制我接下来的浏览器操作。\n\n步骤:\n1. 执行:{{command}}\n2. 你确认已开始后,我会在打开的窗口里自己操作。\n3. 我点“结束”后,读取返回的 trace.json 并总结操作(pages + steps),不要重跑或“验证”。"
+ }
},
"controlOverlay": {
"status": "Agent 正在控制",
@@ -68,5 +82,9 @@
"expand": "展开",
"dragHandle": "拖动以移动",
"notificationTitle": "BrowserSkill:Agent 需要你的帮助"
+ },
+ "recordOverlay": {
+ "recording": "正在录制用户操作",
+ "finish": "结束"
}
}
diff --git a/skill/SKILL.md b/skill/SKILL.md
index 5edf9ea..ab32ff7 100644
--- a/skill/SKILL.md
+++ b/skill/SKILL.md
@@ -46,6 +46,21 @@ Optional: `bsk session start --browser ` when multiple bro
Emergency cleanup: `bsk session stop --all` or the Agent Window overlay **Stop all**.
+## Stop when the goal is met
+
+Every task is a **bounded goal**, not open-ended browsing. The goal may come from the user's request, a recorded `trace.json`, or both.
+
+1. **Define success first** — one concrete, observable condition derived from the user's words, `purpose`, or the last meaningful step in a trace (e.g. "form submitted", "item added to cart", "playback started").
+2. **Take the shortest path** — snapshot → act → at most one check. Do not wander, re-try unrelated actions, or stack exploratory steps.
+3. **Stop as soon as success is reached** — run `bsk session stop ` immediately unless the user explicitly asked to keep the session open (e.g. "don't close yet", "keep browsing").
+4. **No post-success work** — once the goal is met, do not click, refresh, navigate, re-search, switch tabs, or "double-check" that it worked. Further verification is a new task.
+5. **When blocked, pause — do not brute-force** — if the page requires human input (login, captcha, OTP, payment confirmation) or an action fails twice with no progress, call `bsk request-help` instead of retrying blindly. See **Ask the human for help** below.
+6. **When unsure** — at most one extra `bsk snapshot`. If success looks met, stop. If not, ask the user; do not keep clicking.
+
+**With a trace:** replay steps in order using `target` role/name/tag and raw `value`/`selection` fields. After the last step (or when its `effect.navigated_to` / success hint is satisfied), apply rules 3–4 immediately. The trace guides execution; it does not extend control beyond the goal.
+
+**Without a trace:** the user's request *is* the success condition. Satisfying it ends the task — same stop rules apply.
+
## Core interaction loop
Write operations only affect tabs in the **Agent Window** (or tabs you **borrowed** into it).
@@ -191,6 +206,23 @@ acts. The result `outcome` is one of:
`note` carries any text the user typed back. `resolved_targets` reports
which refs/selectors matched a live element.
+### Recording — `bsk record`
+
+Capture the user's own actions in the Agent Window to a `trace.json`, for later LLM-driven automation:
+
+```bash
+bsk record start --browser [--url https://…] [--purpose "publish a wiki doc"] [--output trace.json]
+# `--url` is optional; default https://example.com/ when omitted (must be http(s)).
+# Blocks until the user clicks Finish in the recording panel, then writes ./trace.json and closes the window.
+
+bsk record stop [--output trace.json] # terminal fallback if the browser panel is unavailable
+```
+
+- The trace is a **record-only action log** (a `pages[]` dictionary + `navigate`/`click`/`fill`/`select`/`press` steps with `target` descriptors). It records *what the user did*; deciding which inputs are variable is left to the executing agent.
+- `--purpose` is optional context metadata; it does **not** change what gets captured.
+- There is **no** `bsk replay` — to redo a flow, read the trace and reuse the existing `session` / `snapshot` / `@eN` / `click` / `fill` tools. Follow **Stop when the goal is met**.
+- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
+
## Error handling
### Exit codes (`echo $?` after `bsk …`)
@@ -221,8 +253,9 @@ Always **`bsk session stop `** in a `finally`-style path so the Agent Window
1. **No token theft** — do not `bsk evaluate` on sensitive sites to read `localStorage`, cookies, or auth headers for exfiltration.
2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks.
3. **No skip stop** — always `bsk session stop `; never assume idle timeout will clean up.
-4. **No observe escalation before snapshot** — use `bsk snapshot` first; only use `bsk get-html` or `bsk screenshot` when the snapshot is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot ref — never skip snapshot just to grab a visual.
-5. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
+4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open.
+5. **No observe escalation before snapshot** — use `bsk snapshot` first; only use `bsk get-html` or `bsk screenshot` when the snapshot is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot ref — never skip snapshot just to grab a visual.
+6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
---