feat: add bsk record for semantic user-action traces

Capture user actions in the Agent Window and export a record-only
trace (pages[] + navigate/click/fill/select/press steps with target
descriptors). Adds the record overlay UI aligned with the control
overlay, and wires record start/stop/await through CLI, daemon,
protocol, and extension.
This commit is contained in:
“haonan”
2026-07-22 11:27:30 +08:00
parent ccf3c32140
commit ac028a7a18
47 changed files with 6038 additions and 46 deletions
@@ -7,7 +7,7 @@ describe("ControlOverlay", () => {
cleanup();
});
it("sets pointer-events none on blocker and pill when automationBypass is true", () => {
it("keeps page blocker none under automationBypass but Interrupt stays clickable", () => {
const { container } = render(
<ControlOverlay
visible={true}
@@ -17,13 +17,17 @@ describe("ControlOverlay", () => {
/>,
);
const blocker = container.querySelector("[data-slot='control-overlay']")?.nextElementSibling;
const blocker = container.querySelector("[data-slot='control-overlay-blocker']");
expect(blocker).toBeTruthy();
expect((blocker as HTMLElement).style.pointerEvents).toBe("none");
const pill = container.querySelector("[data-slot='control-overlay-pill']");
expect(pill).toBeTruthy();
expect((pill as HTMLElement).style.pointerEvents).toBe("auto");
const stopBtn = container.querySelector("[data-slot='control-overlay-stop-all']");
expect(stopBtn).toBeTruthy();
expect((stopBtn as HTMLElement).style.pointerEvents).toBe("none");
expect((stopBtn as HTMLElement).style.pointerEvents).toBe("auto");
});
it("uses pointer-events auto on blocker when automationBypass is false", () => {
@@ -36,7 +40,7 @@ describe("ControlOverlay", () => {
/>,
);
const blocker = container.querySelector("[data-slot='control-overlay']")?.nextElementSibling;
const blocker = container.querySelector("[data-slot='control-overlay-blocker']");
expect(blocker).toBeTruthy();
expect((blocker as HTMLElement).style.pointerEvents).toBe("auto");
});
@@ -0,0 +1,37 @@
import { cleanup, fireEvent, render } from "@testing-library/react";
import { afterEach, describe, expect, it, vi } from "vitest";
import { RecordOverlay } from "../RecordOverlay";
describe("RecordOverlay", () => {
afterEach(() => {
cleanup();
});
it("renders the bottom pill with orange finish CTA and no logo", () => {
const onFinish = vi.fn();
const { container } = render(<RecordOverlay request={{ id: "rec-1", onFinish }} />);
const pill = container.querySelector("[data-slot='record-overlay-pill']");
expect(pill).toBeTruthy();
expect(pill?.querySelector("img")).toBeNull();
expect((pill as HTMLElement).style.borderRadius).toBe("9999px");
expect((pill as HTMLElement).style.backgroundColor).toBe("#fff");
const finish = container.querySelector("[data-slot='record-overlay-finish']");
expect(finish).toBeTruthy();
expect((finish as HTMLElement).style.backgroundColor).toBe("#f97316");
expect((finish as HTMLElement).style.borderRadius).toBe("9999px");
fireEvent.click(finish!);
expect(onFinish).toHaveBeenCalledTimes(1);
});
it("shows a pulsing recording indicator and no full-screen glow layer", () => {
const { container } = render(<RecordOverlay request={{ id: "rec-1", onFinish: vi.fn() }} />);
expect(container.querySelector("[data-slot='record-overlay']")).toBeNull();
const indicator = container.querySelector("[data-slot='record-overlay-indicator']");
expect(indicator).toBeTruthy();
expect((indicator as HTMLElement).style.animation).toContain("bsk-rec-pulse");
});
});
@@ -1,5 +1,5 @@
import { describe, expect, it, vi } from "vitest";
import { OverlayController } from "../overlay-controller";
import { OverlayController, shouldShowAgentControlOverlay } from "../overlay-controller";
describe("OverlayController", () => {
it("resets agent overlays without clearing user-tab borrow requests", () => {
@@ -31,6 +31,7 @@ describe("OverlayController", () => {
expect(state.controlVisible).toBe(false);
expect(state.activeSessionId).toBeNull();
expect(state.activeHelp).toBeNull();
expect(state.activeRecord).toBeNull();
expect(state.automationBypassCount).toBe(0);
});
@@ -69,4 +70,50 @@ describe("OverlayController", () => {
expect(previous?.id).toBe("help-1");
expect(controller.snapshot().activeHelp?.id).toBe("help-2");
});
it("tracks active record request state", () => {
const controller = new OverlayController();
controller.activateAgentSession("sess-1");
controller.setAgentRecordRequest({
id: "rec-1",
onFinish: vi.fn(),
});
expect(controller.snapshot().activeRecord?.id).toBe("rec-1");
});
it("hides agent control overlay while recording is active", () => {
const controller = new OverlayController();
controller.activateAgentSession("sess-1");
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(true);
controller.setAgentRecordRequest({
id: "rec-1",
onFinish: vi.fn(),
});
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(false);
expect(controller.snapshot().controlVisible).toBe(true);
});
it("keeps control hidden after record ends until session overlays reset", () => {
const controller = new OverlayController();
controller.activateAgentSession("sess-1");
controller.setAgentRecordRequest({
id: "rec-1",
onFinish: vi.fn(),
});
controller.setAutomationBypass(true);
controller.setAutomationBypass(true);
controller.clearAgentRecordRequest("rec-1");
expect(controller.snapshot().activeRecord).toBeNull();
expect(controller.snapshot().suppressControlAfterRecord).toBe(true);
expect(controller.snapshot().automationBypassCount).toBe(0);
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(false);
controller.resetAgentOverlays("sess-1");
expect(controller.snapshot().suppressControlAfterRecord).toBe(false);
expect(controller.snapshot().controlVisible).toBe(false);
});
});
@@ -0,0 +1,163 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { handleRecordContentMessage, startRecordCapture } from "../record-capture";
import { RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
vi.stubGlobal("chrome", {
runtime: {
sendMessage: vi.fn(() => Promise.resolve()),
},
});
describe("handleRecordContentMessage stop/cancel", () => {
it("ignores STOP when no recording is active", () => {
const dispose = vi.fn();
const onStop = vi.fn();
const setActiveRequestId = vi.fn();
const setCapture = vi.fn();
const sendResponse = vi.fn();
const needsAsync = handleRecordContentMessage(
{ type: RECORD_STOP, requestId: "rec-stale" },
{
activeRequestId: null,
capture: { dispose },
setActiveRequestId,
setCapture,
onStart: vi.fn(),
onStop,
},
sendResponse,
);
expect(needsAsync).toBe(false);
expect(dispose).not.toHaveBeenCalled();
expect(onStop).not.toHaveBeenCalled();
expect(setActiveRequestId).not.toHaveBeenCalled();
expect(setCapture).not.toHaveBeenCalled();
expect(sendResponse).not.toHaveBeenCalled();
});
it("ignores STOP for a mismatched requestId", () => {
const dispose = vi.fn();
const onStop = vi.fn();
const needsAsync = handleRecordContentMessage(
{ type: RECORD_STOP, requestId: "rec-other" },
{
activeRequestId: "rec-1",
capture: { dispose },
setActiveRequestId: vi.fn(),
setCapture: vi.fn(),
onStart: vi.fn(),
onStop,
},
vi.fn(),
);
expect(needsAsync).toBe(false);
expect(dispose).not.toHaveBeenCalled();
expect(onStop).not.toHaveBeenCalled();
});
});
describe("record-capture semantic", () => {
let steps: RecordStepPayload[];
beforeEach(() => {
steps = [];
document.body.innerHTML = `
<label for="q">查询</label>
<input id="q" name="q" />
<button type="button" aria-label="搜索">搜索</button>
<div role="listbox" id="sug">
<div role="option">建议项</div>
</div>
`;
});
it("commits final fill value and records semantic click", () => {
const capture = startRecordCapture("rec-1", (step) => steps.push(step));
const input = document.querySelector("input")!;
const button = document.querySelector("button")!;
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
input.value = "hello";
input.dispatchEvent(new Event("input", { bubbles: true }));
input.dispatchEvent(new FocusEvent("focusout", { bubbles: true }));
button.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
capture.dispose();
expect(steps).toEqual(
expect.arrayContaining([
expect.objectContaining({
op: "fill",
value: "hello",
target: expect.objectContaining({ name: "查询", tag: "input" }),
}),
expect.objectContaining({
op: "click",
target: expect.objectContaining({ name: "搜索", role: "button" }),
}),
]),
);
expect(JSON.stringify(steps)).not.toMatch(/@e\d+/);
expect(steps.some((s) => "selector" in s)).toBe(false);
});
it("ignores autocomplete suggestion clicks while a fill session is open", () => {
const capture = startRecordCapture("rec-2", (step) => steps.push(step));
const input = document.querySelector("input")!;
const option = document.querySelector('[role="option"]')!;
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
input.value = "hel";
input.dispatchEvent(new Event("input", { bubbles: true }));
option.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
input.value = "hello";
input.dispatchEvent(new Event("input", { bubbles: true }));
input.dispatchEvent(new FocusEvent("focusout", { bubbles: true }));
capture.dispose();
expect(steps.filter((s) => s.op === "click")).toHaveLength(0);
expect(steps.some((s) => s.op === "fill" && s.value === "hello")).toBe(true);
});
it("records Enter press but not bare typing keys", () => {
const capture = startRecordCapture("rec-press", (step) => steps.push(step));
const input = document.querySelector("input")!;
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
input.dispatchEvent(
new KeyboardEvent("keydown", { key: "a", bubbles: true }),
);
input.dispatchEvent(
new KeyboardEvent("keydown", { key: "Enter", bubbles: true }),
);
capture.dispose();
expect(steps.filter((s) => s.op === "press")).toEqual([
expect.objectContaining({ op: "press", key: "Enter" }),
]);
});
it("does not record clicks on anonymous layout divs", () => {
document.body.innerHTML = `
<div id="chrome">page chrome</div>
<button type="button" aria-label="下一步">下一步</button>
`;
const capture = startRecordCapture("rec-3", (step) => steps.push(step));
document
.querySelector("#chrome")!
.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
document
.querySelector("button")!
.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
capture.dispose();
expect(steps).toHaveLength(1);
expect(steps[0]).toMatchObject({
op: "click",
target: { name: "下一步", role: "button" },
});
});
});