mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat: add bsk record for semantic user-action traces
Capture user actions in the Agent Window and export a record-only trace (pages[] + navigate/click/fill/select/press steps with target descriptors). Adds the record overlay UI aligned with the control overlay, and wires record start/stop/await through CLI, daemon, protocol, and extension.
This commit is contained in:
@@ -7,7 +7,7 @@ describe("ControlOverlay", () => {
|
||||
cleanup();
|
||||
});
|
||||
|
||||
it("sets pointer-events none on blocker and pill when automationBypass is true", () => {
|
||||
it("keeps page blocker none under automationBypass but Interrupt stays clickable", () => {
|
||||
const { container } = render(
|
||||
<ControlOverlay
|
||||
visible={true}
|
||||
@@ -17,13 +17,17 @@ describe("ControlOverlay", () => {
|
||||
/>,
|
||||
);
|
||||
|
||||
const blocker = container.querySelector("[data-slot='control-overlay']")?.nextElementSibling;
|
||||
const blocker = container.querySelector("[data-slot='control-overlay-blocker']");
|
||||
expect(blocker).toBeTruthy();
|
||||
expect((blocker as HTMLElement).style.pointerEvents).toBe("none");
|
||||
|
||||
const pill = container.querySelector("[data-slot='control-overlay-pill']");
|
||||
expect(pill).toBeTruthy();
|
||||
expect((pill as HTMLElement).style.pointerEvents).toBe("auto");
|
||||
|
||||
const stopBtn = container.querySelector("[data-slot='control-overlay-stop-all']");
|
||||
expect(stopBtn).toBeTruthy();
|
||||
expect((stopBtn as HTMLElement).style.pointerEvents).toBe("none");
|
||||
expect((stopBtn as HTMLElement).style.pointerEvents).toBe("auto");
|
||||
});
|
||||
|
||||
it("uses pointer-events auto on blocker when automationBypass is false", () => {
|
||||
@@ -36,7 +40,7 @@ describe("ControlOverlay", () => {
|
||||
/>,
|
||||
);
|
||||
|
||||
const blocker = container.querySelector("[data-slot='control-overlay']")?.nextElementSibling;
|
||||
const blocker = container.querySelector("[data-slot='control-overlay-blocker']");
|
||||
expect(blocker).toBeTruthy();
|
||||
expect((blocker as HTMLElement).style.pointerEvents).toBe("auto");
|
||||
});
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import { cleanup, fireEvent, render } from "@testing-library/react";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { RecordOverlay } from "../RecordOverlay";
|
||||
|
||||
describe("RecordOverlay", () => {
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
});
|
||||
|
||||
it("renders the bottom pill with orange finish CTA and no logo", () => {
|
||||
const onFinish = vi.fn();
|
||||
const { container } = render(<RecordOverlay request={{ id: "rec-1", onFinish }} />);
|
||||
|
||||
const pill = container.querySelector("[data-slot='record-overlay-pill']");
|
||||
expect(pill).toBeTruthy();
|
||||
expect(pill?.querySelector("img")).toBeNull();
|
||||
expect((pill as HTMLElement).style.borderRadius).toBe("9999px");
|
||||
expect((pill as HTMLElement).style.backgroundColor).toBe("#fff");
|
||||
|
||||
const finish = container.querySelector("[data-slot='record-overlay-finish']");
|
||||
expect(finish).toBeTruthy();
|
||||
expect((finish as HTMLElement).style.backgroundColor).toBe("#f97316");
|
||||
expect((finish as HTMLElement).style.borderRadius).toBe("9999px");
|
||||
|
||||
fireEvent.click(finish!);
|
||||
expect(onFinish).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("shows a pulsing recording indicator and no full-screen glow layer", () => {
|
||||
const { container } = render(<RecordOverlay request={{ id: "rec-1", onFinish: vi.fn() }} />);
|
||||
expect(container.querySelector("[data-slot='record-overlay']")).toBeNull();
|
||||
|
||||
const indicator = container.querySelector("[data-slot='record-overlay-indicator']");
|
||||
expect(indicator).toBeTruthy();
|
||||
expect((indicator as HTMLElement).style.animation).toContain("bsk-rec-pulse");
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { OverlayController } from "../overlay-controller";
|
||||
import { OverlayController, shouldShowAgentControlOverlay } from "../overlay-controller";
|
||||
|
||||
describe("OverlayController", () => {
|
||||
it("resets agent overlays without clearing user-tab borrow requests", () => {
|
||||
@@ -31,6 +31,7 @@ describe("OverlayController", () => {
|
||||
expect(state.controlVisible).toBe(false);
|
||||
expect(state.activeSessionId).toBeNull();
|
||||
expect(state.activeHelp).toBeNull();
|
||||
expect(state.activeRecord).toBeNull();
|
||||
expect(state.automationBypassCount).toBe(0);
|
||||
});
|
||||
|
||||
@@ -69,4 +70,50 @@ describe("OverlayController", () => {
|
||||
expect(previous?.id).toBe("help-1");
|
||||
expect(controller.snapshot().activeHelp?.id).toBe("help-2");
|
||||
});
|
||||
|
||||
it("tracks active record request state", () => {
|
||||
const controller = new OverlayController();
|
||||
|
||||
controller.activateAgentSession("sess-1");
|
||||
controller.setAgentRecordRequest({
|
||||
id: "rec-1",
|
||||
onFinish: vi.fn(),
|
||||
});
|
||||
|
||||
expect(controller.snapshot().activeRecord?.id).toBe("rec-1");
|
||||
});
|
||||
|
||||
it("hides agent control overlay while recording is active", () => {
|
||||
const controller = new OverlayController();
|
||||
controller.activateAgentSession("sess-1");
|
||||
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(true);
|
||||
|
||||
controller.setAgentRecordRequest({
|
||||
id: "rec-1",
|
||||
onFinish: vi.fn(),
|
||||
});
|
||||
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(false);
|
||||
expect(controller.snapshot().controlVisible).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps control hidden after record ends until session overlays reset", () => {
|
||||
const controller = new OverlayController();
|
||||
controller.activateAgentSession("sess-1");
|
||||
controller.setAgentRecordRequest({
|
||||
id: "rec-1",
|
||||
onFinish: vi.fn(),
|
||||
});
|
||||
controller.setAutomationBypass(true);
|
||||
controller.setAutomationBypass(true);
|
||||
|
||||
controller.clearAgentRecordRequest("rec-1");
|
||||
expect(controller.snapshot().activeRecord).toBeNull();
|
||||
expect(controller.snapshot().suppressControlAfterRecord).toBe(true);
|
||||
expect(controller.snapshot().automationBypassCount).toBe(0);
|
||||
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(false);
|
||||
|
||||
controller.resetAgentOverlays("sess-1");
|
||||
expect(controller.snapshot().suppressControlAfterRecord).toBe(false);
|
||||
expect(controller.snapshot().controlVisible).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { handleRecordContentMessage, startRecordCapture } from "../record-capture";
|
||||
import { RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
|
||||
|
||||
vi.stubGlobal("chrome", {
|
||||
runtime: {
|
||||
sendMessage: vi.fn(() => Promise.resolve()),
|
||||
},
|
||||
});
|
||||
|
||||
describe("handleRecordContentMessage stop/cancel", () => {
|
||||
it("ignores STOP when no recording is active", () => {
|
||||
const dispose = vi.fn();
|
||||
const onStop = vi.fn();
|
||||
const setActiveRequestId = vi.fn();
|
||||
const setCapture = vi.fn();
|
||||
const sendResponse = vi.fn();
|
||||
|
||||
const needsAsync = handleRecordContentMessage(
|
||||
{ type: RECORD_STOP, requestId: "rec-stale" },
|
||||
{
|
||||
activeRequestId: null,
|
||||
capture: { dispose },
|
||||
setActiveRequestId,
|
||||
setCapture,
|
||||
onStart: vi.fn(),
|
||||
onStop,
|
||||
},
|
||||
sendResponse,
|
||||
);
|
||||
|
||||
expect(needsAsync).toBe(false);
|
||||
expect(dispose).not.toHaveBeenCalled();
|
||||
expect(onStop).not.toHaveBeenCalled();
|
||||
expect(setActiveRequestId).not.toHaveBeenCalled();
|
||||
expect(setCapture).not.toHaveBeenCalled();
|
||||
expect(sendResponse).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("ignores STOP for a mismatched requestId", () => {
|
||||
const dispose = vi.fn();
|
||||
const onStop = vi.fn();
|
||||
|
||||
const needsAsync = handleRecordContentMessage(
|
||||
{ type: RECORD_STOP, requestId: "rec-other" },
|
||||
{
|
||||
activeRequestId: "rec-1",
|
||||
capture: { dispose },
|
||||
setActiveRequestId: vi.fn(),
|
||||
setCapture: vi.fn(),
|
||||
onStart: vi.fn(),
|
||||
onStop,
|
||||
},
|
||||
vi.fn(),
|
||||
);
|
||||
|
||||
expect(needsAsync).toBe(false);
|
||||
expect(dispose).not.toHaveBeenCalled();
|
||||
expect(onStop).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("record-capture semantic", () => {
|
||||
let steps: RecordStepPayload[];
|
||||
|
||||
beforeEach(() => {
|
||||
steps = [];
|
||||
document.body.innerHTML = `
|
||||
<label for="q">查询</label>
|
||||
<input id="q" name="q" />
|
||||
<button type="button" aria-label="搜索">搜索</button>
|
||||
<div role="listbox" id="sug">
|
||||
<div role="option">建议项</div>
|
||||
</div>
|
||||
`;
|
||||
});
|
||||
|
||||
it("commits final fill value and records semantic click", () => {
|
||||
const capture = startRecordCapture("rec-1", (step) => steps.push(step));
|
||||
const input = document.querySelector("input")!;
|
||||
const button = document.querySelector("button")!;
|
||||
|
||||
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||
input.value = "hello";
|
||||
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
input.dispatchEvent(new FocusEvent("focusout", { bubbles: true }));
|
||||
|
||||
button.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
||||
|
||||
capture.dispose();
|
||||
|
||||
expect(steps).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
op: "fill",
|
||||
value: "hello",
|
||||
target: expect.objectContaining({ name: "查询", tag: "input" }),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
op: "click",
|
||||
target: expect.objectContaining({ name: "搜索", role: "button" }),
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(JSON.stringify(steps)).not.toMatch(/@e\d+/);
|
||||
expect(steps.some((s) => "selector" in s)).toBe(false);
|
||||
});
|
||||
|
||||
it("ignores autocomplete suggestion clicks while a fill session is open", () => {
|
||||
const capture = startRecordCapture("rec-2", (step) => steps.push(step));
|
||||
const input = document.querySelector("input")!;
|
||||
const option = document.querySelector('[role="option"]')!;
|
||||
|
||||
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||
input.value = "hel";
|
||||
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
option.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
||||
input.value = "hello";
|
||||
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
input.dispatchEvent(new FocusEvent("focusout", { bubbles: true }));
|
||||
capture.dispose();
|
||||
|
||||
expect(steps.filter((s) => s.op === "click")).toHaveLength(0);
|
||||
expect(steps.some((s) => s.op === "fill" && s.value === "hello")).toBe(true);
|
||||
});
|
||||
|
||||
it("records Enter press but not bare typing keys", () => {
|
||||
const capture = startRecordCapture("rec-press", (step) => steps.push(step));
|
||||
const input = document.querySelector("input")!;
|
||||
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||
input.dispatchEvent(
|
||||
new KeyboardEvent("keydown", { key: "a", bubbles: true }),
|
||||
);
|
||||
input.dispatchEvent(
|
||||
new KeyboardEvent("keydown", { key: "Enter", bubbles: true }),
|
||||
);
|
||||
capture.dispose();
|
||||
expect(steps.filter((s) => s.op === "press")).toEqual([
|
||||
expect.objectContaining({ op: "press", key: "Enter" }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not record clicks on anonymous layout divs", () => {
|
||||
document.body.innerHTML = `
|
||||
<div id="chrome">page chrome</div>
|
||||
<button type="button" aria-label="下一步">下一步</button>
|
||||
`;
|
||||
const capture = startRecordCapture("rec-3", (step) => steps.push(step));
|
||||
document
|
||||
.querySelector("#chrome")!
|
||||
.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
||||
document
|
||||
.querySelector("button")!
|
||||
.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
||||
capture.dispose();
|
||||
|
||||
expect(steps).toHaveLength(1);
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "click",
|
||||
target: { name: "下一步", role: "button" },
|
||||
});
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user