mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat: add bsk record for semantic user-action traces
Capture user actions in the Agent Window and export a record-only trace (pages[] + navigate/click/fill/select/press steps with target descriptors). Adds the record overlay UI aligned with the control overlay, and wires record start/stop/await through CLI, daemon, protocol, and extension.
This commit is contained in:
“haonan”
@@ -58,6 +58,7 @@ export function ControlOverlay({
|
||||
/>
|
||||
|
||||
<div
|
||||
data-slot="control-overlay-blocker"
|
||||
onPointerDown={(event) => {
|
||||
if (automationBypass) return;
|
||||
event.preventDefault();
|
||||
@@ -80,13 +81,16 @@ export function ControlOverlay({
|
||||
/>
|
||||
|
||||
<div
|
||||
data-slot="control-overlay-pill"
|
||||
style={{
|
||||
position: "fixed",
|
||||
bottom: 32,
|
||||
left: "50%",
|
||||
transform: "translateX(-50%)",
|
||||
zIndex: 2147483647,
|
||||
pointerEvents,
|
||||
// Always receive clicks so Interrupt works even while the page
|
||||
// blocker is in automation-bypass mode (CDP pass-through).
|
||||
pointerEvents: "auto",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 12,
|
||||
@@ -122,7 +126,7 @@ export function ControlOverlay({
|
||||
disabled={interrupting}
|
||||
onClick={onInterrupt}
|
||||
style={{
|
||||
pointerEvents,
|
||||
pointerEvents: "auto",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 6,
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
import { useTranslation } from "@browser-skill/i18n/react";
|
||||
import { RiCheckboxCircleLine } from "@remixicon/react";
|
||||
import { useEffect, useState } from "react";
|
||||
|
||||
export interface RecordRequestData {
|
||||
id: string;
|
||||
onFinish: () => void;
|
||||
}
|
||||
|
||||
type Props = {
|
||||
request: RecordRequestData | null;
|
||||
};
|
||||
|
||||
export function RecordOverlay({ request }: Props) {
|
||||
const { t } = useTranslation("extension");
|
||||
const [show, setShow] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (request) {
|
||||
const raf = requestAnimationFrame(() => setShow(true));
|
||||
return () => cancelAnimationFrame(raf);
|
||||
}
|
||||
setShow(false);
|
||||
}, [request]);
|
||||
|
||||
if (!request) return null;
|
||||
|
||||
return (
|
||||
<>
|
||||
<style>{`
|
||||
@keyframes bsk-rec-pulse {
|
||||
0%, 100% { opacity: 1; transform: scale(1); }
|
||||
50% { opacity: 0.35; transform: scale(0.82); }
|
||||
}
|
||||
`}</style>
|
||||
|
||||
<div
|
||||
data-slot="record-overlay-pill"
|
||||
style={{
|
||||
position: "fixed",
|
||||
bottom: 32,
|
||||
left: "50%",
|
||||
zIndex: 2147483647,
|
||||
pointerEvents: "auto",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 10,
|
||||
backgroundColor: "#fff",
|
||||
borderRadius: 9999,
|
||||
padding: "10px 10px 10px 20px",
|
||||
boxShadow: "0 8px 32px rgba(15,23,42,0.16), 0 2px 8px rgba(0,0,0,0.1)",
|
||||
opacity: show ? 1 : 0,
|
||||
transform: show ? "translateX(-50%) translateY(0)" : "translateX(-50%) translateY(8px)",
|
||||
transition: "opacity 300ms ease-out, transform 300ms ease-out",
|
||||
fontFamily:
|
||||
'-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif',
|
||||
maxWidth: "min(420px, calc(100vw - 32px))",
|
||||
}}
|
||||
>
|
||||
<span
|
||||
data-slot="record-overlay-indicator"
|
||||
style={{
|
||||
width: 9,
|
||||
height: 9,
|
||||
borderRadius: "50%",
|
||||
backgroundColor: "#ef4444",
|
||||
flexShrink: 0,
|
||||
animation: "bsk-rec-pulse 1.4s ease-in-out infinite",
|
||||
}}
|
||||
/>
|
||||
<span
|
||||
style={{
|
||||
flex: 1,
|
||||
fontSize: 16,
|
||||
fontWeight: 500,
|
||||
color: "#333",
|
||||
whiteSpace: "nowrap",
|
||||
userSelect: "none",
|
||||
overflow: "hidden",
|
||||
textOverflow: "ellipsis",
|
||||
}}
|
||||
>
|
||||
{t("recordOverlay.recording")}
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
data-slot="record-overlay-finish"
|
||||
onClick={request.onFinish}
|
||||
style={{
|
||||
pointerEvents: "auto",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 6,
|
||||
border: "none",
|
||||
borderRadius: 9999,
|
||||
padding: "8px 20px 8px 16px",
|
||||
fontSize: 15,
|
||||
fontWeight: 600,
|
||||
color: "#fff",
|
||||
backgroundColor: "#f97316",
|
||||
cursor: "pointer",
|
||||
transition: "background-color 150ms ease-out, opacity 150ms ease-out",
|
||||
whiteSpace: "nowrap",
|
||||
lineHeight: 1,
|
||||
flexShrink: 0,
|
||||
}}
|
||||
>
|
||||
<RiCheckboxCircleLine size={18} color="#fff" />
|
||||
{t("recordOverlay.finish")}
|
||||
</button>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -7,7 +7,7 @@ describe("ControlOverlay", () => {
|
||||
cleanup();
|
||||
});
|
||||
|
||||
it("sets pointer-events none on blocker and pill when automationBypass is true", () => {
|
||||
it("keeps page blocker none under automationBypass but Interrupt stays clickable", () => {
|
||||
const { container } = render(
|
||||
<ControlOverlay
|
||||
visible={true}
|
||||
@@ -17,13 +17,17 @@ describe("ControlOverlay", () => {
|
||||
/>,
|
||||
);
|
||||
|
||||
const blocker = container.querySelector("[data-slot='control-overlay']")?.nextElementSibling;
|
||||
const blocker = container.querySelector("[data-slot='control-overlay-blocker']");
|
||||
expect(blocker).toBeTruthy();
|
||||
expect((blocker as HTMLElement).style.pointerEvents).toBe("none");
|
||||
|
||||
const pill = container.querySelector("[data-slot='control-overlay-pill']");
|
||||
expect(pill).toBeTruthy();
|
||||
expect((pill as HTMLElement).style.pointerEvents).toBe("auto");
|
||||
|
||||
const stopBtn = container.querySelector("[data-slot='control-overlay-stop-all']");
|
||||
expect(stopBtn).toBeTruthy();
|
||||
expect((stopBtn as HTMLElement).style.pointerEvents).toBe("none");
|
||||
expect((stopBtn as HTMLElement).style.pointerEvents).toBe("auto");
|
||||
});
|
||||
|
||||
it("uses pointer-events auto on blocker when automationBypass is false", () => {
|
||||
@@ -36,7 +40,7 @@ describe("ControlOverlay", () => {
|
||||
/>,
|
||||
);
|
||||
|
||||
const blocker = container.querySelector("[data-slot='control-overlay']")?.nextElementSibling;
|
||||
const blocker = container.querySelector("[data-slot='control-overlay-blocker']");
|
||||
expect(blocker).toBeTruthy();
|
||||
expect((blocker as HTMLElement).style.pointerEvents).toBe("auto");
|
||||
});
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import { cleanup, fireEvent, render } from "@testing-library/react";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { RecordOverlay } from "../RecordOverlay";
|
||||
|
||||
describe("RecordOverlay", () => {
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
});
|
||||
|
||||
it("renders the bottom pill with orange finish CTA and no logo", () => {
|
||||
const onFinish = vi.fn();
|
||||
const { container } = render(<RecordOverlay request={{ id: "rec-1", onFinish }} />);
|
||||
|
||||
const pill = container.querySelector("[data-slot='record-overlay-pill']");
|
||||
expect(pill).toBeTruthy();
|
||||
expect(pill?.querySelector("img")).toBeNull();
|
||||
expect((pill as HTMLElement).style.borderRadius).toBe("9999px");
|
||||
expect((pill as HTMLElement).style.backgroundColor).toBe("#fff");
|
||||
|
||||
const finish = container.querySelector("[data-slot='record-overlay-finish']");
|
||||
expect(finish).toBeTruthy();
|
||||
expect((finish as HTMLElement).style.backgroundColor).toBe("#f97316");
|
||||
expect((finish as HTMLElement).style.borderRadius).toBe("9999px");
|
||||
|
||||
fireEvent.click(finish!);
|
||||
expect(onFinish).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("shows a pulsing recording indicator and no full-screen glow layer", () => {
|
||||
const { container } = render(<RecordOverlay request={{ id: "rec-1", onFinish: vi.fn() }} />);
|
||||
expect(container.querySelector("[data-slot='record-overlay']")).toBeNull();
|
||||
|
||||
const indicator = container.querySelector("[data-slot='record-overlay-indicator']");
|
||||
expect(indicator).toBeTruthy();
|
||||
expect((indicator as HTMLElement).style.animation).toContain("bsk-rec-pulse");
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { OverlayController } from "../overlay-controller";
|
||||
import { OverlayController, shouldShowAgentControlOverlay } from "../overlay-controller";
|
||||
|
||||
describe("OverlayController", () => {
|
||||
it("resets agent overlays without clearing user-tab borrow requests", () => {
|
||||
@@ -31,6 +31,7 @@ describe("OverlayController", () => {
|
||||
expect(state.controlVisible).toBe(false);
|
||||
expect(state.activeSessionId).toBeNull();
|
||||
expect(state.activeHelp).toBeNull();
|
||||
expect(state.activeRecord).toBeNull();
|
||||
expect(state.automationBypassCount).toBe(0);
|
||||
});
|
||||
|
||||
@@ -69,4 +70,50 @@ describe("OverlayController", () => {
|
||||
expect(previous?.id).toBe("help-1");
|
||||
expect(controller.snapshot().activeHelp?.id).toBe("help-2");
|
||||
});
|
||||
|
||||
it("tracks active record request state", () => {
|
||||
const controller = new OverlayController();
|
||||
|
||||
controller.activateAgentSession("sess-1");
|
||||
controller.setAgentRecordRequest({
|
||||
id: "rec-1",
|
||||
onFinish: vi.fn(),
|
||||
});
|
||||
|
||||
expect(controller.snapshot().activeRecord?.id).toBe("rec-1");
|
||||
});
|
||||
|
||||
it("hides agent control overlay while recording is active", () => {
|
||||
const controller = new OverlayController();
|
||||
controller.activateAgentSession("sess-1");
|
||||
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(true);
|
||||
|
||||
controller.setAgentRecordRequest({
|
||||
id: "rec-1",
|
||||
onFinish: vi.fn(),
|
||||
});
|
||||
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(false);
|
||||
expect(controller.snapshot().controlVisible).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps control hidden after record ends until session overlays reset", () => {
|
||||
const controller = new OverlayController();
|
||||
controller.activateAgentSession("sess-1");
|
||||
controller.setAgentRecordRequest({
|
||||
id: "rec-1",
|
||||
onFinish: vi.fn(),
|
||||
});
|
||||
controller.setAutomationBypass(true);
|
||||
controller.setAutomationBypass(true);
|
||||
|
||||
controller.clearAgentRecordRequest("rec-1");
|
||||
expect(controller.snapshot().activeRecord).toBeNull();
|
||||
expect(controller.snapshot().suppressControlAfterRecord).toBe(true);
|
||||
expect(controller.snapshot().automationBypassCount).toBe(0);
|
||||
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(false);
|
||||
|
||||
controller.resetAgentOverlays("sess-1");
|
||||
expect(controller.snapshot().suppressControlAfterRecord).toBe(false);
|
||||
expect(controller.snapshot().controlVisible).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { handleRecordContentMessage, startRecordCapture } from "../record-capture";
|
||||
import { RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
|
||||
|
||||
vi.stubGlobal("chrome", {
|
||||
runtime: {
|
||||
sendMessage: vi.fn(() => Promise.resolve()),
|
||||
},
|
||||
});
|
||||
|
||||
describe("handleRecordContentMessage stop/cancel", () => {
|
||||
it("ignores STOP when no recording is active", () => {
|
||||
const dispose = vi.fn();
|
||||
const onStop = vi.fn();
|
||||
const setActiveRequestId = vi.fn();
|
||||
const setCapture = vi.fn();
|
||||
const sendResponse = vi.fn();
|
||||
|
||||
const needsAsync = handleRecordContentMessage(
|
||||
{ type: RECORD_STOP, requestId: "rec-stale" },
|
||||
{
|
||||
activeRequestId: null,
|
||||
capture: { dispose },
|
||||
setActiveRequestId,
|
||||
setCapture,
|
||||
onStart: vi.fn(),
|
||||
onStop,
|
||||
},
|
||||
sendResponse,
|
||||
);
|
||||
|
||||
expect(needsAsync).toBe(false);
|
||||
expect(dispose).not.toHaveBeenCalled();
|
||||
expect(onStop).not.toHaveBeenCalled();
|
||||
expect(setActiveRequestId).not.toHaveBeenCalled();
|
||||
expect(setCapture).not.toHaveBeenCalled();
|
||||
expect(sendResponse).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("ignores STOP for a mismatched requestId", () => {
|
||||
const dispose = vi.fn();
|
||||
const onStop = vi.fn();
|
||||
|
||||
const needsAsync = handleRecordContentMessage(
|
||||
{ type: RECORD_STOP, requestId: "rec-other" },
|
||||
{
|
||||
activeRequestId: "rec-1",
|
||||
capture: { dispose },
|
||||
setActiveRequestId: vi.fn(),
|
||||
setCapture: vi.fn(),
|
||||
onStart: vi.fn(),
|
||||
onStop,
|
||||
},
|
||||
vi.fn(),
|
||||
);
|
||||
|
||||
expect(needsAsync).toBe(false);
|
||||
expect(dispose).not.toHaveBeenCalled();
|
||||
expect(onStop).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("record-capture semantic", () => {
|
||||
let steps: RecordStepPayload[];
|
||||
|
||||
beforeEach(() => {
|
||||
steps = [];
|
||||
document.body.innerHTML = `
|
||||
<label for="q">查询</label>
|
||||
<input id="q" name="q" />
|
||||
<button type="button" aria-label="搜索">搜索</button>
|
||||
<div role="listbox" id="sug">
|
||||
<div role="option">建议项</div>
|
||||
</div>
|
||||
`;
|
||||
});
|
||||
|
||||
it("commits final fill value and records semantic click", () => {
|
||||
const capture = startRecordCapture("rec-1", (step) => steps.push(step));
|
||||
const input = document.querySelector("input")!;
|
||||
const button = document.querySelector("button")!;
|
||||
|
||||
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||
input.value = "hello";
|
||||
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
input.dispatchEvent(new FocusEvent("focusout", { bubbles: true }));
|
||||
|
||||
button.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
||||
|
||||
capture.dispose();
|
||||
|
||||
expect(steps).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
op: "fill",
|
||||
value: "hello",
|
||||
target: expect.objectContaining({ name: "查询", tag: "input" }),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
op: "click",
|
||||
target: expect.objectContaining({ name: "搜索", role: "button" }),
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(JSON.stringify(steps)).not.toMatch(/@e\d+/);
|
||||
expect(steps.some((s) => "selector" in s)).toBe(false);
|
||||
});
|
||||
|
||||
it("ignores autocomplete suggestion clicks while a fill session is open", () => {
|
||||
const capture = startRecordCapture("rec-2", (step) => steps.push(step));
|
||||
const input = document.querySelector("input")!;
|
||||
const option = document.querySelector('[role="option"]')!;
|
||||
|
||||
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||
input.value = "hel";
|
||||
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
option.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
||||
input.value = "hello";
|
||||
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
input.dispatchEvent(new FocusEvent("focusout", { bubbles: true }));
|
||||
capture.dispose();
|
||||
|
||||
expect(steps.filter((s) => s.op === "click")).toHaveLength(0);
|
||||
expect(steps.some((s) => s.op === "fill" && s.value === "hello")).toBe(true);
|
||||
});
|
||||
|
||||
it("records Enter press but not bare typing keys", () => {
|
||||
const capture = startRecordCapture("rec-press", (step) => steps.push(step));
|
||||
const input = document.querySelector("input")!;
|
||||
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||
input.dispatchEvent(
|
||||
new KeyboardEvent("keydown", { key: "a", bubbles: true }),
|
||||
);
|
||||
input.dispatchEvent(
|
||||
new KeyboardEvent("keydown", { key: "Enter", bubbles: true }),
|
||||
);
|
||||
capture.dispose();
|
||||
expect(steps.filter((s) => s.op === "press")).toEqual([
|
||||
expect.objectContaining({ op: "press", key: "Enter" }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not record clicks on anonymous layout divs", () => {
|
||||
document.body.innerHTML = `
|
||||
<div id="chrome">page chrome</div>
|
||||
<button type="button" aria-label="下一步">下一步</button>
|
||||
`;
|
||||
const capture = startRecordCapture("rec-3", (step) => steps.push(step));
|
||||
document
|
||||
.querySelector("#chrome")!
|
||||
.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
||||
document
|
||||
.querySelector("button")!
|
||||
.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
||||
capture.dispose();
|
||||
|
||||
expect(steps).toHaveLength(1);
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "click",
|
||||
target: { name: "下一步", role: "button" },
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,13 +1,21 @@
|
||||
import type { BorrowRequestData } from "./BorrowConfirmationOverlay";
|
||||
import type { HelpRequestData } from "./HelpRequestOverlay";
|
||||
import type { RecordRequestData } from "./RecordOverlay";
|
||||
|
||||
export interface OverlayState {
|
||||
borrowRequests: BorrowRequestData[];
|
||||
activeHelp: HelpRequestData | null;
|
||||
activeRecord: RecordRequestData | null;
|
||||
controlVisible: boolean;
|
||||
interrupting: boolean;
|
||||
activeSessionId: string | null;
|
||||
automationBypassCount: number;
|
||||
/**
|
||||
* After record Finish/Stop clears `activeRecord`, the session is still
|
||||
* alive until CLI `session.stop`. Suppress the control mask for that gap
|
||||
* so 「Agent 正在控制」does not flash between RecordOverlay and teardown.
|
||||
*/
|
||||
suppressControlAfterRecord: boolean;
|
||||
}
|
||||
|
||||
type MutableOverlayState = Omit<OverlayState, "controlVisible">;
|
||||
@@ -20,9 +28,11 @@ export class OverlayController {
|
||||
private state: MutableOverlayState = {
|
||||
borrowRequests: [],
|
||||
activeHelp: null,
|
||||
activeRecord: null,
|
||||
interrupting: false,
|
||||
activeSessionId: null,
|
||||
automationBypassCount: 0,
|
||||
suppressControlAfterRecord: false,
|
||||
};
|
||||
|
||||
snapshot(): OverlayState {
|
||||
@@ -70,6 +80,25 @@ export class OverlayController {
|
||||
this.state.activeHelp = null;
|
||||
}
|
||||
|
||||
setAgentRecordRequest(request: RecordRequestData): RecordRequestData | null {
|
||||
const previous = this.state.activeRecord;
|
||||
this.state.activeRecord = request;
|
||||
this.state.suppressControlAfterRecord = false;
|
||||
return previous;
|
||||
}
|
||||
|
||||
clearAgentRecordRequest(requestId?: string): void {
|
||||
if (!this.state.activeRecord) return;
|
||||
if (requestId && this.state.activeRecord.id !== requestId) return;
|
||||
this.state.activeRecord = null;
|
||||
// Hide control until session teardown — see OverlayState.suppressControlAfterRecord.
|
||||
this.state.suppressControlAfterRecord = true;
|
||||
// Record start/rearm may have stacked automation-bypass refs; drop them so a
|
||||
// stray ControlOverlay cannot sit with pointer-events:none (page usable,
|
||||
// Interrupt unclickable).
|
||||
this.state.automationBypassCount = 0;
|
||||
}
|
||||
|
||||
setAutomationBypass(enabled: boolean): void {
|
||||
if (enabled) {
|
||||
this.state.automationBypassCount += 1;
|
||||
@@ -86,10 +115,22 @@ export class OverlayController {
|
||||
this.state = {
|
||||
...this.state,
|
||||
activeHelp: null,
|
||||
activeRecord: null,
|
||||
interrupting: false,
|
||||
activeSessionId: null,
|
||||
automationBypassCount: 0,
|
||||
suppressControlAfterRecord: false,
|
||||
};
|
||||
return previousHelp;
|
||||
}
|
||||
}
|
||||
|
||||
/** Control mask ("Agent 正在控制") must hide while help/record overlays own the chrome. */
|
||||
export function shouldShowAgentControlOverlay(state: OverlayState): boolean {
|
||||
return (
|
||||
state.controlVisible &&
|
||||
!state.suppressControlAfterRecord &&
|
||||
state.activeHelp === null &&
|
||||
state.activeRecord === null
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,541 @@
|
||||
import {
|
||||
describeEventTarget,
|
||||
describeTarget,
|
||||
type TargetDescriptor,
|
||||
} from "@/lib/describe-target";
|
||||
import {
|
||||
isRecordCancelMessage,
|
||||
isRecordStartMessage,
|
||||
isRecordStopMessage,
|
||||
RECORD_CANCEL,
|
||||
RECORD_START,
|
||||
RECORD_STEP,
|
||||
RECORD_STOP,
|
||||
type RecordCancelMessage,
|
||||
type RecordStartAck,
|
||||
type RecordStartMessage,
|
||||
type RecordStepPayload,
|
||||
type RecordStopAck,
|
||||
type RecordStopMessage,
|
||||
} from "@/lib/record-bridge";
|
||||
import { shouldRecordPress } from "@/lib/trace-reducer";
|
||||
|
||||
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
|
||||
const failedStepDeliveries = new Set<string>();
|
||||
const knownRecordRequests = new Set<string>();
|
||||
|
||||
function sendRecordStep(requestId: string, step: RecordStepPayload): void {
|
||||
const payload = { type: RECORD_STEP, requestId, step };
|
||||
const pending = Promise.resolve(chrome.runtime.sendMessage(payload)).then(
|
||||
() => true,
|
||||
() => {
|
||||
failedStepDeliveries.add(requestId);
|
||||
return false;
|
||||
},
|
||||
);
|
||||
const sends = pendingStepSends.get(requestId) ?? new Set<Promise<boolean>>();
|
||||
sends.add(pending);
|
||||
pendingStepSends.set(requestId, sends);
|
||||
void pending.then(() => {
|
||||
sends.delete(pending);
|
||||
if (sends.size === 0) pendingStepSends.delete(requestId);
|
||||
});
|
||||
}
|
||||
|
||||
export interface RecordCaptureController {
|
||||
dispose(): void;
|
||||
}
|
||||
|
||||
interface FillSession {
|
||||
element: FillableElement;
|
||||
target: TargetDescriptor;
|
||||
baselineValue: string;
|
||||
lastValue: string;
|
||||
}
|
||||
|
||||
type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement;
|
||||
|
||||
function eventTarget(event: Event): EventTarget | null {
|
||||
return event.composedPath()[0] ?? event.target;
|
||||
}
|
||||
|
||||
function isOverlayTarget(target: EventTarget | null): boolean {
|
||||
if (!(target instanceof Node)) return false;
|
||||
const root = document.documentElement;
|
||||
let node: Node | null = target;
|
||||
while (node && node !== root) {
|
||||
if (node instanceof Element && node.hasAttribute("data-bsk-overlay")) {
|
||||
return true;
|
||||
}
|
||||
const rootNode: Node | Document | ShadowRoot = node.getRootNode();
|
||||
if (rootNode instanceof ShadowRoot) {
|
||||
const host: Element = rootNode.host;
|
||||
if (host.hasAttribute("data-bsk-overlay")) {
|
||||
return true;
|
||||
}
|
||||
node = host;
|
||||
} else {
|
||||
node = node.parentNode;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function isTextFillable(el: Element): el is FillableElement {
|
||||
if (el instanceof HTMLElement && (el.isContentEditable || el.contentEditable === "true")) {
|
||||
return true;
|
||||
}
|
||||
if (el instanceof HTMLTextAreaElement) return true;
|
||||
if (!(el instanceof HTMLInputElement)) return false;
|
||||
const type = el.type.toLowerCase();
|
||||
return type !== "checkbox" && type !== "radio" && type !== "file" && type !== "button";
|
||||
}
|
||||
|
||||
function fillableFromTarget(target: EventTarget | null): FillableElement | null {
|
||||
if (!(target instanceof Element)) return null;
|
||||
if (isTextFillable(target)) return target;
|
||||
const el = target.closest('input,textarea,[contenteditable]:not([contenteditable="false"])');
|
||||
if (el && isTextFillable(el)) return el;
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Clicks on search chrome that only focus the nearby input should not become steps. */
|
||||
function nearbyFillableFromSearchChrome(target: Element): FillableElement | null {
|
||||
const container = target.closest(
|
||||
'[id*="chat-input"], [id*="search"], [class*="search"], form, [role="search"]',
|
||||
);
|
||||
if (!container) return null;
|
||||
const fillable = container.querySelector(
|
||||
'textarea, input[type="search"], input[name="q"], #chat-textarea',
|
||||
);
|
||||
if (
|
||||
fillable instanceof HTMLElement &&
|
||||
isTextFillable(fillable) &&
|
||||
fillable !== target &&
|
||||
!fillable.contains(target)
|
||||
) {
|
||||
return fillable;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function fillableValue(el: FillableElement): string {
|
||||
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
|
||||
return el.value;
|
||||
}
|
||||
return el.textContent ?? "";
|
||||
}
|
||||
|
||||
/** Clicks that only pick an autocomplete/suggestion value — not a semantic submit action. */
|
||||
function isInputCompletionClick(target: EventTarget | null, session: FillSession | null): boolean {
|
||||
if (!session || !(target instanceof Element)) return false;
|
||||
if (session.element.contains(target)) return false;
|
||||
|
||||
const option = target.closest('[role="option"]');
|
||||
if (option?.closest('[role="listbox"]')) return true;
|
||||
|
||||
const suggestionRoot = target.closest(
|
||||
[
|
||||
'[id*="Sug"]',
|
||||
'[id*="sug"]',
|
||||
'[class*="suggest"]',
|
||||
'[class*="autocomplete"]',
|
||||
'[class*="typeahead"]',
|
||||
"[data-autocomplete]",
|
||||
].join(", "),
|
||||
);
|
||||
if (suggestionRoot && !suggestionRoot.contains(session.element)) return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
function scheduleInputCompletionCommit(
|
||||
sessionElement: FillableElement,
|
||||
syncFillSessionValue: (el: FillableElement) => void,
|
||||
commitFillSession: () => void,
|
||||
hasSessionFor: (el: FillableElement) => boolean,
|
||||
): void {
|
||||
const syncAndCommit = () => {
|
||||
if (!hasSessionFor(sessionElement)) return;
|
||||
syncFillSessionValue(sessionElement);
|
||||
commitFillSession();
|
||||
};
|
||||
syncFillSessionValue(sessionElement);
|
||||
queueMicrotask(syncAndCommit);
|
||||
setTimeout(syncAndCommit, 0);
|
||||
}
|
||||
|
||||
export function startRecordCapture(
|
||||
_requestId: string,
|
||||
sendStep: (step: RecordStepPayload) => void,
|
||||
): RecordCaptureController {
|
||||
const emitStep = (step: RecordStepPayload) => {
|
||||
sendStep({ page_url: location.href, ...step });
|
||||
};
|
||||
let fillSession: FillSession | null = null;
|
||||
let composing = false;
|
||||
let lastUrl = location.href;
|
||||
let keyboardActivation: { target: EventTarget | null; recordedAt: number } | undefined;
|
||||
let generatedControlClick: Element | null = null;
|
||||
let navigationActionPending = false;
|
||||
let navigationActionVersion = 0;
|
||||
const committedValues = new WeakMap<FillableElement, string>();
|
||||
|
||||
const markNavigationAction = () => {
|
||||
navigationActionPending = true;
|
||||
const version = ++navigationActionVersion;
|
||||
setTimeout(() => {
|
||||
if (navigationActionVersion === version) {
|
||||
navigationActionPending = false;
|
||||
}
|
||||
}, 0);
|
||||
};
|
||||
|
||||
const emitFill = (session: FillSession) => {
|
||||
if (session.lastValue === session.baselineValue) return;
|
||||
const isPassword =
|
||||
session.element instanceof HTMLInputElement && session.element.type === "password";
|
||||
const value = isPassword ? "***" : session.lastValue;
|
||||
emitStep({
|
||||
op: "fill",
|
||||
target: session.target,
|
||||
value,
|
||||
...(isPassword ? { redacted: true } : {}),
|
||||
});
|
||||
};
|
||||
|
||||
const commitFillSession = () => {
|
||||
if (!fillSession || composing) return;
|
||||
const session = fillSession;
|
||||
fillSession = null;
|
||||
emitFill(session);
|
||||
committedValues.set(session.element, session.lastValue);
|
||||
};
|
||||
|
||||
const syncFillSessionValue = (el: FillableElement) => {
|
||||
if (!fillSession || fillSession.element !== el) return;
|
||||
fillSession.lastValue = fillableValue(el);
|
||||
};
|
||||
|
||||
const ensureFillSession = (el: FillableElement) => {
|
||||
if (fillSession?.element === el) return;
|
||||
if (fillSession) commitFillSession();
|
||||
const currentValue = fillableValue(el);
|
||||
const baselineValue = committedValues.get(el) ?? currentValue;
|
||||
fillSession = {
|
||||
element: el,
|
||||
target: describeTarget(el),
|
||||
baselineValue,
|
||||
lastValue: currentValue,
|
||||
};
|
||||
};
|
||||
|
||||
const emitNavigateIfChanged = (causedByAction?: boolean) => {
|
||||
if (location.href === lastUrl) return;
|
||||
commitFillSession();
|
||||
lastUrl = location.href;
|
||||
emitStep({
|
||||
op: "navigate",
|
||||
url: location.href,
|
||||
...(causedByAction !== undefined ? { navigation_caused_by_action: causedByAction } : {}),
|
||||
});
|
||||
};
|
||||
|
||||
const emitClick = (event: MouseEvent) => {
|
||||
// Only record clicks an LLM can re-identify (named interactive controls).
|
||||
const target = describeEventTarget(eventTarget(event));
|
||||
if (!target) return;
|
||||
markNavigationAction();
|
||||
emitStep({
|
||||
op: "click",
|
||||
target,
|
||||
expects_navigation: true,
|
||||
});
|
||||
};
|
||||
|
||||
const onClick = (event: MouseEvent) => {
|
||||
if (event.button !== 0) return;
|
||||
const target = eventTarget(event);
|
||||
if (isOverlayTarget(target)) return;
|
||||
if (event.detail === 0 && generatedControlClick !== null && target === generatedControlClick) {
|
||||
generatedControlClick = null;
|
||||
return;
|
||||
}
|
||||
if (
|
||||
event.detail === 0 &&
|
||||
keyboardActivation?.target === target &&
|
||||
Date.now() - keyboardActivation.recordedAt < 500
|
||||
) {
|
||||
keyboardActivation = undefined;
|
||||
return;
|
||||
}
|
||||
|
||||
const label = target instanceof Element ? target.closest("label") : null;
|
||||
const nestedInteractive =
|
||||
target instanceof Element ? target.closest("a,button,input,select,textarea") : null;
|
||||
if (label instanceof HTMLLabelElement && !nestedInteractive) {
|
||||
commitFillSession();
|
||||
generatedControlClick = label.control;
|
||||
emitClick(event);
|
||||
return;
|
||||
}
|
||||
|
||||
const fillable = fillableFromTarget(target);
|
||||
if (fillable) {
|
||||
ensureFillSession(fillable);
|
||||
return;
|
||||
}
|
||||
|
||||
if (target instanceof Element) {
|
||||
const nearbyFillable = nearbyFillableFromSearchChrome(target);
|
||||
if (nearbyFillable) {
|
||||
ensureFillSession(nearbyFillable);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (isInputCompletionClick(target, fillSession)) {
|
||||
markNavigationAction();
|
||||
const sessionElement = fillSession!.element;
|
||||
scheduleInputCompletionCommit(
|
||||
sessionElement,
|
||||
syncFillSessionValue,
|
||||
commitFillSession,
|
||||
(el) => fillSession?.element === el,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
commitFillSession();
|
||||
if (target instanceof Element && target.closest("select")) return;
|
||||
emitClick(event);
|
||||
};
|
||||
|
||||
const onFocusIn = (event: FocusEvent) => {
|
||||
const target = fillableFromTarget(eventTarget(event));
|
||||
if (target) ensureFillSession(target);
|
||||
};
|
||||
|
||||
const onFocusOut = (event: FocusEvent) => {
|
||||
const target = fillableFromTarget(eventTarget(event));
|
||||
if (!target) return;
|
||||
if (!fillSession || fillSession.element !== target) return;
|
||||
syncFillSessionValue(target);
|
||||
commitFillSession();
|
||||
};
|
||||
|
||||
const onInput = (event: Event) => {
|
||||
const target = fillableFromTarget(eventTarget(event));
|
||||
if (!target) return;
|
||||
if (composing) return;
|
||||
ensureFillSession(target);
|
||||
syncFillSessionValue(target);
|
||||
};
|
||||
|
||||
const onCompositionStart = () => {
|
||||
composing = true;
|
||||
};
|
||||
|
||||
const onCompositionEnd = (event: CompositionEvent) => {
|
||||
composing = false;
|
||||
const target = fillableFromTarget(eventTarget(event));
|
||||
if (target) {
|
||||
ensureFillSession(target);
|
||||
syncFillSessionValue(target);
|
||||
}
|
||||
};
|
||||
|
||||
const onChange = (event: Event) => {
|
||||
commitFillSession();
|
||||
const target = eventTarget(event);
|
||||
if (target instanceof HTMLSelectElement) {
|
||||
const values = Array.from(target.selectedOptions).map((opt) => opt.value);
|
||||
const labels = Array.from(target.selectedOptions).map((opt) =>
|
||||
(opt.label || opt.textContent || opt.value).trim(),
|
||||
);
|
||||
const desc = describeTarget(target);
|
||||
markNavigationAction();
|
||||
emitStep({
|
||||
op: "select",
|
||||
target: desc,
|
||||
values,
|
||||
labels,
|
||||
expects_navigation: true,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const onKeyDown = (event: KeyboardEvent) => {
|
||||
if (isOverlayTarget(eventTarget(event))) return;
|
||||
const target = eventTarget(event);
|
||||
const fillable = fillableFromTarget(target);
|
||||
if (fillable) {
|
||||
ensureFillSession(fillable);
|
||||
syncFillSessionValue(fillable);
|
||||
}
|
||||
const modifiers = [
|
||||
...(event.altKey ? (["alt"] as const) : []),
|
||||
...(event.ctrlKey ? (["ctrl"] as const) : []),
|
||||
...(event.metaKey ? (["meta"] as const) : []),
|
||||
...(event.shiftKey ? (["shift"] as const) : []),
|
||||
];
|
||||
if (!shouldRecordPress(event.key, modifiers)) {
|
||||
return;
|
||||
}
|
||||
markNavigationAction();
|
||||
if (event.key === "Enter" || event.key === " ") {
|
||||
const submitTarget =
|
||||
event.key === "Enter" && fillable
|
||||
? fillable
|
||||
.closest("form")
|
||||
?.querySelector(
|
||||
'button:not([type]),button[type="submit"],input[type="submit"],input[type="image"]',
|
||||
)
|
||||
: null;
|
||||
keyboardActivation = {
|
||||
target: submitTarget ?? target,
|
||||
recordedAt: Date.now(),
|
||||
};
|
||||
}
|
||||
if (fillable) {
|
||||
commitFillSession();
|
||||
}
|
||||
const desc = describeEventTarget(target);
|
||||
if (!desc && !event.key) return;
|
||||
emitStep({
|
||||
op: "press",
|
||||
key: event.key,
|
||||
...(desc ? { target: desc } : {}),
|
||||
...(modifiers.length ? { modifiers } : {}),
|
||||
expects_navigation: event.key === "Enter",
|
||||
});
|
||||
};
|
||||
|
||||
document.addEventListener("click", onClick, true);
|
||||
document.addEventListener("focusin", onFocusIn, true);
|
||||
document.addEventListener("focusout", onFocusOut, true);
|
||||
document.addEventListener("input", onInput, true);
|
||||
document.addEventListener("compositionstart", onCompositionStart, true);
|
||||
document.addEventListener("compositionend", onCompositionEnd, true);
|
||||
document.addEventListener("change", onChange, true);
|
||||
document.addEventListener("keydown", onKeyDown, true);
|
||||
|
||||
const urlObserver = new MutationObserver(() => emitNavigateIfChanged());
|
||||
urlObserver.observe(document, { subtree: true, childList: true });
|
||||
const onUrlEvent = () => emitNavigateIfChanged();
|
||||
window.addEventListener("hashchange", onUrlEvent);
|
||||
window.addEventListener("popstate", onUrlEvent);
|
||||
window.addEventListener("pagehide", commitFillSession);
|
||||
|
||||
const originalPushState = history.pushState;
|
||||
const originalReplaceState = history.replaceState;
|
||||
history.pushState = function (...args: Parameters<History["pushState"]>) {
|
||||
originalPushState.apply(this, args);
|
||||
emitNavigateIfChanged(navigationActionPending ? true : undefined);
|
||||
};
|
||||
history.replaceState = function (...args: Parameters<History["replaceState"]>) {
|
||||
originalReplaceState.apply(this, args);
|
||||
emitNavigateIfChanged(navigationActionPending ? true : undefined);
|
||||
};
|
||||
|
||||
return {
|
||||
dispose() {
|
||||
commitFillSession();
|
||||
document.removeEventListener("click", onClick, true);
|
||||
document.removeEventListener("focusin", onFocusIn, true);
|
||||
document.removeEventListener("focusout", onFocusOut, true);
|
||||
document.removeEventListener("input", onInput, true);
|
||||
document.removeEventListener("compositionstart", onCompositionStart, true);
|
||||
document.removeEventListener("compositionend", onCompositionEnd, true);
|
||||
document.removeEventListener("change", onChange, true);
|
||||
document.removeEventListener("keydown", onKeyDown, true);
|
||||
urlObserver.disconnect();
|
||||
window.removeEventListener("hashchange", onUrlEvent);
|
||||
window.removeEventListener("popstate", onUrlEvent);
|
||||
window.removeEventListener("pagehide", commitFillSession);
|
||||
history.pushState = originalPushState;
|
||||
history.replaceState = originalReplaceState;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export type RecordContentMessage = RecordStartMessage | RecordStopMessage | RecordCancelMessage;
|
||||
|
||||
export function isRecordContentMessage(msg: unknown): msg is RecordContentMessage {
|
||||
return isRecordStartMessage(msg) || isRecordStopMessage(msg) || isRecordCancelMessage(msg);
|
||||
}
|
||||
|
||||
export function handleRecordContentMessage(
|
||||
message: RecordContentMessage,
|
||||
state: {
|
||||
activeRequestId: string | null;
|
||||
capture: RecordCaptureController | null;
|
||||
setActiveRequestId(id: string | null): void;
|
||||
setCapture(capture: RecordCaptureController | null): void;
|
||||
onStart(requestId: string): void;
|
||||
onStop(): void;
|
||||
},
|
||||
sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
|
||||
): boolean {
|
||||
if (isRecordStartMessage(message)) {
|
||||
if (!knownRecordRequests.has(message.requestId)) {
|
||||
knownRecordRequests.add(message.requestId);
|
||||
failedStepDeliveries.delete(message.requestId);
|
||||
}
|
||||
state.capture?.dispose();
|
||||
state.setCapture(
|
||||
startRecordCapture(message.requestId, (step) => {
|
||||
sendRecordStep(message.requestId, step);
|
||||
}),
|
||||
);
|
||||
state.setActiveRequestId(message.requestId);
|
||||
state.onStart(message.requestId);
|
||||
sendResponse?.({ ok: true });
|
||||
return sendResponse !== undefined;
|
||||
}
|
||||
|
||||
if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
|
||||
// Require an active recording that matches this requestId — otherwise a
|
||||
// stray STOP/CANCEL (e.g. after teardown) would still run finishStop and
|
||||
// clear overlay state even though nothing was capturing.
|
||||
if (!state.activeRequestId || state.activeRequestId !== message.requestId) {
|
||||
return false;
|
||||
}
|
||||
state.capture?.dispose();
|
||||
state.setCapture(null);
|
||||
const finishStop = () => {
|
||||
state.onStop();
|
||||
state.setActiveRequestId(null);
|
||||
};
|
||||
if (isRecordStopMessage(message) && sendResponse) {
|
||||
const pending = [...(pendingStepSends.get(message.requestId) ?? [])];
|
||||
void Promise.all(pending).then((delivered) => {
|
||||
finishStop();
|
||||
const succeeded = delivered.every(Boolean) && !failedStepDeliveries.has(message.requestId);
|
||||
if (succeeded) {
|
||||
failedStepDeliveries.delete(message.requestId);
|
||||
knownRecordRequests.delete(message.requestId);
|
||||
}
|
||||
sendResponse(
|
||||
succeeded
|
||||
? { ok: true }
|
||||
: {
|
||||
ok: false,
|
||||
error: "failed to deliver one or more recorded steps",
|
||||
},
|
||||
);
|
||||
});
|
||||
return true;
|
||||
}
|
||||
if (isRecordCancelMessage(message)) {
|
||||
failedStepDeliveries.delete(message.requestId);
|
||||
knownRecordRequests.delete(message.requestId);
|
||||
}
|
||||
finishStop();
|
||||
return false;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
export { RECORD_CANCEL, RECORD_START, RECORD_STEP, RECORD_STOP };
|
||||
Reference in New Issue
Block a user