feat: add bsk record for semantic user-action traces

Capture user actions in the Agent Window and export a record-only
trace (pages[] + navigate/click/fill/select/press steps with target
descriptors). Adds the record overlay UI aligned with the control
overlay, and wires record start/stop/await through CLI, daemon,
protocol, and extension.
This commit is contained in:
“haonan”
2026-07-22 11:27:30 +08:00
parent ccf3c32140
commit ac028a7a18
47 changed files with 6038 additions and 46 deletions
@@ -58,6 +58,7 @@ export function ControlOverlay({
/>
<div
data-slot="control-overlay-blocker"
onPointerDown={(event) => {
if (automationBypass) return;
event.preventDefault();
@@ -80,13 +81,16 @@ export function ControlOverlay({
/>
<div
data-slot="control-overlay-pill"
style={{
position: "fixed",
bottom: 32,
left: "50%",
transform: "translateX(-50%)",
zIndex: 2147483647,
pointerEvents,
// Always receive clicks so Interrupt works even while the page
// blocker is in automation-bypass mode (CDP pass-through).
pointerEvents: "auto",
display: "flex",
alignItems: "center",
gap: 12,
@@ -122,7 +126,7 @@ export function ControlOverlay({
disabled={interrupting}
onClick={onInterrupt}
style={{
pointerEvents,
pointerEvents: "auto",
display: "flex",
alignItems: "center",
gap: 6,
@@ -0,0 +1,114 @@
import { useTranslation } from "@browser-skill/i18n/react";
import { RiCheckboxCircleLine } from "@remixicon/react";
import { useEffect, useState } from "react";
export interface RecordRequestData {
id: string;
onFinish: () => void;
}
type Props = {
request: RecordRequestData | null;
};
export function RecordOverlay({ request }: Props) {
const { t } = useTranslation("extension");
const [show, setShow] = useState(false);
useEffect(() => {
if (request) {
const raf = requestAnimationFrame(() => setShow(true));
return () => cancelAnimationFrame(raf);
}
setShow(false);
}, [request]);
if (!request) return null;
return (
<>
<style>{`
@keyframes bsk-rec-pulse {
0%, 100% { opacity: 1; transform: scale(1); }
50% { opacity: 0.35; transform: scale(0.82); }
}
`}</style>
<div
data-slot="record-overlay-pill"
style={{
position: "fixed",
bottom: 32,
left: "50%",
zIndex: 2147483647,
pointerEvents: "auto",
display: "flex",
alignItems: "center",
gap: 10,
backgroundColor: "#fff",
borderRadius: 9999,
padding: "10px 10px 10px 20px",
boxShadow: "0 8px 32px rgba(15,23,42,0.16), 0 2px 8px rgba(0,0,0,0.1)",
opacity: show ? 1 : 0,
transform: show ? "translateX(-50%) translateY(0)" : "translateX(-50%) translateY(8px)",
transition: "opacity 300ms ease-out, transform 300ms ease-out",
fontFamily:
'-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif',
maxWidth: "min(420px, calc(100vw - 32px))",
}}
>
<span
data-slot="record-overlay-indicator"
style={{
width: 9,
height: 9,
borderRadius: "50%",
backgroundColor: "#ef4444",
flexShrink: 0,
animation: "bsk-rec-pulse 1.4s ease-in-out infinite",
}}
/>
<span
style={{
flex: 1,
fontSize: 16,
fontWeight: 500,
color: "#333",
whiteSpace: "nowrap",
userSelect: "none",
overflow: "hidden",
textOverflow: "ellipsis",
}}
>
{t("recordOverlay.recording")}
</span>
<button
type="button"
data-slot="record-overlay-finish"
onClick={request.onFinish}
style={{
pointerEvents: "auto",
display: "flex",
alignItems: "center",
gap: 6,
border: "none",
borderRadius: 9999,
padding: "8px 20px 8px 16px",
fontSize: 15,
fontWeight: 600,
color: "#fff",
backgroundColor: "#f97316",
cursor: "pointer",
transition: "background-color 150ms ease-out, opacity 150ms ease-out",
whiteSpace: "nowrap",
lineHeight: 1,
flexShrink: 0,
}}
>
<RiCheckboxCircleLine size={18} color="#fff" />
{t("recordOverlay.finish")}
</button>
</div>
</>
);
}
@@ -7,7 +7,7 @@ describe("ControlOverlay", () => {
cleanup();
});
it("sets pointer-events none on blocker and pill when automationBypass is true", () => {
it("keeps page blocker none under automationBypass but Interrupt stays clickable", () => {
const { container } = render(
<ControlOverlay
visible={true}
@@ -17,13 +17,17 @@ describe("ControlOverlay", () => {
/>,
);
const blocker = container.querySelector("[data-slot='control-overlay']")?.nextElementSibling;
const blocker = container.querySelector("[data-slot='control-overlay-blocker']");
expect(blocker).toBeTruthy();
expect((blocker as HTMLElement).style.pointerEvents).toBe("none");
const pill = container.querySelector("[data-slot='control-overlay-pill']");
expect(pill).toBeTruthy();
expect((pill as HTMLElement).style.pointerEvents).toBe("auto");
const stopBtn = container.querySelector("[data-slot='control-overlay-stop-all']");
expect(stopBtn).toBeTruthy();
expect((stopBtn as HTMLElement).style.pointerEvents).toBe("none");
expect((stopBtn as HTMLElement).style.pointerEvents).toBe("auto");
});
it("uses pointer-events auto on blocker when automationBypass is false", () => {
@@ -36,7 +40,7 @@ describe("ControlOverlay", () => {
/>,
);
const blocker = container.querySelector("[data-slot='control-overlay']")?.nextElementSibling;
const blocker = container.querySelector("[data-slot='control-overlay-blocker']");
expect(blocker).toBeTruthy();
expect((blocker as HTMLElement).style.pointerEvents).toBe("auto");
});
@@ -0,0 +1,37 @@
import { cleanup, fireEvent, render } from "@testing-library/react";
import { afterEach, describe, expect, it, vi } from "vitest";
import { RecordOverlay } from "../RecordOverlay";
describe("RecordOverlay", () => {
afterEach(() => {
cleanup();
});
it("renders the bottom pill with orange finish CTA and no logo", () => {
const onFinish = vi.fn();
const { container } = render(<RecordOverlay request={{ id: "rec-1", onFinish }} />);
const pill = container.querySelector("[data-slot='record-overlay-pill']");
expect(pill).toBeTruthy();
expect(pill?.querySelector("img")).toBeNull();
expect((pill as HTMLElement).style.borderRadius).toBe("9999px");
expect((pill as HTMLElement).style.backgroundColor).toBe("#fff");
const finish = container.querySelector("[data-slot='record-overlay-finish']");
expect(finish).toBeTruthy();
expect((finish as HTMLElement).style.backgroundColor).toBe("#f97316");
expect((finish as HTMLElement).style.borderRadius).toBe("9999px");
fireEvent.click(finish!);
expect(onFinish).toHaveBeenCalledTimes(1);
});
it("shows a pulsing recording indicator and no full-screen glow layer", () => {
const { container } = render(<RecordOverlay request={{ id: "rec-1", onFinish: vi.fn() }} />);
expect(container.querySelector("[data-slot='record-overlay']")).toBeNull();
const indicator = container.querySelector("[data-slot='record-overlay-indicator']");
expect(indicator).toBeTruthy();
expect((indicator as HTMLElement).style.animation).toContain("bsk-rec-pulse");
});
});
@@ -1,5 +1,5 @@
import { describe, expect, it, vi } from "vitest";
import { OverlayController } from "../overlay-controller";
import { OverlayController, shouldShowAgentControlOverlay } from "../overlay-controller";
describe("OverlayController", () => {
it("resets agent overlays without clearing user-tab borrow requests", () => {
@@ -31,6 +31,7 @@ describe("OverlayController", () => {
expect(state.controlVisible).toBe(false);
expect(state.activeSessionId).toBeNull();
expect(state.activeHelp).toBeNull();
expect(state.activeRecord).toBeNull();
expect(state.automationBypassCount).toBe(0);
});
@@ -69,4 +70,50 @@ describe("OverlayController", () => {
expect(previous?.id).toBe("help-1");
expect(controller.snapshot().activeHelp?.id).toBe("help-2");
});
it("tracks active record request state", () => {
const controller = new OverlayController();
controller.activateAgentSession("sess-1");
controller.setAgentRecordRequest({
id: "rec-1",
onFinish: vi.fn(),
});
expect(controller.snapshot().activeRecord?.id).toBe("rec-1");
});
it("hides agent control overlay while recording is active", () => {
const controller = new OverlayController();
controller.activateAgentSession("sess-1");
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(true);
controller.setAgentRecordRequest({
id: "rec-1",
onFinish: vi.fn(),
});
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(false);
expect(controller.snapshot().controlVisible).toBe(true);
});
it("keeps control hidden after record ends until session overlays reset", () => {
const controller = new OverlayController();
controller.activateAgentSession("sess-1");
controller.setAgentRecordRequest({
id: "rec-1",
onFinish: vi.fn(),
});
controller.setAutomationBypass(true);
controller.setAutomationBypass(true);
controller.clearAgentRecordRequest("rec-1");
expect(controller.snapshot().activeRecord).toBeNull();
expect(controller.snapshot().suppressControlAfterRecord).toBe(true);
expect(controller.snapshot().automationBypassCount).toBe(0);
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(false);
controller.resetAgentOverlays("sess-1");
expect(controller.snapshot().suppressControlAfterRecord).toBe(false);
expect(controller.snapshot().controlVisible).toBe(false);
});
});
@@ -0,0 +1,163 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { handleRecordContentMessage, startRecordCapture } from "../record-capture";
import { RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
vi.stubGlobal("chrome", {
runtime: {
sendMessage: vi.fn(() => Promise.resolve()),
},
});
describe("handleRecordContentMessage stop/cancel", () => {
it("ignores STOP when no recording is active", () => {
const dispose = vi.fn();
const onStop = vi.fn();
const setActiveRequestId = vi.fn();
const setCapture = vi.fn();
const sendResponse = vi.fn();
const needsAsync = handleRecordContentMessage(
{ type: RECORD_STOP, requestId: "rec-stale" },
{
activeRequestId: null,
capture: { dispose },
setActiveRequestId,
setCapture,
onStart: vi.fn(),
onStop,
},
sendResponse,
);
expect(needsAsync).toBe(false);
expect(dispose).not.toHaveBeenCalled();
expect(onStop).not.toHaveBeenCalled();
expect(setActiveRequestId).not.toHaveBeenCalled();
expect(setCapture).not.toHaveBeenCalled();
expect(sendResponse).not.toHaveBeenCalled();
});
it("ignores STOP for a mismatched requestId", () => {
const dispose = vi.fn();
const onStop = vi.fn();
const needsAsync = handleRecordContentMessage(
{ type: RECORD_STOP, requestId: "rec-other" },
{
activeRequestId: "rec-1",
capture: { dispose },
setActiveRequestId: vi.fn(),
setCapture: vi.fn(),
onStart: vi.fn(),
onStop,
},
vi.fn(),
);
expect(needsAsync).toBe(false);
expect(dispose).not.toHaveBeenCalled();
expect(onStop).not.toHaveBeenCalled();
});
});
describe("record-capture semantic", () => {
let steps: RecordStepPayload[];
beforeEach(() => {
steps = [];
document.body.innerHTML = `
<label for="q">查询</label>
<input id="q" name="q" />
<button type="button" aria-label="搜索">搜索</button>
<div role="listbox" id="sug">
<div role="option">建议项</div>
</div>
`;
});
it("commits final fill value and records semantic click", () => {
const capture = startRecordCapture("rec-1", (step) => steps.push(step));
const input = document.querySelector("input")!;
const button = document.querySelector("button")!;
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
input.value = "hello";
input.dispatchEvent(new Event("input", { bubbles: true }));
input.dispatchEvent(new FocusEvent("focusout", { bubbles: true }));
button.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
capture.dispose();
expect(steps).toEqual(
expect.arrayContaining([
expect.objectContaining({
op: "fill",
value: "hello",
target: expect.objectContaining({ name: "查询", tag: "input" }),
}),
expect.objectContaining({
op: "click",
target: expect.objectContaining({ name: "搜索", role: "button" }),
}),
]),
);
expect(JSON.stringify(steps)).not.toMatch(/@e\d+/);
expect(steps.some((s) => "selector" in s)).toBe(false);
});
it("ignores autocomplete suggestion clicks while a fill session is open", () => {
const capture = startRecordCapture("rec-2", (step) => steps.push(step));
const input = document.querySelector("input")!;
const option = document.querySelector('[role="option"]')!;
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
input.value = "hel";
input.dispatchEvent(new Event("input", { bubbles: true }));
option.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
input.value = "hello";
input.dispatchEvent(new Event("input", { bubbles: true }));
input.dispatchEvent(new FocusEvent("focusout", { bubbles: true }));
capture.dispose();
expect(steps.filter((s) => s.op === "click")).toHaveLength(0);
expect(steps.some((s) => s.op === "fill" && s.value === "hello")).toBe(true);
});
it("records Enter press but not bare typing keys", () => {
const capture = startRecordCapture("rec-press", (step) => steps.push(step));
const input = document.querySelector("input")!;
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
input.dispatchEvent(
new KeyboardEvent("keydown", { key: "a", bubbles: true }),
);
input.dispatchEvent(
new KeyboardEvent("keydown", { key: "Enter", bubbles: true }),
);
capture.dispose();
expect(steps.filter((s) => s.op === "press")).toEqual([
expect.objectContaining({ op: "press", key: "Enter" }),
]);
});
it("does not record clicks on anonymous layout divs", () => {
document.body.innerHTML = `
<div id="chrome">page chrome</div>
<button type="button" aria-label="下一步">下一步</button>
`;
const capture = startRecordCapture("rec-3", (step) => steps.push(step));
document
.querySelector("#chrome")!
.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
document
.querySelector("button")!
.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
capture.dispose();
expect(steps).toHaveLength(1);
expect(steps[0]).toMatchObject({
op: "click",
target: { name: "下一步", role: "button" },
});
});
});
@@ -1,13 +1,21 @@
import type { BorrowRequestData } from "./BorrowConfirmationOverlay";
import type { HelpRequestData } from "./HelpRequestOverlay";
import type { RecordRequestData } from "./RecordOverlay";
export interface OverlayState {
borrowRequests: BorrowRequestData[];
activeHelp: HelpRequestData | null;
activeRecord: RecordRequestData | null;
controlVisible: boolean;
interrupting: boolean;
activeSessionId: string | null;
automationBypassCount: number;
/**
* After record Finish/Stop clears `activeRecord`, the session is still
* alive until CLI `session.stop`. Suppress the control mask for that gap
* so 「Agent 正在控制」does not flash between RecordOverlay and teardown.
*/
suppressControlAfterRecord: boolean;
}
type MutableOverlayState = Omit<OverlayState, "controlVisible">;
@@ -20,9 +28,11 @@ export class OverlayController {
private state: MutableOverlayState = {
borrowRequests: [],
activeHelp: null,
activeRecord: null,
interrupting: false,
activeSessionId: null,
automationBypassCount: 0,
suppressControlAfterRecord: false,
};
snapshot(): OverlayState {
@@ -70,6 +80,25 @@ export class OverlayController {
this.state.activeHelp = null;
}
setAgentRecordRequest(request: RecordRequestData): RecordRequestData | null {
const previous = this.state.activeRecord;
this.state.activeRecord = request;
this.state.suppressControlAfterRecord = false;
return previous;
}
clearAgentRecordRequest(requestId?: string): void {
if (!this.state.activeRecord) return;
if (requestId && this.state.activeRecord.id !== requestId) return;
this.state.activeRecord = null;
// Hide control until session teardown — see OverlayState.suppressControlAfterRecord.
this.state.suppressControlAfterRecord = true;
// Record start/rearm may have stacked automation-bypass refs; drop them so a
// stray ControlOverlay cannot sit with pointer-events:none (page usable,
// Interrupt unclickable).
this.state.automationBypassCount = 0;
}
setAutomationBypass(enabled: boolean): void {
if (enabled) {
this.state.automationBypassCount += 1;
@@ -86,10 +115,22 @@ export class OverlayController {
this.state = {
...this.state,
activeHelp: null,
activeRecord: null,
interrupting: false,
activeSessionId: null,
automationBypassCount: 0,
suppressControlAfterRecord: false,
};
return previousHelp;
}
}
/** Control mask ("Agent 正在控制") must hide while help/record overlays own the chrome. */
export function shouldShowAgentControlOverlay(state: OverlayState): boolean {
return (
state.controlVisible &&
!state.suppressControlAfterRecord &&
state.activeHelp === null &&
state.activeRecord === null
);
}
@@ -0,0 +1,541 @@
import {
describeEventTarget,
describeTarget,
type TargetDescriptor,
} from "@/lib/describe-target";
import {
isRecordCancelMessage,
isRecordStartMessage,
isRecordStopMessage,
RECORD_CANCEL,
RECORD_START,
RECORD_STEP,
RECORD_STOP,
type RecordCancelMessage,
type RecordStartAck,
type RecordStartMessage,
type RecordStepPayload,
type RecordStopAck,
type RecordStopMessage,
} from "@/lib/record-bridge";
import { shouldRecordPress } from "@/lib/trace-reducer";
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
const failedStepDeliveries = new Set<string>();
const knownRecordRequests = new Set<string>();
function sendRecordStep(requestId: string, step: RecordStepPayload): void {
const payload = { type: RECORD_STEP, requestId, step };
const pending = Promise.resolve(chrome.runtime.sendMessage(payload)).then(
() => true,
() => {
failedStepDeliveries.add(requestId);
return false;
},
);
const sends = pendingStepSends.get(requestId) ?? new Set<Promise<boolean>>();
sends.add(pending);
pendingStepSends.set(requestId, sends);
void pending.then(() => {
sends.delete(pending);
if (sends.size === 0) pendingStepSends.delete(requestId);
});
}
export interface RecordCaptureController {
dispose(): void;
}
interface FillSession {
element: FillableElement;
target: TargetDescriptor;
baselineValue: string;
lastValue: string;
}
type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement;
function eventTarget(event: Event): EventTarget | null {
return event.composedPath()[0] ?? event.target;
}
function isOverlayTarget(target: EventTarget | null): boolean {
if (!(target instanceof Node)) return false;
const root = document.documentElement;
let node: Node | null = target;
while (node && node !== root) {
if (node instanceof Element && node.hasAttribute("data-bsk-overlay")) {
return true;
}
const rootNode: Node | Document | ShadowRoot = node.getRootNode();
if (rootNode instanceof ShadowRoot) {
const host: Element = rootNode.host;
if (host.hasAttribute("data-bsk-overlay")) {
return true;
}
node = host;
} else {
node = node.parentNode;
}
}
return false;
}
function isTextFillable(el: Element): el is FillableElement {
if (el instanceof HTMLElement && (el.isContentEditable || el.contentEditable === "true")) {
return true;
}
if (el instanceof HTMLTextAreaElement) return true;
if (!(el instanceof HTMLInputElement)) return false;
const type = el.type.toLowerCase();
return type !== "checkbox" && type !== "radio" && type !== "file" && type !== "button";
}
function fillableFromTarget(target: EventTarget | null): FillableElement | null {
if (!(target instanceof Element)) return null;
if (isTextFillable(target)) return target;
const el = target.closest('input,textarea,[contenteditable]:not([contenteditable="false"])');
if (el && isTextFillable(el)) return el;
return null;
}
/** Clicks on search chrome that only focus the nearby input should not become steps. */
function nearbyFillableFromSearchChrome(target: Element): FillableElement | null {
const container = target.closest(
'[id*="chat-input"], [id*="search"], [class*="search"], form, [role="search"]',
);
if (!container) return null;
const fillable = container.querySelector(
'textarea, input[type="search"], input[name="q"], #chat-textarea',
);
if (
fillable instanceof HTMLElement &&
isTextFillable(fillable) &&
fillable !== target &&
!fillable.contains(target)
) {
return fillable;
}
return null;
}
function fillableValue(el: FillableElement): string {
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
return el.value;
}
return el.textContent ?? "";
}
/** Clicks that only pick an autocomplete/suggestion value — not a semantic submit action. */
function isInputCompletionClick(target: EventTarget | null, session: FillSession | null): boolean {
if (!session || !(target instanceof Element)) return false;
if (session.element.contains(target)) return false;
const option = target.closest('[role="option"]');
if (option?.closest('[role="listbox"]')) return true;
const suggestionRoot = target.closest(
[
'[id*="Sug"]',
'[id*="sug"]',
'[class*="suggest"]',
'[class*="autocomplete"]',
'[class*="typeahead"]',
"[data-autocomplete]",
].join(", "),
);
if (suggestionRoot && !suggestionRoot.contains(session.element)) return true;
return false;
}
function scheduleInputCompletionCommit(
sessionElement: FillableElement,
syncFillSessionValue: (el: FillableElement) => void,
commitFillSession: () => void,
hasSessionFor: (el: FillableElement) => boolean,
): void {
const syncAndCommit = () => {
if (!hasSessionFor(sessionElement)) return;
syncFillSessionValue(sessionElement);
commitFillSession();
};
syncFillSessionValue(sessionElement);
queueMicrotask(syncAndCommit);
setTimeout(syncAndCommit, 0);
}
export function startRecordCapture(
_requestId: string,
sendStep: (step: RecordStepPayload) => void,
): RecordCaptureController {
const emitStep = (step: RecordStepPayload) => {
sendStep({ page_url: location.href, ...step });
};
let fillSession: FillSession | null = null;
let composing = false;
let lastUrl = location.href;
let keyboardActivation: { target: EventTarget | null; recordedAt: number } | undefined;
let generatedControlClick: Element | null = null;
let navigationActionPending = false;
let navigationActionVersion = 0;
const committedValues = new WeakMap<FillableElement, string>();
const markNavigationAction = () => {
navigationActionPending = true;
const version = ++navigationActionVersion;
setTimeout(() => {
if (navigationActionVersion === version) {
navigationActionPending = false;
}
}, 0);
};
const emitFill = (session: FillSession) => {
if (session.lastValue === session.baselineValue) return;
const isPassword =
session.element instanceof HTMLInputElement && session.element.type === "password";
const value = isPassword ? "***" : session.lastValue;
emitStep({
op: "fill",
target: session.target,
value,
...(isPassword ? { redacted: true } : {}),
});
};
const commitFillSession = () => {
if (!fillSession || composing) return;
const session = fillSession;
fillSession = null;
emitFill(session);
committedValues.set(session.element, session.lastValue);
};
const syncFillSessionValue = (el: FillableElement) => {
if (!fillSession || fillSession.element !== el) return;
fillSession.lastValue = fillableValue(el);
};
const ensureFillSession = (el: FillableElement) => {
if (fillSession?.element === el) return;
if (fillSession) commitFillSession();
const currentValue = fillableValue(el);
const baselineValue = committedValues.get(el) ?? currentValue;
fillSession = {
element: el,
target: describeTarget(el),
baselineValue,
lastValue: currentValue,
};
};
const emitNavigateIfChanged = (causedByAction?: boolean) => {
if (location.href === lastUrl) return;
commitFillSession();
lastUrl = location.href;
emitStep({
op: "navigate",
url: location.href,
...(causedByAction !== undefined ? { navigation_caused_by_action: causedByAction } : {}),
});
};
const emitClick = (event: MouseEvent) => {
// Only record clicks an LLM can re-identify (named interactive controls).
const target = describeEventTarget(eventTarget(event));
if (!target) return;
markNavigationAction();
emitStep({
op: "click",
target,
expects_navigation: true,
});
};
const onClick = (event: MouseEvent) => {
if (event.button !== 0) return;
const target = eventTarget(event);
if (isOverlayTarget(target)) return;
if (event.detail === 0 && generatedControlClick !== null && target === generatedControlClick) {
generatedControlClick = null;
return;
}
if (
event.detail === 0 &&
keyboardActivation?.target === target &&
Date.now() - keyboardActivation.recordedAt < 500
) {
keyboardActivation = undefined;
return;
}
const label = target instanceof Element ? target.closest("label") : null;
const nestedInteractive =
target instanceof Element ? target.closest("a,button,input,select,textarea") : null;
if (label instanceof HTMLLabelElement && !nestedInteractive) {
commitFillSession();
generatedControlClick = label.control;
emitClick(event);
return;
}
const fillable = fillableFromTarget(target);
if (fillable) {
ensureFillSession(fillable);
return;
}
if (target instanceof Element) {
const nearbyFillable = nearbyFillableFromSearchChrome(target);
if (nearbyFillable) {
ensureFillSession(nearbyFillable);
return;
}
}
if (isInputCompletionClick(target, fillSession)) {
markNavigationAction();
const sessionElement = fillSession!.element;
scheduleInputCompletionCommit(
sessionElement,
syncFillSessionValue,
commitFillSession,
(el) => fillSession?.element === el,
);
return;
}
commitFillSession();
if (target instanceof Element && target.closest("select")) return;
emitClick(event);
};
const onFocusIn = (event: FocusEvent) => {
const target = fillableFromTarget(eventTarget(event));
if (target) ensureFillSession(target);
};
const onFocusOut = (event: FocusEvent) => {
const target = fillableFromTarget(eventTarget(event));
if (!target) return;
if (!fillSession || fillSession.element !== target) return;
syncFillSessionValue(target);
commitFillSession();
};
const onInput = (event: Event) => {
const target = fillableFromTarget(eventTarget(event));
if (!target) return;
if (composing) return;
ensureFillSession(target);
syncFillSessionValue(target);
};
const onCompositionStart = () => {
composing = true;
};
const onCompositionEnd = (event: CompositionEvent) => {
composing = false;
const target = fillableFromTarget(eventTarget(event));
if (target) {
ensureFillSession(target);
syncFillSessionValue(target);
}
};
const onChange = (event: Event) => {
commitFillSession();
const target = eventTarget(event);
if (target instanceof HTMLSelectElement) {
const values = Array.from(target.selectedOptions).map((opt) => opt.value);
const labels = Array.from(target.selectedOptions).map((opt) =>
(opt.label || opt.textContent || opt.value).trim(),
);
const desc = describeTarget(target);
markNavigationAction();
emitStep({
op: "select",
target: desc,
values,
labels,
expects_navigation: true,
});
}
};
const onKeyDown = (event: KeyboardEvent) => {
if (isOverlayTarget(eventTarget(event))) return;
const target = eventTarget(event);
const fillable = fillableFromTarget(target);
if (fillable) {
ensureFillSession(fillable);
syncFillSessionValue(fillable);
}
const modifiers = [
...(event.altKey ? (["alt"] as const) : []),
...(event.ctrlKey ? (["ctrl"] as const) : []),
...(event.metaKey ? (["meta"] as const) : []),
...(event.shiftKey ? (["shift"] as const) : []),
];
if (!shouldRecordPress(event.key, modifiers)) {
return;
}
markNavigationAction();
if (event.key === "Enter" || event.key === " ") {
const submitTarget =
event.key === "Enter" && fillable
? fillable
.closest("form")
?.querySelector(
'button:not([type]),button[type="submit"],input[type="submit"],input[type="image"]',
)
: null;
keyboardActivation = {
target: submitTarget ?? target,
recordedAt: Date.now(),
};
}
if (fillable) {
commitFillSession();
}
const desc = describeEventTarget(target);
if (!desc && !event.key) return;
emitStep({
op: "press",
key: event.key,
...(desc ? { target: desc } : {}),
...(modifiers.length ? { modifiers } : {}),
expects_navigation: event.key === "Enter",
});
};
document.addEventListener("click", onClick, true);
document.addEventListener("focusin", onFocusIn, true);
document.addEventListener("focusout", onFocusOut, true);
document.addEventListener("input", onInput, true);
document.addEventListener("compositionstart", onCompositionStart, true);
document.addEventListener("compositionend", onCompositionEnd, true);
document.addEventListener("change", onChange, true);
document.addEventListener("keydown", onKeyDown, true);
const urlObserver = new MutationObserver(() => emitNavigateIfChanged());
urlObserver.observe(document, { subtree: true, childList: true });
const onUrlEvent = () => emitNavigateIfChanged();
window.addEventListener("hashchange", onUrlEvent);
window.addEventListener("popstate", onUrlEvent);
window.addEventListener("pagehide", commitFillSession);
const originalPushState = history.pushState;
const originalReplaceState = history.replaceState;
history.pushState = function (...args: Parameters<History["pushState"]>) {
originalPushState.apply(this, args);
emitNavigateIfChanged(navigationActionPending ? true : undefined);
};
history.replaceState = function (...args: Parameters<History["replaceState"]>) {
originalReplaceState.apply(this, args);
emitNavigateIfChanged(navigationActionPending ? true : undefined);
};
return {
dispose() {
commitFillSession();
document.removeEventListener("click", onClick, true);
document.removeEventListener("focusin", onFocusIn, true);
document.removeEventListener("focusout", onFocusOut, true);
document.removeEventListener("input", onInput, true);
document.removeEventListener("compositionstart", onCompositionStart, true);
document.removeEventListener("compositionend", onCompositionEnd, true);
document.removeEventListener("change", onChange, true);
document.removeEventListener("keydown", onKeyDown, true);
urlObserver.disconnect();
window.removeEventListener("hashchange", onUrlEvent);
window.removeEventListener("popstate", onUrlEvent);
window.removeEventListener("pagehide", commitFillSession);
history.pushState = originalPushState;
history.replaceState = originalReplaceState;
},
};
}
export type RecordContentMessage = RecordStartMessage | RecordStopMessage | RecordCancelMessage;
export function isRecordContentMessage(msg: unknown): msg is RecordContentMessage {
return isRecordStartMessage(msg) || isRecordStopMessage(msg) || isRecordCancelMessage(msg);
}
export function handleRecordContentMessage(
message: RecordContentMessage,
state: {
activeRequestId: string | null;
capture: RecordCaptureController | null;
setActiveRequestId(id: string | null): void;
setCapture(capture: RecordCaptureController | null): void;
onStart(requestId: string): void;
onStop(): void;
},
sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
): boolean {
if (isRecordStartMessage(message)) {
if (!knownRecordRequests.has(message.requestId)) {
knownRecordRequests.add(message.requestId);
failedStepDeliveries.delete(message.requestId);
}
state.capture?.dispose();
state.setCapture(
startRecordCapture(message.requestId, (step) => {
sendRecordStep(message.requestId, step);
}),
);
state.setActiveRequestId(message.requestId);
state.onStart(message.requestId);
sendResponse?.({ ok: true });
return sendResponse !== undefined;
}
if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
// Require an active recording that matches this requestId — otherwise a
// stray STOP/CANCEL (e.g. after teardown) would still run finishStop and
// clear overlay state even though nothing was capturing.
if (!state.activeRequestId || state.activeRequestId !== message.requestId) {
return false;
}
state.capture?.dispose();
state.setCapture(null);
const finishStop = () => {
state.onStop();
state.setActiveRequestId(null);
};
if (isRecordStopMessage(message) && sendResponse) {
const pending = [...(pendingStepSends.get(message.requestId) ?? [])];
void Promise.all(pending).then((delivered) => {
finishStop();
const succeeded = delivered.every(Boolean) && !failedStepDeliveries.has(message.requestId);
if (succeeded) {
failedStepDeliveries.delete(message.requestId);
knownRecordRequests.delete(message.requestId);
}
sendResponse(
succeeded
? { ok: true }
: {
ok: false,
error: "failed to deliver one or more recorded steps",
},
);
});
return true;
}
if (isRecordCancelMessage(message)) {
failedStepDeliveries.delete(message.requestId);
knownRecordRequests.delete(message.requestId);
}
finishStop();
return false;
}
return false;
}
export { RECORD_CANCEL, RECORD_START, RECORD_STEP, RECORD_STOP };
@@ -8,6 +8,7 @@ import {
} from "@/lib/instance-id";
import { startKeepalive } from "@/lib/keepalive";
import {
OVERLAY_AUTOMATION_BYPASS,
OVERLAY_MSG_INTERRUPT,
OVERLAY_MSG_WHO_AM_I,
type OverlayInterruptRequest,
@@ -27,6 +28,11 @@ import {
requestBorrowConfirmation,
} from "@/tools/borrow-confirmation";
import { ToolDispatcher } from "@/tools/dispatcher";
import {
attachRecordFinishListener,
attachRecordQueryListener,
attachRecordStepListener,
} from "@/tools/record";
import { detectBrowserMeta } from "@/transport/handshake";
import type { Transport } from "@/transport/transport";
import { WSTransport } from "@/transport/ws-transport";
@@ -67,6 +73,27 @@ export default defineBackground(() => {
}),
});
dispatcher.start();
const recordDeps = {
tabsApi: chrome.tabs,
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
chrome.tabs.sendMessage(tabId, msg),
bypassOverlay: async (tabId: number, enabled: boolean) => {
try {
await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
};
// Message listeners stay up (cheap; fire only for record message types).
// Tab / webNavigation observation attaches lazily while a recording is
// active — see ensureBrowserObservationListeners in tools/record.ts.
attachRecordStepListener();
attachRecordFinishListener(recordDeps);
attachRecordQueryListener(recordDeps);
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
attachBorrowNotificationClickHandler({
onClicked: chrome.notifications.onClicked,
+101 -5
View File
@@ -6,7 +6,13 @@ import { BorrowConfirmationOverlay } from "@/content/BorrowConfirmationOverlay";
import { ControlOverlay } from "@/content/ControlOverlay";
import { HelpRequestOverlay } from "@/content/HelpRequestOverlay";
import overlayCss from "@/content/overlay.css?inline";
import { OverlayController } from "@/content/overlay-controller";
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
import { RecordOverlay } from "@/content/RecordOverlay";
import {
handleRecordContentMessage,
isRecordContentMessage,
type RecordCaptureController,
} from "@/content/record-capture";
import {
HELP_RESPONSE,
type HelpCancelMessage,
@@ -24,6 +30,12 @@ import {
type OverlayWhoAmIResponse,
} from "@/lib/overlay-bridge";
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
import {
RECORD_FINISH,
RECORD_QUERY,
type RecordStartAck,
type RecordStopAck,
} from "@/lib/record-bridge";
import { SESSIONS_LIVE_FLAG_KEY } from "@/lib/sessions-live-flag";
import type {
BorrowCancelMessage,
@@ -45,6 +57,8 @@ export default defineContentScript({
const overlays = new OverlayController();
let activeHelpRespond: ((outcome: "continued" | "cancelled", note?: string) => void) | null =
null;
let recordCapture: RecordCaptureController | null = null;
let activeRecordRequestId: string | null = null;
let reactRoot: ReactDOM.Root | null = null;
let overlayHost: HTMLElement | null = null;
let hostLossReported = false;
@@ -87,12 +101,13 @@ export default defineContentScript({
requests: overlayState.borrowRequests,
}),
React.createElement(ControlOverlay, {
visible: overlayState.controlVisible && overlayState.activeHelp === null,
visible: shouldShowAgentControlOverlay(overlayState),
interrupting: overlayState.interrupting,
automationBypass: overlayState.automationBypassCount > 0,
onInterrupt: handleInterrupt,
}),
React.createElement(HelpRequestOverlay, { request: overlayState.activeHelp }),
React.createElement(RecordOverlay, { request: overlayState.activeRecord }),
),
),
);
@@ -104,6 +119,9 @@ export default defineContentScript({
activeHelpRespond?.("cancelled");
activeHelpRespond = null;
}
recordCapture?.dispose();
recordCapture = null;
activeRecordRequestId = null;
renderOverlay();
}
@@ -140,6 +158,42 @@ export default defineContentScript({
_sender: chrome.runtime.MessageSender,
sendResponse: (response: BorrowResponseMessage | HelpResponseMessage) => void,
) => {
if (isRecordContentMessage(message)) {
const needsAsync = handleRecordContentMessage(
message,
{
activeRequestId: activeRecordRequestId,
capture: recordCapture,
setActiveRequestId: (id) => {
activeRecordRequestId = id;
},
setCapture: (capture) => {
recordCapture = capture;
},
onStart: (requestId) => {
overlays.setAgentRecordRequest({
id: requestId,
onFinish: () => {
void chrome.runtime.sendMessage({
type: RECORD_FINISH,
requestId,
});
},
});
renderOverlay();
},
onStop: () => {
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
renderOverlay();
},
},
sendResponse as unknown as
| ((response: RecordStartAck | RecordStopAck) => void)
| undefined,
);
return needsAsync;
}
if (
message &&
typeof message === "object" &&
@@ -228,23 +282,59 @@ export default defineContentScript({
return false;
};
async function mountOverlayIfAgent(): Promise<void> {
async function syncAgentOverlay(): Promise<void> {
if (!(await anySessionLive())) return;
try {
const reply = (await chrome.runtime.sendMessage({
kind: OVERLAY_MSG_WHO_AM_I,
})) as OverlayWhoAmIResponse | undefined;
if (!reply?.sessionId) return;
// Query / re-arm recording *before* activating the control overlay so
// record start (navigate → content load) does not flash
// 「Agent 正在控制」before RecordOverlay mounts.
let recordQuery: { active?: boolean; requestId?: string } | undefined;
try {
recordQuery = (await chrome.runtime.sendMessage({
type: RECORD_QUERY,
})) as { active?: boolean; requestId?: string } | undefined;
} catch (err) {
console.debug("[bsk overlay] record query failed", err);
}
if (
recordQuery?.active &&
typeof recordQuery.requestId === "string" &&
overlays.snapshot().activeRecord === null
) {
const requestId = recordQuery.requestId;
overlays.setAgentRecordRequest({
id: requestId,
onFinish: () => {
void chrome.runtime.sendMessage({
type: RECORD_FINISH,
requestId,
});
},
});
}
overlays.activateAgentSession(reply.sessionId);
renderOverlay();
} catch (err) {
console.debug("[bsk overlay] who_am_i failed", err);
console.debug("[bsk overlay] syncAgentOverlay failed", err);
}
}
const onPageShow = (event: PageTransitionEvent) => {
if (event.persisted) void syncAgentOverlay();
};
ui.mount();
chrome.runtime.onMessage.addListener(onMessage);
void mountOverlayIfAgent();
void syncAgentOverlay();
window.addEventListener("pageshow", onPageShow);
const hostObserver = new MutationObserver(() => {
const connected = overlayHost?.isConnected ?? false;
@@ -254,6 +344,7 @@ export default defineContentScript({
remountInProgress = true;
try {
ui.mount();
void syncAgentOverlay();
} finally {
remountInProgress = false;
}
@@ -268,6 +359,11 @@ export default defineContentScript({
ctx.onInvalidated(() => {
hostObserver.disconnect();
chrome.runtime.onMessage.removeListener(onMessage);
window.removeEventListener("pageshow", onPageShow);
// Restore history hooks / remove capture listeners before the CS unloads.
recordCapture?.dispose();
recordCapture = null;
activeRecordRequestId = null;
});
},
});
@@ -0,0 +1,117 @@
import { beforeEach, describe, expect, it } from "vitest";
import {
describeEventTarget,
describeTarget,
isMeaningfulClickTarget,
} from "../describe-target";
describe("describeTarget", () => {
beforeEach(() => {
document.body.innerHTML = "";
});
it("prefers aria-label and button role", () => {
document.body.innerHTML = `<button aria-label="发布">go</button>`;
const el = document.querySelector("button")!;
expect(describeTarget(el)).toEqual({
tag: "button",
role: "button",
name: "发布",
});
});
it("uses label text and name_attr for inputs", () => {
document.body.innerHTML = `
<label for="svc">服务名称</label>
<input id="svc" name="serviceName" />
`;
const el = document.querySelector("input")!;
expect(describeTarget(el)).toEqual({
tag: "input",
role: "textbox",
name: "服务名称",
name_attr: "serviceName",
});
});
it("never invents CSS selectors or @eN", () => {
document.body.innerHTML = `<a href="/x">详情</a>`;
const desc = describeTarget(document.querySelector("a")!);
expect(JSON.stringify(desc)).not.toMatch(/@e\d+|nth-of-type|#/);
expect(desc).toEqual({ tag: "a", role: "link", name: "详情" });
});
it("prefers heading text inside SERP-style links", () => {
document.body.innerHTML = `
<a href="https://www.tencent.com">
<h3>Tencent 腾讯</h3>
<cite>https://www.tencent.com</cite>
</a>
`;
expect(describeTarget(document.querySelector("a")!).name).toBe("Tencent 腾讯");
});
});
describe("LLM textbook click targets", () => {
beforeEach(() => {
document.body.innerHTML = "";
});
it("rejects tag-only targets the LLM cannot find", () => {
expect(isMeaningfulClickTarget({ tag: "div" })).toBe(false);
});
it("treats a short visible name as the teaching signal", () => {
expect(isMeaningfulClickTarget({ tag: "div", name: "发布" })).toBe(true);
});
it("ignores anonymous layout chrome and lifts child clicks to the named button", () => {
document.body.innerHTML = `
<div id="noise">background</div>
<button type="button"><span id="icon">发布</span></button>
`;
expect(describeEventTarget(document.querySelector("#noise"))).toBeNull();
expect(describeEventTarget(document.querySelector("#icon"))).toEqual({
tag: "button",
role: "button",
name: "发布",
});
});
it("does not attach layout sibling text as nearby_label on buttons", () => {
document.body.innerHTML = `
<div>background</div>
<button type="button" id="pub">发布</button>
`;
expect(describeTarget(document.querySelector("#pub")!)).toEqual({
tag: "button",
role: "button",
name: "发布",
});
});
it("keeps nearby_label for form fields from a preceding label-like sibling", () => {
document.body.innerHTML = `
<div>服务名称</div>
<input id="svc" />
`;
expect(describeTarget(document.querySelector("#svc")!)).toEqual(
expect.objectContaining({
tag: "input",
nearby_label: "服务名称",
}),
);
});
it("does not treat plain text blocks as clicks", () => {
document.body.innerHTML = `<p id="copy">说明文字</p>`;
expect(describeEventTarget(document.querySelector("#copy"))).toBeNull();
});
it("records custom button-like controls by their visible label", () => {
document.body.innerHTML = `<div class="btn" id="pub" style="cursor: pointer">发布</div>`;
expect(describeEventTarget(document.querySelector("#pub"))).toEqual(
expect.objectContaining({ tag: "div", name: "发布" }),
);
});
});
@@ -0,0 +1,80 @@
import { describe, expect, it } from "vitest";
import { appendRecordedPayload, observeRecordedNavigation } from "../recording-step-buffer";
describe("recording-step-buffer", () => {
it("stores semantic click without summary", () => {
const buffer = { steps: [], pendingNavigation: false };
appendRecordedPayload(buffer, {
op: "click",
target: { tag: "button", role: "button", name: "发布" },
expects_navigation: true,
});
expect(buffer.steps).toEqual([
{
op: "click",
target: { tag: "button", role: "button", name: "发布" },
},
]);
expect(buffer.pendingNavigation).toBe(true);
});
it("annotates navigated_to on action-caused navigation instead of wait_for_navigation", () => {
const buffer = {
steps: [
{
op: "click" as const,
target: { tag: "button", role: "button", name: "发布" },
},
],
currentUrl: "https://example.com/a",
pendingNavigation: true,
pendingNavigationDeadline: Date.now() + 5_000,
};
observeRecordedNavigation(buffer, "https://example.com/b", true);
expect(buffer.steps).toEqual([
{
op: "click",
target: { tag: "button", role: "button", name: "发布" },
navigated_to: "https://example.com/b",
},
]);
expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation");
});
it("annotates navigated_to onto a select that auto-submits a navigation", () => {
const buffer = {
steps: [
{
op: "select" as const,
target: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
},
],
currentUrl: "https://example.com/list",
pendingNavigation: true,
pendingNavigationDeadline: Date.now() + 5_000,
};
observeRecordedNavigation(buffer, "https://example.com/list?cat=tech", true);
expect(buffer.steps).toEqual([
{
op: "select",
target: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
navigated_to: "https://example.com/list?cat=tech",
},
]);
});
it("emits navigate for uncaused URL changes", () => {
const buffer = {
steps: [],
currentUrl: "https://example.com/a",
pendingNavigation: false,
};
observeRecordedNavigation(buffer, "https://example.com/b", false);
expect(buffer.steps[0]).toMatchObject({
op: "navigate",
url: "https://example.com/b",
});
});
});
@@ -0,0 +1,141 @@
import { describe, expect, it } from "vitest";
import type { DraftTraceStep } from "@/transport/types";
import { reduceTraceSteps, resolveTraceStartUrl, shouldRecordPress } from "../trace-reducer";
describe("shouldRecordPress", () => {
it("keeps Enter and Escape", () => {
expect(shouldRecordPress("Enter")).toBe(true);
expect(shouldRecordPress("Escape")).toBe(true);
});
it("drops modifiers, clipboard shortcuts, and bare typing", () => {
expect(shouldRecordPress("Meta")).toBe(false);
expect(shouldRecordPress("c", ["meta"])).toBe(false);
expect(shouldRecordPress("a", ["ctrl"])).toBe(false);
expect(shouldRecordPress("x")).toBe(false);
expect(shouldRecordPress("中")).toBe(false);
});
});
describe("reduceTraceSteps", () => {
it("builds steps with pages dictionary and page id references", () => {
const drafts: DraftTraceStep[] = [
{
op: "navigate",
url: "https://example.com/search?q=hello&utm_source=x",
page_url: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "fill",
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
value: "browser skill",
page_url: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "press",
key: "Enter",
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
navigated_to: "https://example.com/results/42",
page_url: "https://example.com/search?q=hello&utm_source=x",
},
{
op: "click",
target: { tag: "button", role: "button", name: "发布" },
navigated_to: "https://example.com/p/99",
page_url: "https://example.com/results/42",
},
{
op: "press",
key: "a",
page_url: "https://example.com/p/99",
},
];
const { pages, steps } = reduceTraceSteps(
drafts,
"https://example.com/search?q=hello&utm_source=x",
);
expect(JSON.stringify(steps)).not.toContain("parameters");
expect(JSON.stringify(steps)).not.toContain("intent");
expect(JSON.stringify(steps)).not.toContain("summary");
expect(steps.map((s) => s.op)).toEqual(["navigate", "fill", "press", "click"]);
expect(pages.map((p) => p.url)).toEqual([
"https://example.com/search?q=hello&utm_source=x",
"https://example.com/results/42",
"https://example.com/p/99",
]);
expect(steps[0]).toMatchObject({
id: 1,
op: "navigate",
page: "p1",
to: "https://example.com/search?q=hello&utm_source=x",
});
expect(steps[1]).toMatchObject({
op: "fill",
page: "p1",
value: "browser skill",
});
expect(steps[2]).toMatchObject({
op: "press",
key: "Enter",
page: "p1",
effect: { navigated_to: "p2" },
});
expect(steps[3]).toMatchObject({
op: "click",
page: "p2",
effect: { navigated_to: "p3" },
});
});
it("collapses consecutive navigations", () => {
const { steps } = reduceTraceSteps([
{ op: "navigate", url: "https://a.example/redirect1" },
{ op: "navigate", url: "https://a.example/final" },
]);
expect(steps).toHaveLength(1);
expect(steps[0]).toMatchObject({
op: "navigate",
to: "https://a.example/final",
});
});
it("maps select navigated_to onto effect.navigated_to (page id)", () => {
const { pages, steps } = reduceTraceSteps(
[
{
op: "select",
target: { tag: "select", role: "combobox", name: "分类" },
values: ["tech"],
labels: ["技术"],
navigated_to: "https://example.com/list?cat=tech",
page_url: "https://example.com/list",
},
],
"https://example.com/list",
);
expect(pages.map((p) => p.url)).toEqual([
"https://example.com/list",
"https://example.com/list?cat=tech",
]);
expect(steps[0]).toMatchObject({
op: "select",
page: "p1",
selection: [{ value: "tech", label: "技术" }],
effect: { navigated_to: "p2" },
});
});
it("resolveTraceStartUrl prefers explicit start URL", () => {
expect(
resolveTraceStartUrl(
[{ op: "navigate", url: "https://example.com/other" }],
"https://example.com/start",
),
).toBe("https://example.com/start");
});
});
+342
View File
@@ -0,0 +1,342 @@
/**
* Build a semantic TargetDescriptor for an interacted element.
*
* Trace steps are an LLM *textbook*: each click must say what to look for
* on screen (usually a short visible name). Tag-only noise like
* `{ "tag": "div" }` / “点击div” is useless and must not be recorded.
*/
export interface TargetDescriptor {
role?: string;
name?: string;
tag: string;
name_attr?: string;
placeholder?: string;
nearby_label?: string;
}
/** Max length for a label that is still a useful “find this on screen” hint. */
const ACTIONABLE_LABEL_MAX = 48;
function normalizeWhitespace(value: string): string {
return value.replace(/\s+/g, " ").trim();
}
function truncate(value: string, max = 80): string {
const trimmed = normalizeWhitespace(value);
if (trimmed.length <= max) return trimmed;
return `${trimmed.slice(0, max - 1)}…`;
}
/** Short, human label an LLM can search for in a snapshot. */
export function isActionableLabel(name: string): boolean {
const trimmed = normalizeWhitespace(name);
if (!trimmed) return false;
if (trimmed.length > ACTIONABLE_LABEL_MAX) return false;
if (/^https?:\/\//i.test(trimmed)) return false;
return true;
}
function labelledByText(el: Element): string | null {
const ids = el.getAttribute("aria-labelledby");
if (!ids) return null;
const parts = ids
.split(/\s+/)
.map((id) => document.getElementById(id)?.textContent ?? "")
.map(normalizeWhitespace)
.filter(Boolean);
return parts.length ? truncate(parts.join(" ")) : null;
}
function associatedLabelText(el: Element): string | null {
if (!(el instanceof HTMLElement) || !el.id) return null;
const label = document.querySelector(`label[for="${CSS.escape(el.id)}"]`);
if (!label) return null;
const text = normalizeWhitespace(label.textContent ?? "");
return text ? truncate(text) : null;
}
function wrappingLabelText(el: Element): string | null {
const label = el.closest("label");
if (!label) return null;
const clone = label.cloneNode(true) as HTMLElement;
for (const control of clone.querySelectorAll("input,textarea,select,button")) {
control.remove();
}
const text = normalizeWhitespace(clone.textContent ?? "");
return text ? truncate(text) : null;
}
function placeholderOrTitle(el: Element): string | null {
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
const ph = normalizeWhitespace(el.placeholder);
if (ph) return truncate(ph);
}
const title = normalizeWhitespace(el.getAttribute("title") ?? "");
return title ? truncate(title) : null;
}
function ownText(el: Element): string | null {
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
return null;
}
const text = normalizeWhitespace(el.textContent ?? "");
return text ? truncate(text) : null;
}
/** Prefer a compact title inside links (e.g. SERP result `<h3>`), not the whole blob. */
function compactLinkName(el: Element): string | null {
const heading = el.querySelector("h1, h2, h3, h4");
if (heading) {
const text = normalizeWhitespace(heading.textContent ?? "");
if (text) return truncate(text);
}
const img = el.querySelector("img[alt]");
if (img instanceof HTMLImageElement) {
const alt = normalizeWhitespace(img.alt);
if (alt) return truncate(alt);
}
const clone = el.cloneNode(true) as HTMLElement;
for (const junk of clone.querySelectorAll("cite, script, style, svg, noscript")) {
junk.remove();
}
const text = normalizeWhitespace(clone.textContent ?? "");
if (!text) return null;
const withoutUrl = text.replace(/https?:\/\/\S+/g, "").trim();
return truncate(withoutUrl || text);
}
/** Visible accessible name preference order for LLM textbooks. */
export function accessibleName(el: Element): string | undefined {
const ariaLabel = normalizeWhitespace(el.getAttribute("aria-label") ?? "");
if (ariaLabel) return truncate(ariaLabel);
const labelledBy = labelledByText(el);
if (labelledBy) return labelledBy;
const forLabel = associatedLabelText(el);
if (forLabel) return forLabel;
const wrapLabel = wrappingLabelText(el);
if (wrapLabel) return wrapLabel;
const alt =
el instanceof HTMLImageElement
? normalizeWhitespace(el.alt)
: normalizeWhitespace(el.getAttribute("alt") ?? "");
if (alt) return truncate(alt);
const ph = placeholderOrTitle(el);
if (ph) return ph;
const tag = el.tagName.toLowerCase();
if (tag === "a" || el.getAttribute("role") === "link") {
const linkName = compactLinkName(el);
if (linkName) return linkName;
}
const text = ownText(el);
if (text) return text;
if (el instanceof HTMLInputElement || el instanceof HTMLButtonElement) {
const value = normalizeWhitespace(el.value);
if (value && (el instanceof HTMLButtonElement || el.type === "submit" || el.type === "button")) {
return truncate(value);
}
}
return undefined;
}
const INTERACTIVE_SELECTOR = [
"a[href]",
"button",
"summary",
"select",
'input:not([type="hidden"])',
"textarea",
'[role="button"]',
'[role="link"]',
'[role="tab"]',
'[role="menuitem"]',
'[role="menuitemcheckbox"]',
'[role="menuitemradio"]',
'[role="option"]',
'[role="checkbox"]',
'[role="radio"]',
'[role="switch"]',
'[role="treeitem"]',
'[role="combobox"]',
'[contenteditable]:not([contenteditable="false"])',
].join(",");
const INTERACTIVE_ROLES = new Set([
"button",
"link",
"tab",
"menuitem",
"menuitemcheckbox",
"menuitemradio",
"option",
"checkbox",
"radio",
"switch",
"treeitem",
"combobox",
"textbox",
"searchbox",
"slider",
]);
function looksClickable(el: Element): boolean {
if (!(el instanceof HTMLElement)) return false;
if (el.matches(INTERACTIVE_SELECTOR)) return true;
const role = normalizeWhitespace(el.getAttribute("role") ?? "").toLowerCase();
if (role && INTERACTIVE_ROLES.has(role)) return true;
if (el.tabIndex >= 0) return true;
try {
if (getComputedStyle(el).cursor === "pointer") return true;
} catch {
// happy-dom / detached nodes
}
const cls = typeof el.className === "string" ? el.className : "";
if (/\b(btn|button|link|clickable|action)\b/i.test(cls)) return true;
return false;
}
/**
* Resolve the element a click should describe for the LLM textbook.
* Never falls back to an anonymous layout `div`.
*/
export function resolveClickableElement(target: Element): Element | null {
const interactive = target.closest(INTERACTIVE_SELECTOR);
if (interactive instanceof Element) return interactive;
let node: Element | null = target;
let depth = 0;
while (node && node !== document.body && node !== document.documentElement && depth < 8) {
const name = accessibleName(node);
if (name && isActionableLabel(name) && looksClickable(node)) {
return node;
}
node = node.parentElement;
depth += 1;
}
return null;
}
/**
* Teachable clicks: the LLM must get a label it can later find via snapshot
* (visible name), or at least a form `name_attr` for checkbox/radio.
* Recording “点击div” with no name fails this bar.
*/
export function isMeaningfulClickTarget(target: TargetDescriptor): boolean {
const name = target.name?.trim();
if (name && isActionableLabel(name)) return true;
if (
(target.role === "checkbox" || target.role === "radio" || target.tag === "input") &&
target.name_attr
) {
return true;
}
return false;
}
export function inferRole(el: Element): string | undefined {
const explicit = normalizeWhitespace(el.getAttribute("role") ?? "").toLowerCase();
if (explicit) return explicit;
const tag = el.tagName.toLowerCase();
if (tag === "button") return "button";
if (tag === "a" && el.hasAttribute("href")) return "link";
if (tag === "select") return "combobox";
if (tag === "textarea") return "textbox";
if (tag === "img") return "img";
if (tag === "summary") return "button";
if (el instanceof HTMLElement && (el.isContentEditable || el.contentEditable === "true")) {
return "textbox";
}
if (el instanceof HTMLInputElement) {
const type = el.type.toLowerCase();
if (type === "checkbox") return "checkbox";
if (type === "radio") return "radio";
if (type === "submit" || type === "button" || type === "image" || type === "reset") {
return "button";
}
if (type === "range") return "slider";
return "textbox";
}
return undefined;
}
/**
* Nearby labels help the LLM name form fields. Only apply to fillable
* controls — never to buttons/links, where a previous layout sibling
* (e.g. a chrome `<div>`) would pollute the textbook target.
*/
function nearbyLabelText(el: Element): string | undefined {
if (
!(
el instanceof HTMLInputElement ||
el instanceof HTMLTextAreaElement ||
el instanceof HTMLSelectElement
)
) {
return undefined;
}
if (el.id) {
const byFor = document.querySelector(`label[for="${CSS.escape(el.id)}"]`);
const text = byFor ? normalizeWhitespace(byFor.textContent ?? "") : "";
if (text) return truncate(text, ACTIONABLE_LABEL_MAX);
}
const wrapped = el.closest("label");
if (wrapped) {
const clone = wrapped.cloneNode(true) as HTMLLabelElement;
for (const control of clone.querySelectorAll("input, textarea, select, button")) {
control.remove();
}
const text = normalizeWhitespace(clone.textContent ?? "");
if (text) return truncate(text, ACTIONABLE_LABEL_MAX);
}
const prev = el.previousElementSibling;
if (prev && /^(LABEL|SPAN|DIV|P|DT)$/i.test(prev.tagName)) {
const text = normalizeWhitespace(prev.textContent ?? "");
if (text && text.length <= ACTIONABLE_LABEL_MAX) return text;
}
return undefined;
}
export function describeTarget(el: Element): TargetDescriptor {
const tag = el.tagName.toLowerCase();
const role = inferRole(el);
const name = accessibleName(el);
const nameAttr =
el instanceof HTMLInputElement ||
el instanceof HTMLTextAreaElement ||
el instanceof HTMLSelectElement ||
el instanceof HTMLButtonElement
? normalizeWhitespace(el.name) || undefined
: normalizeWhitespace(el.getAttribute("name") ?? "") || undefined;
const placeholder =
el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement
? normalizeWhitespace(el.placeholder) || undefined
: normalizeWhitespace(el.getAttribute("placeholder") ?? "") || undefined;
const nearby = nearbyLabelText(el);
return {
tag,
...(role ? { role } : {}),
...(name ? { name } : {}),
...(nameAttr ? { name_attr: nameAttr } : {}),
...(placeholder ? { placeholder } : {}),
...(nearby && nearby !== name ? { nearby_label: nearby } : {}),
};
}
export function describeEventTarget(target: EventTarget | null): TargetDescriptor | null {
if (!(target instanceof Element)) return null;
const clickable = resolveClickableElement(target);
if (!clickable) return null;
const desc = describeTarget(clickable);
return isMeaningfulClickTarget(desc) ? desc : null;
}
+111
View File
@@ -0,0 +1,111 @@
/**
* Wire protocol for user-action recording, sent between the background
* service worker and a tab's content script.
*/
import type { TargetDescriptor } from "./describe-target";
export const RECORD_START = "bsk-record-start";
export const RECORD_STEP = "bsk-record-step";
export const RECORD_STOP = "bsk-record-stop";
export const RECORD_CANCEL = "bsk-record-cancel";
export const RECORD_FINISH = "bsk-record-finish";
export const RECORD_QUERY = "bsk-record-query";
export interface RecordStartAck {
ok: true;
}
export type RecordStopAck = { ok: true } | { ok: false; error: string };
export interface RecordQueryMessage {
type: typeof RECORD_QUERY;
}
export interface RecordQueryResponse {
active: boolean;
requestId?: string;
}
export interface RecordStartMessage {
type: typeof RECORD_START;
requestId: string;
}
export interface RecordStepPayload {
op: "click" | "fill" | "press" | "select" | "navigate";
target?: TargetDescriptor;
value?: string;
key?: string;
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">;
values?: string[];
labels?: string[];
url?: string;
redacted?: boolean;
/** Page URL when the step was captured. */
page_url?: string;
/** Capture-only hint; never persisted unless converted to navigated_to. */
expects_navigation?: boolean;
/** Whether an observed URL change was synchronously caused by the action. */
navigation_caused_by_action?: boolean;
}
export interface RecordStepMessage {
type: typeof RECORD_STEP;
requestId: string;
step: RecordStepPayload;
}
export interface RecordStopMessage {
type: typeof RECORD_STOP;
requestId: string;
}
export interface RecordCancelMessage {
type: typeof RECORD_CANCEL;
requestId: string;
}
export interface RecordFinishMessage {
type: typeof RECORD_FINISH;
requestId: string;
}
export function isRecordStartMessage(msg: unknown): msg is RecordStartMessage {
if (typeof msg !== "object" || msg === null) return false;
const m = msg as Record<string, unknown>;
return m.type === RECORD_START && typeof m.requestId === "string";
}
export function isRecordStopMessage(msg: unknown): msg is RecordStopMessage {
if (typeof msg !== "object" || msg === null) return false;
const m = msg as Record<string, unknown>;
return m.type === RECORD_STOP && typeof m.requestId === "string";
}
export function isRecordCancelMessage(msg: unknown): msg is RecordCancelMessage {
if (typeof msg !== "object" || msg === null) return false;
const m = msg as Record<string, unknown>;
return m.type === RECORD_CANCEL && typeof m.requestId === "string";
}
export function isRecordFinishMessage(msg: unknown): msg is RecordFinishMessage {
if (typeof msg !== "object" || msg === null) return false;
const m = msg as Record<string, unknown>;
return m.type === RECORD_FINISH && typeof m.requestId === "string";
}
export function isRecordQueryMessage(msg: unknown): msg is RecordQueryMessage {
if (typeof msg !== "object" || msg === null) return false;
const m = msg as Record<string, unknown>;
return m.type === RECORD_QUERY;
}
export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage {
if (typeof msg !== "object" || msg === null) return false;
const m = msg as Record<string, unknown>;
if (m.type !== RECORD_STEP || typeof m.requestId !== "string") return false;
const step = m.step;
if (typeof step !== "object" || step === null) return false;
return typeof (step as RecordStepPayload).op === "string";
}
@@ -0,0 +1,128 @@
import type { DraftTraceStep } from "@/transport/types";
import type { RecordStepPayload } from "./record-bridge";
export interface RecordingStepBuffer {
steps: DraftTraceStep[];
currentUrl?: string;
pendingNavigation: boolean;
pendingNavigationDeadline?: number;
}
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
const pageUrl = payload.page_url;
switch (payload.op) {
case "click":
return payload.target
? {
op: "click",
target: payload.target,
...(pageUrl ? { page_url: pageUrl } : {}),
}
: null;
case "fill":
return payload.target
? {
op: "fill",
target: payload.target,
value: payload.value ?? "",
...(payload.redacted ? { redacted: true } : {}),
...(pageUrl ? { page_url: pageUrl } : {}),
}
: null;
case "press":
return payload.key
? {
op: "press",
key: payload.key,
...(payload.target ? { target: payload.target } : {}),
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
...(pageUrl ? { page_url: pageUrl } : {}),
}
: null;
case "select":
return payload.target && payload.values
? {
op: "select",
target: payload.target,
values: payload.values,
...(payload.labels?.length ? { labels: payload.labels } : {}),
...(pageUrl ? { page_url: pageUrl } : {}),
}
: null;
case "navigate":
return null;
}
}
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): boolean {
for (let i = buffer.steps.length - 1; i >= 0; i -= 1) {
const step = buffer.steps[i];
if (!step) continue;
if (step.op === "click" || step.op === "press" || step.op === "select") {
buffer.steps[i] = { ...step, navigated_to: url };
return true;
}
break;
}
return false;
}
export function observeRecordedNavigation(
buffer: RecordingStepBuffer,
url: string,
causedByAction?: boolean,
): void {
if (!url || url === buffer.currentUrl) return;
buffer.currentUrl = url;
const pendingIsCurrent =
buffer.pendingNavigation &&
(buffer.pendingNavigationDeadline === undefined ||
buffer.pendingNavigationDeadline >= Date.now());
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
buffer.pendingNavigation = false;
buffer.pendingNavigationDeadline = undefined;
if (!annotateLastStepNavigation(buffer, url)) {
buffer.steps.push({
op: "navigate",
url,
page_url: url,
});
}
return;
}
buffer.pendingNavigation = false;
buffer.pendingNavigationDeadline = undefined;
buffer.steps.push({
op: "navigate",
url,
page_url: url,
});
}
export function appendRecordedPayload(
buffer: RecordingStepBuffer,
payload: RecordStepPayload,
): void {
if (payload.op === "navigate") {
if (payload.url) {
observeRecordedNavigation(buffer, payload.url, payload.navigation_caused_by_action);
}
return;
}
const step = toDraftStep({
...payload,
page_url: payload.page_url ?? buffer.currentUrl,
});
if (!step) return;
buffer.steps.push(step);
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
buffer.pendingNavigation = payload.expects_navigation === true;
buffer.pendingNavigationDeadline = buffer.pendingNavigation
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
: undefined;
}
}
+218
View File
@@ -0,0 +1,218 @@
import type { DraftTraceStep, PageRef, SelectedOption, Step } from "@/transport/types";
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
export function shouldRecordPress(
key: string,
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
): boolean {
if (MODIFIER_ONLY_KEYS.has(key)) return false;
const mods = modifiers ?? [];
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
if (key === "Enter" || key === "Escape") return true;
// Drop bare character typing — FillSession already records the value.
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
return false;
}
function shouldIncludeDraft(step: DraftTraceStep): boolean {
if (step.op === "fill" && !(step.value ?? "").trim() && !step.redacted) return false;
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
return true;
}
/** Collapse consecutive navigations to the last hop. */
function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
const out: DraftTraceStep[] = [];
for (const step of steps) {
const prev = out[out.length - 1];
if (step.op === "navigate" && prev?.op === "navigate") {
out[out.length - 1] = step;
continue;
}
out.push(step);
}
return out;
}
function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
const urls: string[] = [];
if (startUrl) urls.push(startUrl);
for (const step of steps) {
if (step.op === "navigate") {
urls.push(step.url);
continue;
}
if ("page_url" in step && step.page_url) urls.push(step.page_url);
if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to);
}
const seen = new Set<string>();
const unique: string[] = [];
for (const url of urls) {
if (!url || seen.has(url)) continue;
seen.add(url);
unique.push(url);
}
return unique;
}
function buildPageRegistry(
steps: DraftTraceStep[],
startUrl?: string,
): { pages: PageRef[]; urlToId: Map<string, string> } {
const urls = collectUrls(steps, startUrl);
const urlToId = new Map<string, string>();
const pages = urls.map((url, index) => {
const id = `p${index + 1}`;
urlToId.set(url, id);
return { id, url };
});
return { pages, urlToId };
}
function pageIdFor(
url: string | undefined,
urlToId: Map<string, string>,
fallbackUrl?: string,
): string {
if (url && urlToId.has(url)) return urlToId.get(url)!;
if (fallbackUrl && urlToId.has(fallbackUrl)) return urlToId.get(fallbackUrl)!;
return urlToId.values().next().value ?? "p1";
}
function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined {
if (step.op === "navigate") return step.page_url ?? step.url;
if ("page_url" in step && step.page_url) return step.page_url;
return fallbackUrl;
}
function effectForNavigation(
navigatedTo: string | undefined,
urlToId: Map<string, string>,
): Step["effect"] {
if (!navigatedTo) return undefined;
const pageId = urlToId.get(navigatedTo);
if (!pageId) return undefined;
return { navigated_to: pageId };
}
function withEffect(step: Step, effect: Step["effect"]): Step {
if (!effect) return step;
return { ...step, effect };
}
function toSelection(values: string[], labels?: string[]): SelectedOption[] {
return values.map((value, index) => ({
value,
...(labels?.[index] ? { label: labels[index] } : {}),
}));
}
function toV2Step(
step: DraftTraceStep,
id: number,
urlToId: Map<string, string>,
fallbackUrl?: string,
): Step | null {
if (!shouldIncludeDraft(step)) return null;
const pageUrl = pageUrlForDraft(step, fallbackUrl);
const page = pageIdFor(pageUrl, urlToId, fallbackUrl);
switch (step.op) {
case "navigate":
return {
op: "navigate",
id,
page: pageIdFor(step.url, urlToId, fallbackUrl),
to: step.url,
};
case "click":
return withEffect(
{
op: "click",
id,
page,
target: step.target,
},
effectForNavigation(step.navigated_to, urlToId),
);
case "fill":
return {
op: "fill",
id,
page,
target: step.target,
value: step.value,
...(step.redacted ? { redacted: true } : {}),
};
case "press":
return withEffect(
{
op: "press",
id,
page,
key: step.key,
...(step.target ? { target: step.target } : {}),
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
},
effectForNavigation(step.navigated_to, urlToId),
);
case "select":
return withEffect(
{
op: "select",
id,
page,
target: step.target,
selection: toSelection(step.values, step.labels),
},
effectForNavigation(step.navigated_to, urlToId),
);
}
}
export interface ReducedTrace {
pages: PageRef[];
steps: Step[];
}
/**
* Compile capture drafts into record-only trace v2 steps.
* Variable inputs are NOT classified here — executing agents infer that at run time.
*/
export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): ReducedTrace {
const collapsed = collapseNavigations(steps);
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
const out: Step[] = [];
let id = 1;
let lastUrl = startUrl;
for (const draft of collapsed) {
if (draft.op === "navigate") lastUrl = draft.url;
else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to;
else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url;
const step = toV2Step(draft, id, urlToId, lastUrl);
if (!step) continue;
out.push(step);
id += 1;
}
return { pages, steps: out };
}
export function resolveTraceStartUrl(
drafts: DraftTraceStep[],
startUrl?: string,
pages?: PageRef[],
): string {
if (startUrl) return startUrl;
const navigate = drafts.find((step): step is Extract<DraftTraceStep, { op: "navigate" }> => {
return step.op === "navigate";
});
if (navigate) return navigate.url;
for (const draft of drafts) {
if ("page_url" in draft && draft.page_url) return draft.page_url;
}
return pages?.[0]?.url ?? "about:blank";
}
@@ -0,0 +1,52 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import {
ensureBrowserObservationListeners,
isBrowserObservationAttachedForTests,
releaseBrowserObservationListenersIfIdle,
resetBrowserObservationForTests,
setBrowserObservationAttachForTests,
} from "../record";
describe("browser observation lifecycle", () => {
afterEach(() => {
resetBrowserObservationForTests();
});
it("does not attach listeners until ensure is called", () => {
const tab = vi.fn(() => () => undefined);
const nav = vi.fn(() => () => undefined);
setBrowserObservationAttachForTests(tab, nav);
expect(isBrowserObservationAttachedForTests()).toBe(false);
expect(tab).not.toHaveBeenCalled();
expect(nav).not.toHaveBeenCalled();
});
it("attaches once on ensure and detaches when idle", () => {
const detachTab = vi.fn();
const detachNav = vi.fn();
const tab = vi.fn(() => detachTab);
const nav = vi.fn(() => detachNav);
setBrowserObservationAttachForTests(tab, nav);
ensureBrowserObservationListeners({
tabsApi: { get: vi.fn(), query: vi.fn(), create: vi.fn() } as never,
sendToTab: vi.fn(),
});
expect(isBrowserObservationAttachedForTests()).toBe(true);
expect(tab).toHaveBeenCalledTimes(1);
expect(nav).toHaveBeenCalledTimes(1);
// Idempotent.
ensureBrowserObservationListeners({
tabsApi: { get: vi.fn(), query: vi.fn(), create: vi.fn() } as never,
sendToTab: vi.fn(),
});
expect(tab).toHaveBeenCalledTimes(1);
releaseBrowserObservationListenersIfIdle();
expect(isBrowserObservationAttachedForTests()).toBe(false);
expect(detachTab).toHaveBeenCalledTimes(1);
expect(detachNav).toHaveBeenCalledTimes(1);
});
});
+43
View File
@@ -13,6 +13,9 @@ import type {
NetworkParams,
PressParams,
ProtocolFrame,
RecordAwaitParams,
RecordStartParams,
RecordStopParams,
ReloadParams,
RequestFrame,
RequestHelpParams,
@@ -42,6 +45,7 @@ import {
handleScreenshot,
handleSnapshot,
} from "./observation";
import { handleRecordAwait, handleRecordStart, handleRecordStop } from "./record";
import {
handleSessionStart,
handleSessionStop,
@@ -396,6 +400,45 @@ export class ToolDispatcher {
notificationCopy: this.helpNotificationCopy?.(),
signal,
});
case "tool.record_start":
return handleRecordStart(this.sessions, req.params as RecordStartParams, {
tabsApi: chromeTabsApi,
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
bypassOverlay: async (tabId, enabled) => {
try {
await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
...(this.cdp ? { cdp: this.cdp } : {}),
signal,
});
case "tool.record_stop":
return handleRecordStop(this.sessions, req.params as RecordStopParams, {
tabsApi: chromeTabsApi,
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
bypassOverlay: async (tabId, enabled) => {
try {
await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
signal,
});
case "tool.record_await":
return handleRecordAwait(this.sessions, req.params as RecordAwaitParams, {
tabsApi: chromeTabsApi,
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
signal,
});
default:
return {
code: "unknown_method",
+825
View File
@@ -0,0 +1,825 @@
// `tool.record_start` / `tool.record_stop` / `tool.record_await` — capture
// user actions in the Agent Window via the content script and return a
// semantic (LLM textbook) trace.
import {
isRecordFinishMessage,
isRecordQueryMessage,
isRecordStepMessage,
RECORD_CANCEL,
RECORD_START,
RECORD_STEP,
RECORD_STOP,
type RecordCancelMessage,
type RecordFinishMessage,
type RecordQueryResponse,
type RecordStartAck,
type RecordStartMessage,
type RecordStopMessage,
} from "@/lib/record-bridge";
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording-step-buffer";
import { reduceTraceSteps, resolveTraceStartUrl } from "@/lib/trace-reducer";
import type { SessionManager } from "@/session-manager/manager";
import type {
DraftTraceStep,
RecordAwaitParams,
RecordAwaitResult,
RecordStartParams,
RecordStartResult,
RecordStopParams,
RecordStopResult,
RpcError,
Trace,
} from "@/transport/types";
import { handleNavigate } from "./navigation";
import {
type CdpRunner,
type ChromeTabsApi,
chromeTabsApi,
isRpcError,
lookupSession,
resolveTargetTab,
} from "./shared";
interface ActiveRecording {
requestId: string;
tabId: number;
agentWindowId: number;
startUrl?: string;
purpose?: string;
steps: DraftTraceStep[];
startedAt: string;
finishPromise: Promise<Trace>;
resolveFinish: (trace: Trace) => void;
rejectFinish: (err: Error) => void;
settled: boolean;
finishing: boolean;
currentUrl?: string;
pendingNavigation: boolean;
pendingNavigationDeadline?: number;
}
const recordings = new Map<string, ActiveRecording>();
const RECORD_START_RETRIES = 3;
const RECORD_START_RETRY_DELAY_MS = 500;
const RECORD_REARM_DEBOUNCE_MS = 150;
const RECORD_REARM_MAX_ATTEMPTS = 12;
const RECORD_REARM_RETRY_DELAY_MS = 400;
const rearmTimers = new Map<number, ReturnType<typeof setTimeout>>();
function makeRequestId(tabId: number): string {
return `rec-${tabId}-${Date.now().toString(36)}`;
}
function sleep(ms: number): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, ms));
}
/** Pages where MV3 content scripts cannot attach (Agent Window boots here). */
function isContentScriptRestrictedUrl(url: string | undefined): boolean {
if (!url) return true;
const lower = url.toLowerCase();
return (
lower === "about:blank" ||
lower.startsWith("about:") ||
lower.startsWith("chrome://") ||
lower.startsWith("chrome-extension://") ||
lower.startsWith("edge://") ||
lower.startsWith("devtools://") ||
lower.startsWith("devtools:") ||
lower.startsWith("https://chrome.google.com/webstore")
);
}
async function waitForTabReady(
tabId: number,
tabsApi: ChromeTabsApi,
timeoutMs = 10_000,
): Promise<void> {
try {
const tab = await tabsApi.get(tabId);
if (tab.status === "complete") return;
} catch {
return;
}
await new Promise<void>((resolve, reject) => {
const timer = setTimeout(() => {
chrome.tabs.onUpdated.removeListener(listener);
reject(new Error("tab load timeout"));
}, timeoutMs);
const listener = (updatedTabId: number, info: chrome.tabs.TabChangeInfo) => {
if (updatedTabId !== tabId || info.status !== "complete") return;
clearTimeout(timer);
chrome.tabs.onUpdated.removeListener(listener);
resolve();
};
chrome.tabs.onUpdated.addListener(listener);
});
}
function isRecordStartAck(response: unknown): response is RecordStartAck {
return (
typeof response === "object" &&
response !== null &&
"ok" in response &&
(response as RecordStartAck).ok === true
);
}
async function sendRecordStartWithAck(
tabId: number,
msg: RecordStartMessage,
sendToTab: RecordDeps["sendToTab"],
): Promise<void> {
let lastError: unknown;
for (let attempt = 0; attempt < RECORD_START_RETRIES; attempt += 1) {
try {
const response = await sendToTab(tabId, msg);
if (isRecordStartAck(response)) return;
lastError = new Error("content script did not ack RECORD_START");
} catch (err) {
lastError = err;
}
if (attempt + 1 < RECORD_START_RETRIES) {
await sleep(RECORD_START_RETRY_DELAY_MS);
}
}
throw lastError ?? new Error("failed to start recording in content script");
}
function buildTrace(recording: ActiveRecording): Trace {
const { pages, steps } = reduceTraceSteps(recording.steps, recording.startUrl);
const startUrl = resolveTraceStartUrl(recording.steps, recording.startUrl, pages);
return {
...(recording.purpose ? { purpose: recording.purpose } : {}),
recorded_at: new Date().toISOString(),
started_at: recording.startedAt,
entry: { start_url: startUrl },
pages,
steps,
};
}
export interface RecordDeps {
tabsApi: ChromeTabsApi;
sendToTab(
tabId: number,
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
): Promise<unknown>;
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
cdp?: CdpRunner;
signal?: AbortSignal;
}
let defaultDeps: RecordDeps | null = null;
function getDefaultDeps(): RecordDeps {
if (!defaultDeps) {
defaultDeps = {
tabsApi: chromeTabsApi,
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
};
}
return defaultDeps;
}
/** Disposer for lazily attached tab / webNavigation observers. */
let detachBrowserObservation: (() => void) | null = null;
type AttachObservation = (deps: RecordDeps) => () => void;
// Deferred wrappers so we do not capture attach* before their declarations.
let attachTabObservation: AttachObservation = (deps) => attachRecordTabListener(deps);
let attachNavObservation: AttachObservation = (deps) => attachRecordNavigationListener(deps);
/** Test seam: swap real chrome listeners for fakes. */
export function setBrowserObservationAttachForTests(
tab: AttachObservation | null,
nav: AttachObservation | null,
): void {
attachTabObservation = tab ?? ((deps) => attachRecordTabListener(deps));
attachNavObservation = nav ?? ((deps) => attachRecordNavigationListener(deps));
}
export function isBrowserObservationAttachedForTests(): boolean {
return detachBrowserObservation !== null;
}
export function resetBrowserObservationForTests(): void {
detachBrowserObservation?.();
detachBrowserObservation = null;
recordings.clear();
attachTabObservation = (deps) => attachRecordTabListener(deps);
attachNavObservation = (deps) => attachRecordNavigationListener(deps);
}
/**
* Attach tab/webNavigation listeners while any recording is active.
* Must run before navigate-on-start so rearm observes the destination load.
*/
export function ensureBrowserObservationListeners(deps: RecordDeps = getDefaultDeps()): void {
if (detachBrowserObservation) return;
const detachTab = attachTabObservation(deps);
const detachNav = attachNavObservation(deps);
detachBrowserObservation = () => {
detachTab();
detachNav();
};
}
/** Detach when the recordings map is empty. */
export function releaseBrowserObservationListenersIfIdle(): void {
if (recordings.size > 0) return;
if (!detachBrowserObservation) return;
detachBrowserObservation();
detachBrowserObservation = null;
}
export function attachRecordStepListener(): () => void {
const listener = (
message: unknown,
_sender: chrome.runtime.MessageSender,
_sendResponse: () => void,
) => {
if (!isRecordStepMessage(message)) return false;
for (const recording of recordings.values()) {
if (recording.requestId !== message.requestId) continue;
appendRecordedPayload(recording, message.step);
return false;
}
return false;
};
chrome.runtime.onMessage.addListener(listener);
return () => chrome.runtime.onMessage.removeListener(listener);
}
export function attachRecordFinishListener(deps: RecordDeps = getDefaultDeps()): () => void {
const listener = (
message: unknown,
sender: chrome.runtime.MessageSender,
_sendResponse: () => void,
) => {
if (!isRecordFinishMessage(message)) return false;
const tabId = sender.tab?.id;
if (tabId === undefined) return false;
void finishRecordingByRequest(message.requestId, tabId, deps);
return false;
};
chrome.runtime.onMessage.addListener(listener);
return () => chrome.runtime.onMessage.removeListener(listener);
}
function findRecordingByTabId(tabId: number): ActiveRecording | null {
for (const recording of recordings.values()) {
if (recording.tabId === tabId && !recording.settled) return recording;
}
return null;
}
async function findRecordingForTab(
tabId: number,
deps: RecordDeps,
): Promise<ActiveRecording | null> {
const direct = findRecordingByTabId(tabId);
if (direct) return direct;
try {
const tab = await deps.tabsApi.get(tabId);
const windowId = tab.windowId;
if (typeof windowId !== "number") return null;
for (const recording of recordings.values()) {
if (!recording.settled && recording.agentWindowId === windowId) return recording;
}
} catch {
return null;
}
return null;
}
function clearRearmTimer(tabId: number): void {
const timer = rearmTimers.get(tabId);
if (timer) {
clearTimeout(timer);
rearmTimers.delete(tabId);
}
}
async function clearRearmTimersForRecording(
recording: ActiveRecording,
deps: RecordDeps,
): Promise<void> {
clearRearmTimer(recording.tabId);
try {
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
for (const tab of tabs) {
if (typeof tab.id === "number") clearRearmTimer(tab.id);
}
} catch {
// Best-effort cleanup.
}
}
async function stopRecordingOnAllAgentTabs(
recording: ActiveRecording,
deps: RecordDeps,
): Promise<void> {
const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
let tabIds = [recording.tabId];
try {
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
tabIds = [
...new Set([
recording.tabId,
...tabs.flatMap((tab) => (typeof tab.id === "number" ? [tab.id] : [])),
]),
];
} catch {
// Fall back to the current recording tab.
}
for (const tabId of tabIds) {
try {
const response = await deps.sendToTab(tabId, stopMsg);
if (tabId === recording.tabId && !isRecordStartAck(response)) {
throw new Error("content script did not confirm recorded steps");
}
} catch {
if (tabId === recording.tabId) {
throw new Error("failed to flush recorded steps");
}
}
if (deps.bypassOverlay) {
try {
await deps.bypassOverlay(tabId, false);
} catch {
// Best-effort cleanup.
}
}
}
}
async function rearmRecording(
recording: ActiveRecording,
targetTabId: number,
deps: RecordDeps,
): Promise<boolean> {
// Do NOT toggle automation-bypass here: each retry used to increment the
// content-script counter, and a single stop decrement left the ControlOverlay
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
// already hides the control chrome while activeRecord is set.
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
const startMsg: RecordStartMessage = { type: RECORD_START, requestId: recording.requestId };
try {
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
recording.tabId = targetTabId;
return true;
} catch {
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
await sleep(RECORD_REARM_RETRY_DELAY_MS);
}
}
}
return false;
}
function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
const existing = rearmTimers.get(tabId);
if (existing) clearTimeout(existing);
rearmTimers.set(
tabId,
setTimeout(() => {
rearmTimers.delete(tabId);
void (async () => {
const current = await findRecordingForTab(tabId, deps);
if (current) await rearmRecording(current, tabId, deps);
})();
}, RECORD_REARM_DEBOUNCE_MS),
);
}
export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): () => void {
const onCreated = (tab: chrome.tabs.Tab) => {
const tabId = tab.id;
const windowId = tab.windowId;
if (tabId === undefined || windowId === undefined) return;
for (const recording of recordings.values()) {
if (recording.settled || recording.agentWindowId !== windowId) continue;
scheduleRearmForTab(tabId, deps);
return;
}
};
const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => {
scheduleRearmForTab(activeInfo.tabId, deps);
};
chrome.tabs.onCreated.addListener(onCreated);
chrome.tabs.onActivated.addListener(onActivated);
return () => {
chrome.tabs.onCreated.removeListener(onCreated);
chrome.tabs.onActivated.removeListener(onActivated);
};
}
export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps()): () => void {
const observeMainFrame = (tabId: number, url?: string, causedByAction?: boolean) => {
void (async () => {
const recording = await findRecordingForTab(tabId, deps);
if (recording && url) {
observeRecordedNavigation(recording, url, causedByAction);
}
})();
};
const onMainFrameComplete = (tabId: number, url?: string) => {
void (async () => {
observeMainFrame(tabId, url);
scheduleRearmForTab(tabId, deps);
})();
};
if (chrome.webNavigation?.onCompleted) {
const completedListener = (
details: chrome.webNavigation.WebNavigationFramedCallbackDetails,
) => {
if (details.frameId !== 0) return;
onMainFrameComplete(details.tabId, details.url);
};
const committedListener = (
details: chrome.webNavigation.WebNavigationTransitionCallbackDetails,
) => {
if (details.frameId !== 0) return;
const causedByAction =
details.transitionType === "link" || details.transitionType === "form_submit"
? true
: details.transitionType === "typed" ||
details.transitionType === "auto_bookmark" ||
details.transitionType === "generated" ||
details.transitionType === "keyword" ||
details.transitionType === "keyword_generated" ||
details.transitionType === "reload"
? false
: undefined;
observeMainFrame(details.tabId, details.url, causedByAction);
};
chrome.webNavigation.onCompleted.addListener(completedListener);
chrome.webNavigation.onCommitted?.addListener(committedListener);
return () => {
chrome.webNavigation.onCompleted.removeListener(completedListener);
chrome.webNavigation.onCommitted?.removeListener(committedListener);
};
}
const listener = (tabId: number, info: chrome.tabs.TabChangeInfo) => {
if (info.status !== "complete") return;
onMainFrameComplete(tabId, info.url);
};
chrome.tabs.onUpdated.addListener(listener);
return () => chrome.tabs.onUpdated.removeListener(listener);
}
export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()): () => void {
const listener = (
message: unknown,
sender: chrome.runtime.MessageSender,
sendResponse: (response: RecordQueryResponse) => void,
) => {
if (!isRecordQueryMessage(message)) return false;
const tabId = sender.tab?.id;
if (tabId === undefined) {
sendResponse({ active: false });
return false;
}
void (async () => {
const recording = await findRecordingForTab(tabId, deps);
if (!recording) {
sendResponse({ active: false });
return;
}
await rearmRecording(recording, tabId, deps);
sendResponse({ active: true, requestId: recording.requestId });
})();
return true;
};
chrome.runtime.onMessage.addListener(listener);
return () => chrome.runtime.onMessage.removeListener(listener);
}
async function finishRecordingByRequest(
requestId: string,
tabId: number,
deps: RecordDeps,
): Promise<void> {
for (const [sessionId, recording] of recordings) {
if (recording.requestId !== requestId || recording.settled) continue;
const match = await findRecordingForTab(tabId, deps);
if (match !== recording) continue;
await finishRecording(sessionId, deps);
return;
}
}
async function finishRecording(sessionId: string, deps: RecordDeps): Promise<Trace | null> {
const recording = recordings.get(sessionId);
if (!recording || recording.settled || recording.finishing) return null;
recording.finishing = true;
await clearRearmTimersForRecording(recording, deps);
try {
await stopRecordingOnAllAgentTabs(recording, deps);
} catch {
recording.finishing = false;
return null;
}
recording.settled = true;
recordings.delete(sessionId);
releaseBrowserObservationListenersIfIdle();
const trace = buildTrace(recording);
recording.resolveFinish(trace);
return trace;
}
export async function handleRecordStart(
manager: SessionManager,
params: RecordStartParams,
deps: RecordDeps = getDefaultDeps(),
): Promise<RecordStartResult | RpcError> {
const ctxOrErr = lookupSession(manager, params, "record_start");
if (isRpcError(ctxOrErr)) return ctxOrErr;
const ctx = ctxOrErr;
if (recordings.has(params.session_id)) {
return {
code: "protocol_error",
message: `session ${params.session_id} is already recording`,
};
}
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
if (isRpcError(target)) return target;
// Register the recording *before* navigate so content-script syncAgentOverlay
// on the destination page can RECORD_QUERY → rearm → show RecordOverlay
// instead of flashing ControlOverlay ("Agent 正在控制").
const requestId = makeRequestId(target.tabId);
let resolveFinish!: (trace: Trace) => void;
let rejectFinish!: (err: Error) => void;
const finishPromise = new Promise<Trace>((resolve, reject) => {
resolveFinish = resolve;
rejectFinish = reject;
});
const provisionalUrl = params.url;
recordings.set(params.session_id, {
requestId,
tabId: target.tabId,
agentWindowId: ctx.agentWindowId,
startUrl: provisionalUrl,
...(params.purpose ? { purpose: params.purpose } : {}),
steps: [],
startedAt: new Date().toISOString(),
finishPromise,
resolveFinish,
rejectFinish,
settled: false,
finishing: false,
currentUrl: provisionalUrl,
pendingNavigation: false,
pendingNavigationDeadline: undefined,
});
// Observe navigations for the whole recording lifetime; attach before
// optional navigate so the destination load can rearm capture.
ensureBrowserObservationListeners(deps);
const abortPending = async (notifyContent: boolean) => {
recordings.delete(params.session_id);
releaseBrowserObservationListenersIfIdle();
if (notifyContent) {
try {
await deps.sendToTab(target.tabId, {
type: RECORD_CANCEL,
requestId,
});
} catch {
// Content may never have received RECORD_START.
}
}
if (deps.bypassOverlay) {
try {
await deps.bypassOverlay(target.tabId, false);
} catch {
// Ignore cleanup errors.
}
}
};
const cancelledError = (): RpcError => ({
code: "cancelled",
message: "record_start aborted",
});
/** Dispatcher may race-cancel while we still hold a provisional recording. */
const abortIfCancelled = async (notifyContent: boolean): Promise<RpcError | null> => {
if (!deps.signal?.aborted) return null;
await abortPending(notifyContent);
return cancelledError();
};
{
const cancelled = await abortIfCancelled(false);
if (cancelled) return cancelled;
}
if (params.url && deps.cdp) {
const nav = await handleNavigate(
manager,
{
session_id: params.session_id,
url: params.url,
tab_id: target.tabId,
},
{ cdp: deps.cdp, tabsApi: deps.tabsApi, signal: deps.signal },
);
if (isRpcError(nav)) {
await abortPending(false);
return nav;
}
{
const cancelled = await abortIfCancelled(false);
if (cancelled) return cancelled;
}
try {
await waitForTabReady(target.tabId, deps.tabsApi);
} catch {
// Proceed with retries even if the tab never reports complete.
}
{
const cancelled = await abortIfCancelled(false);
if (cancelled) return cancelled;
}
}
let startUrl: string | undefined;
try {
const tab = await deps.tabsApi.get(target.tabId);
startUrl = tab.url;
} catch {
startUrl = params.url;
}
const active = recordings.get(params.session_id);
if (!active) {
// Cleared by a concurrent abort / session teardown.
return cancelledError();
}
active.startUrl = startUrl;
active.currentUrl = startUrl;
if (isContentScriptRestrictedUrl(startUrl)) {
await abortPending(false);
return {
code: "invalid_params",
message: params.url
? `cannot record on restricted URL (${startUrl}); use an http(s) page`
: "Agent Window starts on about:blank where the content script cannot inject — pass --url https://… before recording",
};
}
{
const cancelled = await abortIfCancelled(false);
if (cancelled) return cancelled;
}
if (deps.bypassOverlay) {
try {
// Single ref for the initial race before RecordOverlay mounts; rearm must
// not stack additional refs (see rearmRecording). Cleared on stop.
await deps.bypassOverlay(target.tabId, true);
} catch {
// Best-effort; activeRecord also hides the control overlay.
}
}
{
const cancelled = await abortIfCancelled(true);
if (cancelled) return cancelled;
}
const startMsg: RecordStartMessage = { type: RECORD_START, requestId };
try {
await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
} catch {
await abortPending(true);
return {
code: "protocol_error",
message:
"failed to start recording in content script — reload the BrowserSkill extension, then retry with --url https://…",
};
}
{
const cancelled = await abortIfCancelled(true);
if (cancelled) return cancelled;
}
return { tab_id: target.tabId, recording: true };
}
export async function handleRecordStop(
manager: SessionManager,
params: RecordStopParams,
deps: RecordDeps = getDefaultDeps(),
): Promise<RecordStopResult | RpcError> {
const ctxOrErr = lookupSession(manager, params, "record_stop");
if (isRpcError(ctxOrErr)) return ctxOrErr;
const recording = recordings.get(params.session_id);
if (!recording) {
return {
code: "not_found",
message: `no active recording for session ${params.session_id}`,
};
}
const trace = await finishRecording(params.session_id, deps);
if (!trace) {
return {
code: "protocol_error",
message: `failed to flush recorded steps for session ${params.session_id}; the recording is still active — retry \`bsk record stop\``,
};
}
return { trace };
}
export async function handleRecordAwait(
manager: SessionManager,
params: RecordAwaitParams,
deps: RecordDeps = getDefaultDeps(),
): Promise<RecordAwaitResult | RpcError> {
const ctxOrErr = lookupSession(manager, params, "record_await");
if (isRpcError(ctxOrErr)) return ctxOrErr;
const recording = recordings.get(params.session_id);
if (!recording) {
return {
code: "not_found",
message: `no active recording for session ${params.session_id}`,
};
}
if (deps.signal?.aborted) {
return { code: "cancelled", message: "record_await aborted" };
}
const outcome = await new Promise<{ trace: Trace } | { error: RpcError }>((resolve) => {
let settled = false;
const finish = (result: { trace: Trace } | { error: RpcError }) => {
if (settled) return;
settled = true;
if (timer) clearTimeout(timer);
deps.signal?.removeEventListener("abort", onAbort);
resolve(result);
};
const onAbort = () => finish({ error: { code: "cancelled", message: "record_await aborted" } });
const timer =
params.timeout_ms === undefined
? undefined
: setTimeout(
() =>
finish({
error: {
code: "timeout",
message: `record_await timed out after ${params.timeout_ms}ms`,
},
}),
params.timeout_ms,
);
deps.signal?.addEventListener("abort", onAbort, { once: true });
void recording.finishPromise.then(
(trace) => finish({ trace }),
() =>
finish({
error: { code: "cancelled", message: "recording was cleared" },
}),
);
});
return "trace" in outcome ? { trace: outcome.trace } : outcome.error;
}
export function clearRecordingForSession(sessionId: string): void {
const recording = recordings.get(sessionId);
if (!recording) {
recordings.delete(sessionId);
releaseBrowserObservationListenersIfIdle();
return;
}
void clearRearmTimersForRecording(recording, getDefaultDeps());
if (!recording.settled) {
recording.settled = true;
recording.rejectFinish(new Error("recording cleared"));
}
recordings.delete(sessionId);
releaseBrowserObservationListenersIfIdle();
}
export type { RecordFinishMessage };
+3
View File
@@ -1,5 +1,6 @@
import type { SessionManager } from "@/session-manager/manager";
import type { RpcError } from "@/transport/types";
import { clearRecordingForSession } from "./record";
import { returnBorrowedTab, type TabManagementDeps } from "./tabs";
export interface SessionStartParams {
@@ -156,6 +157,8 @@ export async function handleSessionStop(
// Step 2: clear the per-session RefStore (review M6/M7 parity).
ctx.refStore.clear();
clearRecordingForSession(params.session_id);
// Step 3: detach CDP sessions this session opened (no-op if none).
await deps.cdp?.detachSession(params.session_id);
+132
View File
@@ -537,3 +537,135 @@ export interface RequestHelpResult {
tab_id: number;
resolved_targets?: ResolvedTarget[];
}
// --------------------------------------------------------------------------
// Semantic record payloads — mirror bsk-protocol record.rs (Trace v2)
// --------------------------------------------------------------------------
export interface TargetDescriptor {
role?: string;
name?: string;
tag: string;
name_attr?: string;
placeholder?: string;
nearby_label?: string;
}
export interface TraceEntry {
start_url: string;
}
export interface PageRef {
id: string;
url: string;
title?: string;
}
export interface SelectedOption {
value: string;
label?: string;
}
export interface StepEffect {
navigated_to: string;
}
export interface StepCommon {
id: number;
page: string;
effect?: StepEffect;
}
/** Capture/buffer draft before v2 reduction. */
export type DraftTraceStep =
| {
op: "click";
target: TargetDescriptor;
navigated_to?: string;
page_url?: string;
}
| {
op: "fill";
target: TargetDescriptor;
value: string;
redacted?: boolean;
page_url?: string;
}
| {
op: "press";
key: string;
target?: TargetDescriptor;
modifiers?: KeyModifier[];
navigated_to?: string;
page_url?: string;
}
| {
op: "select";
target: TargetDescriptor;
values: string[];
labels?: string[];
navigated_to?: string;
page_url?: string;
}
| {
op: "navigate";
url: string;
page_url?: string;
};
/** Exported record-only step (trace v2). */
export type Step =
| ({ op: "navigate" } & StepCommon & { to: string })
| ({ op: "click" } & StepCommon & { target: TargetDescriptor })
| ({ op: "fill" } & StepCommon & {
target: TargetDescriptor;
value: string;
redacted?: boolean;
})
| ({ op: "select" } & StepCommon & {
target: TargetDescriptor;
selection: SelectedOption[];
})
| ({ op: "press" } & StepCommon & {
key: string;
modifiers?: KeyModifier[];
target?: TargetDescriptor;
});
export interface Trace {
recorded_at: string;
started_at?: string;
purpose?: string;
entry: TraceEntry;
pages: PageRef[];
steps: Step[];
}
export interface RecordStartParams {
session_id: string;
tab_id?: number;
url?: string;
purpose?: string;
}
export interface RecordStartResult {
tab_id: number;
recording: boolean;
}
export interface RecordStopParams {
session_id: string;
}
export interface RecordStopResult {
trace: Trace;
}
export interface RecordAwaitParams {
session_id: string;
timeout_ms?: number;
}
export interface RecordAwaitResult {
trace: Trace;
}
+1 -1
View File
@@ -25,7 +25,7 @@ export default defineConfig({
name: "BrowserSkill",
description:
"Let AI agents use your logged-in browser in a separate Agent Window—without interrupting your work. Powered by the bsk CLI.",
permissions: ["alarms", "debugger", "notifications", "tabs", "storage", "windows"],
permissions: ["alarms", "debugger", "notifications", "tabs", "storage", "webNavigation", "windows"],
host_permissions: ["<all_urls>"],
icons: {
16: "icon/logo.png",