mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat: add bsk record for semantic user-action traces
Capture user actions in the Agent Window and export a record-only trace (pages[] + navigate/click/fill/select/press steps with target descriptors). Adds the record overlay UI aligned with the control overlay, and wires record start/stop/await through CLI, daemon, protocol, and extension.
This commit is contained in:
“haonan”
@@ -0,0 +1,52 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
ensureBrowserObservationListeners,
|
||||
isBrowserObservationAttachedForTests,
|
||||
releaseBrowserObservationListenersIfIdle,
|
||||
resetBrowserObservationForTests,
|
||||
setBrowserObservationAttachForTests,
|
||||
} from "../record";
|
||||
|
||||
describe("browser observation lifecycle", () => {
|
||||
afterEach(() => {
|
||||
resetBrowserObservationForTests();
|
||||
});
|
||||
|
||||
it("does not attach listeners until ensure is called", () => {
|
||||
const tab = vi.fn(() => () => undefined);
|
||||
const nav = vi.fn(() => () => undefined);
|
||||
setBrowserObservationAttachForTests(tab, nav);
|
||||
|
||||
expect(isBrowserObservationAttachedForTests()).toBe(false);
|
||||
expect(tab).not.toHaveBeenCalled();
|
||||
expect(nav).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("attaches once on ensure and detaches when idle", () => {
|
||||
const detachTab = vi.fn();
|
||||
const detachNav = vi.fn();
|
||||
const tab = vi.fn(() => detachTab);
|
||||
const nav = vi.fn(() => detachNav);
|
||||
setBrowserObservationAttachForTests(tab, nav);
|
||||
|
||||
ensureBrowserObservationListeners({
|
||||
tabsApi: { get: vi.fn(), query: vi.fn(), create: vi.fn() } as never,
|
||||
sendToTab: vi.fn(),
|
||||
});
|
||||
expect(isBrowserObservationAttachedForTests()).toBe(true);
|
||||
expect(tab).toHaveBeenCalledTimes(1);
|
||||
expect(nav).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Idempotent.
|
||||
ensureBrowserObservationListeners({
|
||||
tabsApi: { get: vi.fn(), query: vi.fn(), create: vi.fn() } as never,
|
||||
sendToTab: vi.fn(),
|
||||
});
|
||||
expect(tab).toHaveBeenCalledTimes(1);
|
||||
|
||||
releaseBrowserObservationListenersIfIdle();
|
||||
expect(isBrowserObservationAttachedForTests()).toBe(false);
|
||||
expect(detachTab).toHaveBeenCalledTimes(1);
|
||||
expect(detachNav).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -13,6 +13,9 @@ import type {
|
||||
NetworkParams,
|
||||
PressParams,
|
||||
ProtocolFrame,
|
||||
RecordAwaitParams,
|
||||
RecordStartParams,
|
||||
RecordStopParams,
|
||||
ReloadParams,
|
||||
RequestFrame,
|
||||
RequestHelpParams,
|
||||
@@ -42,6 +45,7 @@ import {
|
||||
handleScreenshot,
|
||||
handleSnapshot,
|
||||
} from "./observation";
|
||||
import { handleRecordAwait, handleRecordStart, handleRecordStop } from "./record";
|
||||
import {
|
||||
handleSessionStart,
|
||||
handleSessionStop,
|
||||
@@ -396,6 +400,45 @@ export class ToolDispatcher {
|
||||
notificationCopy: this.helpNotificationCopy?.(),
|
||||
signal,
|
||||
});
|
||||
case "tool.record_start":
|
||||
return handleRecordStart(this.sessions, req.params as RecordStartParams, {
|
||||
tabsApi: chromeTabsApi,
|
||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||
bypassOverlay: async (tabId, enabled) => {
|
||||
try {
|
||||
await chrome.tabs.sendMessage(tabId, {
|
||||
type: OVERLAY_AUTOMATION_BYPASS,
|
||||
enabled,
|
||||
});
|
||||
} catch {
|
||||
// Content script may be unavailable on restricted pages.
|
||||
}
|
||||
},
|
||||
...(this.cdp ? { cdp: this.cdp } : {}),
|
||||
signal,
|
||||
});
|
||||
case "tool.record_stop":
|
||||
return handleRecordStop(this.sessions, req.params as RecordStopParams, {
|
||||
tabsApi: chromeTabsApi,
|
||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||
bypassOverlay: async (tabId, enabled) => {
|
||||
try {
|
||||
await chrome.tabs.sendMessage(tabId, {
|
||||
type: OVERLAY_AUTOMATION_BYPASS,
|
||||
enabled,
|
||||
});
|
||||
} catch {
|
||||
// Content script may be unavailable on restricted pages.
|
||||
}
|
||||
},
|
||||
signal,
|
||||
});
|
||||
case "tool.record_await":
|
||||
return handleRecordAwait(this.sessions, req.params as RecordAwaitParams, {
|
||||
tabsApi: chromeTabsApi,
|
||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||
signal,
|
||||
});
|
||||
default:
|
||||
return {
|
||||
code: "unknown_method",
|
||||
|
||||
@@ -0,0 +1,825 @@
|
||||
// `tool.record_start` / `tool.record_stop` / `tool.record_await` — capture
|
||||
// user actions in the Agent Window via the content script and return a
|
||||
// semantic (LLM textbook) trace.
|
||||
|
||||
import {
|
||||
isRecordFinishMessage,
|
||||
isRecordQueryMessage,
|
||||
isRecordStepMessage,
|
||||
RECORD_CANCEL,
|
||||
RECORD_START,
|
||||
RECORD_STEP,
|
||||
RECORD_STOP,
|
||||
type RecordCancelMessage,
|
||||
type RecordFinishMessage,
|
||||
type RecordQueryResponse,
|
||||
type RecordStartAck,
|
||||
type RecordStartMessage,
|
||||
type RecordStopMessage,
|
||||
} from "@/lib/record-bridge";
|
||||
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording-step-buffer";
|
||||
import { reduceTraceSteps, resolveTraceStartUrl } from "@/lib/trace-reducer";
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import type {
|
||||
DraftTraceStep,
|
||||
RecordAwaitParams,
|
||||
RecordAwaitResult,
|
||||
RecordStartParams,
|
||||
RecordStartResult,
|
||||
RecordStopParams,
|
||||
RecordStopResult,
|
||||
RpcError,
|
||||
Trace,
|
||||
} from "@/transport/types";
|
||||
import { handleNavigate } from "./navigation";
|
||||
import {
|
||||
type CdpRunner,
|
||||
type ChromeTabsApi,
|
||||
chromeTabsApi,
|
||||
isRpcError,
|
||||
lookupSession,
|
||||
resolveTargetTab,
|
||||
} from "./shared";
|
||||
|
||||
interface ActiveRecording {
|
||||
requestId: string;
|
||||
tabId: number;
|
||||
agentWindowId: number;
|
||||
startUrl?: string;
|
||||
purpose?: string;
|
||||
steps: DraftTraceStep[];
|
||||
startedAt: string;
|
||||
finishPromise: Promise<Trace>;
|
||||
resolveFinish: (trace: Trace) => void;
|
||||
rejectFinish: (err: Error) => void;
|
||||
settled: boolean;
|
||||
finishing: boolean;
|
||||
currentUrl?: string;
|
||||
pendingNavigation: boolean;
|
||||
pendingNavigationDeadline?: number;
|
||||
}
|
||||
|
||||
const recordings = new Map<string, ActiveRecording>();
|
||||
|
||||
const RECORD_START_RETRIES = 3;
|
||||
const RECORD_START_RETRY_DELAY_MS = 500;
|
||||
const RECORD_REARM_DEBOUNCE_MS = 150;
|
||||
const RECORD_REARM_MAX_ATTEMPTS = 12;
|
||||
const RECORD_REARM_RETRY_DELAY_MS = 400;
|
||||
|
||||
const rearmTimers = new Map<number, ReturnType<typeof setTimeout>>();
|
||||
|
||||
function makeRequestId(tabId: number): string {
|
||||
return `rec-${tabId}-${Date.now().toString(36)}`;
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/** Pages where MV3 content scripts cannot attach (Agent Window boots here). */
|
||||
function isContentScriptRestrictedUrl(url: string | undefined): boolean {
|
||||
if (!url) return true;
|
||||
const lower = url.toLowerCase();
|
||||
return (
|
||||
lower === "about:blank" ||
|
||||
lower.startsWith("about:") ||
|
||||
lower.startsWith("chrome://") ||
|
||||
lower.startsWith("chrome-extension://") ||
|
||||
lower.startsWith("edge://") ||
|
||||
lower.startsWith("devtools://") ||
|
||||
lower.startsWith("devtools:") ||
|
||||
lower.startsWith("https://chrome.google.com/webstore")
|
||||
);
|
||||
}
|
||||
|
||||
async function waitForTabReady(
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
timeoutMs = 10_000,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const tab = await tabsApi.get(tabId);
|
||||
if (tab.status === "complete") return;
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
chrome.tabs.onUpdated.removeListener(listener);
|
||||
reject(new Error("tab load timeout"));
|
||||
}, timeoutMs);
|
||||
const listener = (updatedTabId: number, info: chrome.tabs.TabChangeInfo) => {
|
||||
if (updatedTabId !== tabId || info.status !== "complete") return;
|
||||
clearTimeout(timer);
|
||||
chrome.tabs.onUpdated.removeListener(listener);
|
||||
resolve();
|
||||
};
|
||||
chrome.tabs.onUpdated.addListener(listener);
|
||||
});
|
||||
}
|
||||
|
||||
function isRecordStartAck(response: unknown): response is RecordStartAck {
|
||||
return (
|
||||
typeof response === "object" &&
|
||||
response !== null &&
|
||||
"ok" in response &&
|
||||
(response as RecordStartAck).ok === true
|
||||
);
|
||||
}
|
||||
|
||||
async function sendRecordStartWithAck(
|
||||
tabId: number,
|
||||
msg: RecordStartMessage,
|
||||
sendToTab: RecordDeps["sendToTab"],
|
||||
): Promise<void> {
|
||||
let lastError: unknown;
|
||||
for (let attempt = 0; attempt < RECORD_START_RETRIES; attempt += 1) {
|
||||
try {
|
||||
const response = await sendToTab(tabId, msg);
|
||||
if (isRecordStartAck(response)) return;
|
||||
lastError = new Error("content script did not ack RECORD_START");
|
||||
} catch (err) {
|
||||
lastError = err;
|
||||
}
|
||||
if (attempt + 1 < RECORD_START_RETRIES) {
|
||||
await sleep(RECORD_START_RETRY_DELAY_MS);
|
||||
}
|
||||
}
|
||||
throw lastError ?? new Error("failed to start recording in content script");
|
||||
}
|
||||
|
||||
function buildTrace(recording: ActiveRecording): Trace {
|
||||
const { pages, steps } = reduceTraceSteps(recording.steps, recording.startUrl);
|
||||
const startUrl = resolveTraceStartUrl(recording.steps, recording.startUrl, pages);
|
||||
return {
|
||||
...(recording.purpose ? { purpose: recording.purpose } : {}),
|
||||
recorded_at: new Date().toISOString(),
|
||||
started_at: recording.startedAt,
|
||||
entry: { start_url: startUrl },
|
||||
pages,
|
||||
steps,
|
||||
};
|
||||
}
|
||||
|
||||
export interface RecordDeps {
|
||||
tabsApi: ChromeTabsApi;
|
||||
sendToTab(
|
||||
tabId: number,
|
||||
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
|
||||
): Promise<unknown>;
|
||||
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
cdp?: CdpRunner;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
let defaultDeps: RecordDeps | null = null;
|
||||
function getDefaultDeps(): RecordDeps {
|
||||
if (!defaultDeps) {
|
||||
defaultDeps = {
|
||||
tabsApi: chromeTabsApi,
|
||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||
};
|
||||
}
|
||||
return defaultDeps;
|
||||
}
|
||||
|
||||
/** Disposer for lazily attached tab / webNavigation observers. */
|
||||
let detachBrowserObservation: (() => void) | null = null;
|
||||
|
||||
type AttachObservation = (deps: RecordDeps) => () => void;
|
||||
|
||||
// Deferred wrappers so we do not capture attach* before their declarations.
|
||||
let attachTabObservation: AttachObservation = (deps) => attachRecordTabListener(deps);
|
||||
let attachNavObservation: AttachObservation = (deps) => attachRecordNavigationListener(deps);
|
||||
|
||||
/** Test seam: swap real chrome listeners for fakes. */
|
||||
export function setBrowserObservationAttachForTests(
|
||||
tab: AttachObservation | null,
|
||||
nav: AttachObservation | null,
|
||||
): void {
|
||||
attachTabObservation = tab ?? ((deps) => attachRecordTabListener(deps));
|
||||
attachNavObservation = nav ?? ((deps) => attachRecordNavigationListener(deps));
|
||||
}
|
||||
|
||||
export function isBrowserObservationAttachedForTests(): boolean {
|
||||
return detachBrowserObservation !== null;
|
||||
}
|
||||
|
||||
export function resetBrowserObservationForTests(): void {
|
||||
detachBrowserObservation?.();
|
||||
detachBrowserObservation = null;
|
||||
recordings.clear();
|
||||
attachTabObservation = (deps) => attachRecordTabListener(deps);
|
||||
attachNavObservation = (deps) => attachRecordNavigationListener(deps);
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach tab/webNavigation listeners while any recording is active.
|
||||
* Must run before navigate-on-start so rearm observes the destination load.
|
||||
*/
|
||||
export function ensureBrowserObservationListeners(deps: RecordDeps = getDefaultDeps()): void {
|
||||
if (detachBrowserObservation) return;
|
||||
const detachTab = attachTabObservation(deps);
|
||||
const detachNav = attachNavObservation(deps);
|
||||
detachBrowserObservation = () => {
|
||||
detachTab();
|
||||
detachNav();
|
||||
};
|
||||
}
|
||||
|
||||
/** Detach when the recordings map is empty. */
|
||||
export function releaseBrowserObservationListenersIfIdle(): void {
|
||||
if (recordings.size > 0) return;
|
||||
if (!detachBrowserObservation) return;
|
||||
detachBrowserObservation();
|
||||
detachBrowserObservation = null;
|
||||
}
|
||||
|
||||
export function attachRecordStepListener(): () => void {
|
||||
const listener = (
|
||||
message: unknown,
|
||||
_sender: chrome.runtime.MessageSender,
|
||||
_sendResponse: () => void,
|
||||
) => {
|
||||
if (!isRecordStepMessage(message)) return false;
|
||||
for (const recording of recordings.values()) {
|
||||
if (recording.requestId !== message.requestId) continue;
|
||||
appendRecordedPayload(recording, message.step);
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
chrome.runtime.onMessage.addListener(listener);
|
||||
return () => chrome.runtime.onMessage.removeListener(listener);
|
||||
}
|
||||
|
||||
export function attachRecordFinishListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||
const listener = (
|
||||
message: unknown,
|
||||
sender: chrome.runtime.MessageSender,
|
||||
_sendResponse: () => void,
|
||||
) => {
|
||||
if (!isRecordFinishMessage(message)) return false;
|
||||
const tabId = sender.tab?.id;
|
||||
if (tabId === undefined) return false;
|
||||
void finishRecordingByRequest(message.requestId, tabId, deps);
|
||||
return false;
|
||||
};
|
||||
chrome.runtime.onMessage.addListener(listener);
|
||||
return () => chrome.runtime.onMessage.removeListener(listener);
|
||||
}
|
||||
|
||||
function findRecordingByTabId(tabId: number): ActiveRecording | null {
|
||||
for (const recording of recordings.values()) {
|
||||
if (recording.tabId === tabId && !recording.settled) return recording;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function findRecordingForTab(
|
||||
tabId: number,
|
||||
deps: RecordDeps,
|
||||
): Promise<ActiveRecording | null> {
|
||||
const direct = findRecordingByTabId(tabId);
|
||||
if (direct) return direct;
|
||||
|
||||
try {
|
||||
const tab = await deps.tabsApi.get(tabId);
|
||||
const windowId = tab.windowId;
|
||||
if (typeof windowId !== "number") return null;
|
||||
for (const recording of recordings.values()) {
|
||||
if (!recording.settled && recording.agentWindowId === windowId) return recording;
|
||||
}
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function clearRearmTimer(tabId: number): void {
|
||||
const timer = rearmTimers.get(tabId);
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
rearmTimers.delete(tabId);
|
||||
}
|
||||
}
|
||||
|
||||
async function clearRearmTimersForRecording(
|
||||
recording: ActiveRecording,
|
||||
deps: RecordDeps,
|
||||
): Promise<void> {
|
||||
clearRearmTimer(recording.tabId);
|
||||
try {
|
||||
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
|
||||
for (const tab of tabs) {
|
||||
if (typeof tab.id === "number") clearRearmTimer(tab.id);
|
||||
}
|
||||
} catch {
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
}
|
||||
|
||||
async function stopRecordingOnAllAgentTabs(
|
||||
recording: ActiveRecording,
|
||||
deps: RecordDeps,
|
||||
): Promise<void> {
|
||||
const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
|
||||
let tabIds = [recording.tabId];
|
||||
try {
|
||||
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
|
||||
tabIds = [
|
||||
...new Set([
|
||||
recording.tabId,
|
||||
...tabs.flatMap((tab) => (typeof tab.id === "number" ? [tab.id] : [])),
|
||||
]),
|
||||
];
|
||||
} catch {
|
||||
// Fall back to the current recording tab.
|
||||
}
|
||||
|
||||
for (const tabId of tabIds) {
|
||||
try {
|
||||
const response = await deps.sendToTab(tabId, stopMsg);
|
||||
if (tabId === recording.tabId && !isRecordStartAck(response)) {
|
||||
throw new Error("content script did not confirm recorded steps");
|
||||
}
|
||||
} catch {
|
||||
if (tabId === recording.tabId) {
|
||||
throw new Error("failed to flush recorded steps");
|
||||
}
|
||||
}
|
||||
if (deps.bypassOverlay) {
|
||||
try {
|
||||
await deps.bypassOverlay(tabId, false);
|
||||
} catch {
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function rearmRecording(
|
||||
recording: ActiveRecording,
|
||||
targetTabId: number,
|
||||
deps: RecordDeps,
|
||||
): Promise<boolean> {
|
||||
// Do NOT toggle automation-bypass here: each retry used to increment the
|
||||
// content-script counter, and a single stop decrement left the ControlOverlay
|
||||
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
|
||||
// already hides the control chrome while activeRecord is set.
|
||||
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
|
||||
const startMsg: RecordStartMessage = { type: RECORD_START, requestId: recording.requestId };
|
||||
try {
|
||||
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
|
||||
recording.tabId = targetTabId;
|
||||
return true;
|
||||
} catch {
|
||||
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
|
||||
await sleep(RECORD_REARM_RETRY_DELAY_MS);
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
|
||||
const existing = rearmTimers.get(tabId);
|
||||
if (existing) clearTimeout(existing);
|
||||
rearmTimers.set(
|
||||
tabId,
|
||||
setTimeout(() => {
|
||||
rearmTimers.delete(tabId);
|
||||
void (async () => {
|
||||
const current = await findRecordingForTab(tabId, deps);
|
||||
if (current) await rearmRecording(current, tabId, deps);
|
||||
})();
|
||||
}, RECORD_REARM_DEBOUNCE_MS),
|
||||
);
|
||||
}
|
||||
|
||||
export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||
const onCreated = (tab: chrome.tabs.Tab) => {
|
||||
const tabId = tab.id;
|
||||
const windowId = tab.windowId;
|
||||
if (tabId === undefined || windowId === undefined) return;
|
||||
for (const recording of recordings.values()) {
|
||||
if (recording.settled || recording.agentWindowId !== windowId) continue;
|
||||
scheduleRearmForTab(tabId, deps);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => {
|
||||
scheduleRearmForTab(activeInfo.tabId, deps);
|
||||
};
|
||||
|
||||
chrome.tabs.onCreated.addListener(onCreated);
|
||||
chrome.tabs.onActivated.addListener(onActivated);
|
||||
return () => {
|
||||
chrome.tabs.onCreated.removeListener(onCreated);
|
||||
chrome.tabs.onActivated.removeListener(onActivated);
|
||||
};
|
||||
}
|
||||
|
||||
export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||
const observeMainFrame = (tabId: number, url?: string, causedByAction?: boolean) => {
|
||||
void (async () => {
|
||||
const recording = await findRecordingForTab(tabId, deps);
|
||||
if (recording && url) {
|
||||
observeRecordedNavigation(recording, url, causedByAction);
|
||||
}
|
||||
})();
|
||||
};
|
||||
const onMainFrameComplete = (tabId: number, url?: string) => {
|
||||
void (async () => {
|
||||
observeMainFrame(tabId, url);
|
||||
scheduleRearmForTab(tabId, deps);
|
||||
})();
|
||||
};
|
||||
|
||||
if (chrome.webNavigation?.onCompleted) {
|
||||
const completedListener = (
|
||||
details: chrome.webNavigation.WebNavigationFramedCallbackDetails,
|
||||
) => {
|
||||
if (details.frameId !== 0) return;
|
||||
onMainFrameComplete(details.tabId, details.url);
|
||||
};
|
||||
const committedListener = (
|
||||
details: chrome.webNavigation.WebNavigationTransitionCallbackDetails,
|
||||
) => {
|
||||
if (details.frameId !== 0) return;
|
||||
const causedByAction =
|
||||
details.transitionType === "link" || details.transitionType === "form_submit"
|
||||
? true
|
||||
: details.transitionType === "typed" ||
|
||||
details.transitionType === "auto_bookmark" ||
|
||||
details.transitionType === "generated" ||
|
||||
details.transitionType === "keyword" ||
|
||||
details.transitionType === "keyword_generated" ||
|
||||
details.transitionType === "reload"
|
||||
? false
|
||||
: undefined;
|
||||
observeMainFrame(details.tabId, details.url, causedByAction);
|
||||
};
|
||||
chrome.webNavigation.onCompleted.addListener(completedListener);
|
||||
chrome.webNavigation.onCommitted?.addListener(committedListener);
|
||||
return () => {
|
||||
chrome.webNavigation.onCompleted.removeListener(completedListener);
|
||||
chrome.webNavigation.onCommitted?.removeListener(committedListener);
|
||||
};
|
||||
}
|
||||
|
||||
const listener = (tabId: number, info: chrome.tabs.TabChangeInfo) => {
|
||||
if (info.status !== "complete") return;
|
||||
onMainFrameComplete(tabId, info.url);
|
||||
};
|
||||
chrome.tabs.onUpdated.addListener(listener);
|
||||
return () => chrome.tabs.onUpdated.removeListener(listener);
|
||||
}
|
||||
|
||||
export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||
const listener = (
|
||||
message: unknown,
|
||||
sender: chrome.runtime.MessageSender,
|
||||
sendResponse: (response: RecordQueryResponse) => void,
|
||||
) => {
|
||||
if (!isRecordQueryMessage(message)) return false;
|
||||
const tabId = sender.tab?.id;
|
||||
if (tabId === undefined) {
|
||||
sendResponse({ active: false });
|
||||
return false;
|
||||
}
|
||||
void (async () => {
|
||||
const recording = await findRecordingForTab(tabId, deps);
|
||||
if (!recording) {
|
||||
sendResponse({ active: false });
|
||||
return;
|
||||
}
|
||||
await rearmRecording(recording, tabId, deps);
|
||||
sendResponse({ active: true, requestId: recording.requestId });
|
||||
})();
|
||||
return true;
|
||||
};
|
||||
chrome.runtime.onMessage.addListener(listener);
|
||||
return () => chrome.runtime.onMessage.removeListener(listener);
|
||||
}
|
||||
|
||||
async function finishRecordingByRequest(
|
||||
requestId: string,
|
||||
tabId: number,
|
||||
deps: RecordDeps,
|
||||
): Promise<void> {
|
||||
for (const [sessionId, recording] of recordings) {
|
||||
if (recording.requestId !== requestId || recording.settled) continue;
|
||||
const match = await findRecordingForTab(tabId, deps);
|
||||
if (match !== recording) continue;
|
||||
await finishRecording(sessionId, deps);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
async function finishRecording(sessionId: string, deps: RecordDeps): Promise<Trace | null> {
|
||||
const recording = recordings.get(sessionId);
|
||||
if (!recording || recording.settled || recording.finishing) return null;
|
||||
recording.finishing = true;
|
||||
|
||||
await clearRearmTimersForRecording(recording, deps);
|
||||
try {
|
||||
await stopRecordingOnAllAgentTabs(recording, deps);
|
||||
} catch {
|
||||
recording.finishing = false;
|
||||
return null;
|
||||
}
|
||||
|
||||
recording.settled = true;
|
||||
recordings.delete(sessionId);
|
||||
releaseBrowserObservationListenersIfIdle();
|
||||
const trace = buildTrace(recording);
|
||||
recording.resolveFinish(trace);
|
||||
return trace;
|
||||
}
|
||||
|
||||
export async function handleRecordStart(
|
||||
manager: SessionManager,
|
||||
params: RecordStartParams,
|
||||
deps: RecordDeps = getDefaultDeps(),
|
||||
): Promise<RecordStartResult | RpcError> {
|
||||
const ctxOrErr = lookupSession(manager, params, "record_start");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
if (recordings.has(params.session_id)) {
|
||||
return {
|
||||
code: "protocol_error",
|
||||
message: `session ${params.session_id} is already recording`,
|
||||
};
|
||||
}
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
|
||||
// Register the recording *before* navigate so content-script syncAgentOverlay
|
||||
// on the destination page can RECORD_QUERY → rearm → show RecordOverlay
|
||||
// instead of flashing ControlOverlay ("Agent 正在控制").
|
||||
const requestId = makeRequestId(target.tabId);
|
||||
let resolveFinish!: (trace: Trace) => void;
|
||||
let rejectFinish!: (err: Error) => void;
|
||||
const finishPromise = new Promise<Trace>((resolve, reject) => {
|
||||
resolveFinish = resolve;
|
||||
rejectFinish = reject;
|
||||
});
|
||||
const provisionalUrl = params.url;
|
||||
recordings.set(params.session_id, {
|
||||
requestId,
|
||||
tabId: target.tabId,
|
||||
agentWindowId: ctx.agentWindowId,
|
||||
startUrl: provisionalUrl,
|
||||
...(params.purpose ? { purpose: params.purpose } : {}),
|
||||
steps: [],
|
||||
startedAt: new Date().toISOString(),
|
||||
finishPromise,
|
||||
resolveFinish,
|
||||
rejectFinish,
|
||||
settled: false,
|
||||
finishing: false,
|
||||
currentUrl: provisionalUrl,
|
||||
pendingNavigation: false,
|
||||
pendingNavigationDeadline: undefined,
|
||||
});
|
||||
// Observe navigations for the whole recording lifetime; attach before
|
||||
// optional navigate so the destination load can rearm capture.
|
||||
ensureBrowserObservationListeners(deps);
|
||||
|
||||
const abortPending = async (notifyContent: boolean) => {
|
||||
recordings.delete(params.session_id);
|
||||
releaseBrowserObservationListenersIfIdle();
|
||||
if (notifyContent) {
|
||||
try {
|
||||
await deps.sendToTab(target.tabId, {
|
||||
type: RECORD_CANCEL,
|
||||
requestId,
|
||||
});
|
||||
} catch {
|
||||
// Content may never have received RECORD_START.
|
||||
}
|
||||
}
|
||||
if (deps.bypassOverlay) {
|
||||
try {
|
||||
await deps.bypassOverlay(target.tabId, false);
|
||||
} catch {
|
||||
// Ignore cleanup errors.
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const cancelledError = (): RpcError => ({
|
||||
code: "cancelled",
|
||||
message: "record_start aborted",
|
||||
});
|
||||
|
||||
/** Dispatcher may race-cancel while we still hold a provisional recording. */
|
||||
const abortIfCancelled = async (notifyContent: boolean): Promise<RpcError | null> => {
|
||||
if (!deps.signal?.aborted) return null;
|
||||
await abortPending(notifyContent);
|
||||
return cancelledError();
|
||||
};
|
||||
|
||||
{
|
||||
const cancelled = await abortIfCancelled(false);
|
||||
if (cancelled) return cancelled;
|
||||
}
|
||||
|
||||
if (params.url && deps.cdp) {
|
||||
const nav = await handleNavigate(
|
||||
manager,
|
||||
{
|
||||
session_id: params.session_id,
|
||||
url: params.url,
|
||||
tab_id: target.tabId,
|
||||
},
|
||||
{ cdp: deps.cdp, tabsApi: deps.tabsApi, signal: deps.signal },
|
||||
);
|
||||
if (isRpcError(nav)) {
|
||||
await abortPending(false);
|
||||
return nav;
|
||||
}
|
||||
{
|
||||
const cancelled = await abortIfCancelled(false);
|
||||
if (cancelled) return cancelled;
|
||||
}
|
||||
try {
|
||||
await waitForTabReady(target.tabId, deps.tabsApi);
|
||||
} catch {
|
||||
// Proceed with retries even if the tab never reports complete.
|
||||
}
|
||||
{
|
||||
const cancelled = await abortIfCancelled(false);
|
||||
if (cancelled) return cancelled;
|
||||
}
|
||||
}
|
||||
|
||||
let startUrl: string | undefined;
|
||||
try {
|
||||
const tab = await deps.tabsApi.get(target.tabId);
|
||||
startUrl = tab.url;
|
||||
} catch {
|
||||
startUrl = params.url;
|
||||
}
|
||||
|
||||
const active = recordings.get(params.session_id);
|
||||
if (!active) {
|
||||
// Cleared by a concurrent abort / session teardown.
|
||||
return cancelledError();
|
||||
}
|
||||
active.startUrl = startUrl;
|
||||
active.currentUrl = startUrl;
|
||||
|
||||
if (isContentScriptRestrictedUrl(startUrl)) {
|
||||
await abortPending(false);
|
||||
return {
|
||||
code: "invalid_params",
|
||||
message: params.url
|
||||
? `cannot record on restricted URL (${startUrl}); use an http(s) page`
|
||||
: "Agent Window starts on about:blank where the content script cannot inject — pass --url https://… before recording",
|
||||
};
|
||||
}
|
||||
|
||||
{
|
||||
const cancelled = await abortIfCancelled(false);
|
||||
if (cancelled) return cancelled;
|
||||
}
|
||||
|
||||
if (deps.bypassOverlay) {
|
||||
try {
|
||||
// Single ref for the initial race before RecordOverlay mounts; rearm must
|
||||
// not stack additional refs (see rearmRecording). Cleared on stop.
|
||||
await deps.bypassOverlay(target.tabId, true);
|
||||
} catch {
|
||||
// Best-effort; activeRecord also hides the control overlay.
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
const cancelled = await abortIfCancelled(true);
|
||||
if (cancelled) return cancelled;
|
||||
}
|
||||
|
||||
const startMsg: RecordStartMessage = { type: RECORD_START, requestId };
|
||||
|
||||
try {
|
||||
await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
|
||||
} catch {
|
||||
await abortPending(true);
|
||||
return {
|
||||
code: "protocol_error",
|
||||
message:
|
||||
"failed to start recording in content script — reload the BrowserSkill extension, then retry with --url https://…",
|
||||
};
|
||||
}
|
||||
|
||||
{
|
||||
const cancelled = await abortIfCancelled(true);
|
||||
if (cancelled) return cancelled;
|
||||
}
|
||||
|
||||
return { tab_id: target.tabId, recording: true };
|
||||
}
|
||||
|
||||
export async function handleRecordStop(
|
||||
manager: SessionManager,
|
||||
params: RecordStopParams,
|
||||
deps: RecordDeps = getDefaultDeps(),
|
||||
): Promise<RecordStopResult | RpcError> {
|
||||
const ctxOrErr = lookupSession(manager, params, "record_stop");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
|
||||
const recording = recordings.get(params.session_id);
|
||||
if (!recording) {
|
||||
return {
|
||||
code: "not_found",
|
||||
message: `no active recording for session ${params.session_id}`,
|
||||
};
|
||||
}
|
||||
|
||||
const trace = await finishRecording(params.session_id, deps);
|
||||
if (!trace) {
|
||||
return {
|
||||
code: "protocol_error",
|
||||
message: `failed to flush recorded steps for session ${params.session_id}; the recording is still active — retry \`bsk record stop\``,
|
||||
};
|
||||
}
|
||||
return { trace };
|
||||
}
|
||||
|
||||
export async function handleRecordAwait(
|
||||
manager: SessionManager,
|
||||
params: RecordAwaitParams,
|
||||
deps: RecordDeps = getDefaultDeps(),
|
||||
): Promise<RecordAwaitResult | RpcError> {
|
||||
const ctxOrErr = lookupSession(manager, params, "record_await");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
|
||||
const recording = recordings.get(params.session_id);
|
||||
if (!recording) {
|
||||
return {
|
||||
code: "not_found",
|
||||
message: `no active recording for session ${params.session_id}`,
|
||||
};
|
||||
}
|
||||
|
||||
if (deps.signal?.aborted) {
|
||||
return { code: "cancelled", message: "record_await aborted" };
|
||||
}
|
||||
|
||||
const outcome = await new Promise<{ trace: Trace } | { error: RpcError }>((resolve) => {
|
||||
let settled = false;
|
||||
const finish = (result: { trace: Trace } | { error: RpcError }) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (timer) clearTimeout(timer);
|
||||
deps.signal?.removeEventListener("abort", onAbort);
|
||||
resolve(result);
|
||||
};
|
||||
const onAbort = () => finish({ error: { code: "cancelled", message: "record_await aborted" } });
|
||||
const timer =
|
||||
params.timeout_ms === undefined
|
||||
? undefined
|
||||
: setTimeout(
|
||||
() =>
|
||||
finish({
|
||||
error: {
|
||||
code: "timeout",
|
||||
message: `record_await timed out after ${params.timeout_ms}ms`,
|
||||
},
|
||||
}),
|
||||
params.timeout_ms,
|
||||
);
|
||||
deps.signal?.addEventListener("abort", onAbort, { once: true });
|
||||
void recording.finishPromise.then(
|
||||
(trace) => finish({ trace }),
|
||||
() =>
|
||||
finish({
|
||||
error: { code: "cancelled", message: "recording was cleared" },
|
||||
}),
|
||||
);
|
||||
});
|
||||
return "trace" in outcome ? { trace: outcome.trace } : outcome.error;
|
||||
}
|
||||
|
||||
export function clearRecordingForSession(sessionId: string): void {
|
||||
const recording = recordings.get(sessionId);
|
||||
if (!recording) {
|
||||
recordings.delete(sessionId);
|
||||
releaseBrowserObservationListenersIfIdle();
|
||||
return;
|
||||
}
|
||||
void clearRearmTimersForRecording(recording, getDefaultDeps());
|
||||
if (!recording.settled) {
|
||||
recording.settled = true;
|
||||
recording.rejectFinish(new Error("recording cleared"));
|
||||
}
|
||||
recordings.delete(sessionId);
|
||||
releaseBrowserObservationListenersIfIdle();
|
||||
}
|
||||
|
||||
export type { RecordFinishMessage };
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { clearRecordingForSession } from "./record";
|
||||
import { returnBorrowedTab, type TabManagementDeps } from "./tabs";
|
||||
|
||||
export interface SessionStartParams {
|
||||
@@ -156,6 +157,8 @@ export async function handleSessionStop(
|
||||
// Step 2: clear the per-session RefStore (review M6/M7 parity).
|
||||
ctx.refStore.clear();
|
||||
|
||||
clearRecordingForSession(params.session_id);
|
||||
|
||||
// Step 3: detach CDP sessions this session opened (no-op if none).
|
||||
await deps.cdp?.detachSession(params.session_id);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user