Compare commits

...
46 changed files with 2878 additions and 527 deletions
@@ -1,6 +1,6 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { RECORD_START, RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
import { handleRecordContentMessage, startRecordCapture } from "../record-capture";
import type { RecordStepPayload } from "@/lib/record-bridge";
import { startRecordCapture } from "../record-capture";
vi.stubGlobal("chrome", {
runtime: {
@@ -48,118 +48,6 @@ function click(el: Element): void {
el.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
}
describe("handleRecordContentMessage stop/cancel", () => {
it("ignores STOP when no recording is active", () => {
const dispose = vi.fn();
const onStop = vi.fn();
const setActiveRequestId = vi.fn();
const setCapture = vi.fn();
const sendResponse = vi.fn();
const needsAsync = handleRecordContentMessage(
{ type: RECORD_STOP, requestId: "rec-stale" },
{
activeRequestId: null,
capture: { dispose },
setActiveRequestId,
setCapture,
onStart: vi.fn(),
onStop,
},
sendResponse,
);
expect(needsAsync).toBe(false);
expect(dispose).not.toHaveBeenCalled();
expect(onStop).not.toHaveBeenCalled();
expect(setActiveRequestId).not.toHaveBeenCalled();
expect(setCapture).not.toHaveBeenCalled();
expect(sendResponse).not.toHaveBeenCalled();
});
it("ignores STOP for a mismatched requestId", () => {
const dispose = vi.fn();
const onStop = vi.fn();
const needsAsync = handleRecordContentMessage(
{ type: RECORD_STOP, requestId: "rec-other" },
{
activeRequestId: "rec-1",
capture: { dispose },
setActiveRequestId: vi.fn(),
setCapture: vi.fn(),
onStart: vi.fn(),
onStop,
},
vi.fn(),
);
expect(needsAsync).toBe(false);
expect(dispose).not.toHaveBeenCalled();
expect(onStop).not.toHaveBeenCalled();
});
it("keeps a failed STOP retryable and redelivers its recorded step", async () => {
document.body.innerHTML = `<label for="retry">Draft</label><input id="retry" />`;
const sendMessage = vi.mocked(chrome.runtime.sendMessage);
sendMessage.mockReset();
sendMessage.mockRejectedValueOnce(new Error("service worker unavailable"));
sendMessage.mockRejectedValueOnce(new Error("service worker still unavailable"));
sendMessage.mockResolvedValueOnce({ ok: true, sequence: 1 });
let activeRequestId: string | null = null;
let capture: ReturnType<typeof startRecordCapture> | null = null;
const onStop = vi.fn();
const dispatch = (
message:
| { type: typeof RECORD_START; requestId: string; startedAtMs?: number }
| { type: typeof RECORD_STOP; requestId: string },
sendResponse?: (response: unknown) => void,
) =>
handleRecordContentMessage(
message,
{
activeRequestId,
capture,
setActiveRequestId: (id) => {
activeRequestId = id;
},
setCapture: (next) => {
capture = next;
},
onStart: vi.fn(),
onStop,
},
sendResponse,
);
dispatch({ type: RECORD_START, requestId: "rec-retry" });
const input = document.querySelector("input")!;
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
input.value = "dirty final value";
input.dispatchEvent(new Event("input", { bubbles: true }));
const firstResponse = vi.fn();
expect(dispatch({ type: RECORD_STOP, requestId: "rec-retry" }, firstResponse)).toBe(true);
await vi.waitFor(() =>
expect(firstResponse).toHaveBeenCalledWith({
ok: false,
error: "failed to deliver one or more recorded steps",
}),
);
expect(activeRequestId).toBe("rec-retry");
expect(onStop).not.toHaveBeenCalled();
const retryResponse = vi.fn();
expect(dispatch({ type: RECORD_STOP, requestId: "rec-retry" }, retryResponse)).toBe(true);
await vi.waitFor(() => expect(retryResponse).toHaveBeenCalledWith({ ok: true }));
expect(sendMessage).toHaveBeenCalledTimes(3);
expect(activeRequestId).toBeNull();
expect(onStop).toHaveBeenCalledTimes(1);
});
});
describe("record-capture semantic", () => {
let steps: RecordStepPayload[];
@@ -236,6 +124,19 @@ describe("record-capture semantic", () => {
]);
});
it("does not emit page navigation from a child Document capture", () => {
const originalUrl = location.href;
const capture = startRecordCapture("rec-child", (step) => steps.push(step), {
captureNavigation: false,
});
history.pushState({}, "", "#inside-frame");
expect(steps).toEqual([]);
capture.dispose();
history.replaceState({}, "", originalUrl);
});
it("does not record clicks on anonymous layout divs", () => {
document.body.innerHTML = `
<div id="chrome">page chrome</div>
@@ -0,0 +1,102 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { RecordFramePortMessage } from "@/lib/recording/frame-bridge";
import { RECORD_FRAME_START } from "@/lib/recording/frame-bridge";
import { RECORD_DOCUMENT_ATTRIBUTE } from "@/shared/recording-document-identity";
import { RecordFrameAgent } from "../recording/frame-agent";
class PortListeners<T extends (...args: never[]) => unknown> {
readonly values = new Set<T>();
addListener = (listener: T) => this.values.add(listener);
removeListener = (listener: T) => this.values.delete(listener);
}
function portHarness() {
const onMessage = new PortListeners<(message: unknown) => void>();
const onDisconnect = new PortListeners<() => void>();
const outbound: RecordFramePortMessage[] = [];
const port = {
name: "bsk-record-frame",
onMessage,
onDisconnect,
postMessage(message: RecordFramePortMessage) {
outbound.push(message);
if (message.type === "ready") {
queueMicrotask(() => {
for (const listener of onMessage.values) {
listener({
type: "ready_ack",
requestId: message.requestId,
producerId: message.producerId,
});
}
});
}
},
disconnect: vi.fn(),
} as unknown as chrome.runtime.Port;
return {
port,
outbound,
receive(message: RecordFramePortMessage) {
for (const listener of onMessage.values) listener(message);
},
};
}
describe("RecordFrameAgent", () => {
afterEach(() => {
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
document.body.replaceChildren();
});
it("keeps a failed stop retryable and flushes the final dirty fill", async () => {
const harness = portHarness();
const sendMessage = vi
.fn<(message: { sequence: number }) => Promise<unknown>>()
.mockRejectedValueOnce(new Error("offline"))
.mockRejectedValueOnce(new Error("still offline"))
.mockImplementation(async (message) => ({ ok: true, sequence: message.sequence }));
vi.stubGlobal("chrome", {
runtime: {
connect: vi.fn(() => harness.port),
sendMessage,
},
});
document.body.innerHTML = `<label for="draft">Draft</label><input id="draft" />`;
const agent = new RecordFrameAgent();
await expect(
agent.start({ type: RECORD_FRAME_START, requestId: "rec-1", startedAtMs: 10 }),
).resolves.toEqual({ ok: true });
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true);
const input = document.querySelector("input")!;
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
input.value = "final value";
input.dispatchEvent(new Event("input", { bubbles: true }));
harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-1" });
await vi.waitFor(() =>
expect(harness.outbound).toContainEqual({
type: "stopped",
requestId: "rec-1",
commandId: "stop-1",
ok: false,
error: "failed to deliver one or more recorded steps",
}),
);
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true);
harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-2" });
await vi.waitFor(() =>
expect(harness.outbound).toContainEqual({
type: "stopped",
requestId: "rec-1",
commandId: "stop-2",
ok: true,
}),
);
expect(sendMessage).toHaveBeenCalledTimes(3);
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(false);
});
});
+30 -131
View File
@@ -12,21 +12,7 @@ import {
hasDirectHoverInteractiveSignal,
hasStrongHoverExpansionSignal,
} from "@/lib/hover-trigger-policy";
import {
isRecordCancelMessage,
isRecordStartMessage,
isRecordStopMessage,
RECORD_CANCEL,
RECORD_START,
RECORD_STEP,
RECORD_STOP,
type RecordCancelMessage,
type RecordStartAck,
type RecordStartMessage,
type RecordStepPayload,
type RecordStopAck,
type RecordStopMessage,
} from "@/lib/record-bridge";
import type { RecordStepPayload } from "@/lib/record-bridge";
import { shouldRecordPress } from "@/lib/recording/draft-policy";
import {
closestHoverSurfaceCandidate,
@@ -36,18 +22,6 @@ import {
isHoverSurfaceCandidateElement,
isLikelyHoverSurfaceOwner,
} from "./record-hover-surface";
import { RecordStepDelivery } from "./record-step-delivery";
const stepDeliveries = new Map<string, RecordStepDelivery>();
const pendingStopFlushes = new Map<string, Promise<RecordStopAck>>();
function deliveryFor(requestId: string): RecordStepDelivery {
const existing = stepDeliveries.get(requestId);
if (existing) return existing;
const created = new RecordStepDelivery(requestId);
stepDeliveries.set(requestId, created);
return created;
}
export interface RecordCaptureController {
dispose(): void;
@@ -385,6 +359,7 @@ function scheduleInputCompletionCommit(
export function startRecordCapture(
_requestId: string,
sendStep: (step: RecordStepPayload) => void,
options: { captureNavigation?: boolean } = {},
): RecordCaptureController {
const emitStep = (step: RecordStepPayload) => {
sendStep({ page_url: location.href, ...step });
@@ -921,26 +896,33 @@ export function startRecordCapture(
document.addEventListener("change", onChange, true);
document.addEventListener("keydown", onKeyDown, true);
const urlObserver = new MutationObserver(() => emitNavigateIfChanged());
urlObserver.observe(document, { subtree: true, childList: true });
const captureNavigation = options.captureNavigation ?? true;
const urlObserver = captureNavigation
? new MutationObserver(() => emitNavigateIfChanged())
: null;
urlObserver?.observe(document, { subtree: true, childList: true });
const onUrlEvent = () => emitNavigateIfChanged();
// Wrap: passing commitFillSession directly would forward the DOM event as
// the `commit` argument and stamp it onto the recorded fill step.
const onPageHide = () => commitFillSession();
window.addEventListener("hashchange", onUrlEvent);
window.addEventListener("popstate", onUrlEvent);
if (captureNavigation) {
window.addEventListener("hashchange", onUrlEvent);
window.addEventListener("popstate", onUrlEvent);
}
window.addEventListener("pagehide", onPageHide);
const originalPushState = history.pushState;
const originalReplaceState = history.replaceState;
history.pushState = function (...args: Parameters<History["pushState"]>) {
originalPushState.apply(this, args);
emitNavigateIfChanged(navigationActionPending ? true : undefined);
};
history.replaceState = function (...args: Parameters<History["replaceState"]>) {
originalReplaceState.apply(this, args);
emitNavigateIfChanged(navigationActionPending ? true : undefined);
};
if (captureNavigation) {
history.pushState = function (...args: Parameters<History["pushState"]>) {
originalPushState.apply(this, args);
emitNavigateIfChanged(navigationActionPending ? true : undefined);
};
history.replaceState = function (...args: Parameters<History["replaceState"]>) {
originalReplaceState.apply(this, args);
emitNavigateIfChanged(navigationActionPending ? true : undefined);
};
}
return {
dispose() {
@@ -954,99 +936,16 @@ export function startRecordCapture(
document.removeEventListener("compositionend", onCompositionEnd, true);
document.removeEventListener("change", onChange, true);
document.removeEventListener("keydown", onKeyDown, true);
urlObserver.disconnect();
window.removeEventListener("hashchange", onUrlEvent);
window.removeEventListener("popstate", onUrlEvent);
urlObserver?.disconnect();
if (captureNavigation) {
window.removeEventListener("hashchange", onUrlEvent);
window.removeEventListener("popstate", onUrlEvent);
}
window.removeEventListener("pagehide", onPageHide);
history.pushState = originalPushState;
history.replaceState = originalReplaceState;
if (captureNavigation) {
history.pushState = originalPushState;
history.replaceState = originalReplaceState;
}
},
};
}
export type RecordContentMessage = RecordStartMessage | RecordStopMessage | RecordCancelMessage;
export function isRecordContentMessage(msg: unknown): msg is RecordContentMessage {
return isRecordStartMessage(msg) || isRecordStopMessage(msg) || isRecordCancelMessage(msg);
}
export function handleRecordContentMessage(
message: RecordContentMessage,
state: {
activeRequestId: string | null;
capture: RecordCaptureController | null;
setActiveRequestId(id: string | null): void;
setCapture(capture: RecordCaptureController | null): void;
onStart(requestId: string, startedAtMs?: number): void;
onStop(): void;
},
sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
): boolean {
if (isRecordStartMessage(message)) {
const delivery = deliveryFor(message.requestId);
state.capture?.dispose();
state.setCapture(
startRecordCapture(message.requestId, (step) => {
delivery.enqueue(step);
}),
);
state.setActiveRequestId(message.requestId);
state.onStart(message.requestId, message.startedAtMs);
sendResponse?.({ ok: true });
return sendResponse !== undefined;
}
if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
// Require an active recording that matches this requestId — otherwise a
// stray STOP/CANCEL (e.g. after teardown) would still run finishStop and
// clear overlay state even though nothing was capturing.
if (!state.activeRequestId || state.activeRequestId !== message.requestId) {
return false;
}
state.capture?.dispose();
state.setCapture(null);
const finishStop = () => {
state.onStop();
state.setActiveRequestId(null);
};
if (isRecordStopMessage(message) && sendResponse) {
const existingFlush = pendingStopFlushes.get(message.requestId);
if (existingFlush) {
void existingFlush.then(sendResponse);
return true;
}
const flush = deliveryFor(message.requestId)
.flush()
.then((succeeded): RecordStopAck => {
if (succeeded) {
stepDeliveries.delete(message.requestId);
finishStop();
}
return succeeded
? { ok: true }
: {
ok: false,
error: "failed to deliver one or more recorded steps",
};
});
pendingStopFlushes.set(message.requestId, flush);
void flush.then(sendResponse).finally(() => {
if (pendingStopFlushes.get(message.requestId) === flush) {
pendingStopFlushes.delete(message.requestId);
}
});
return true;
}
if (isRecordCancelMessage(message)) {
stepDeliveries.delete(message.requestId);
pendingStopFlushes.delete(message.requestId);
}
finishStop();
return false;
}
return false;
}
export { RECORD_CANCEL, RECORD_START, RECORD_STEP, RECORD_STOP };
@@ -0,0 +1,209 @@
import {
markRecordingDocument,
type RecordingDocumentMarker,
waitForDocumentElement,
} from "@/lib/recording/document-marker";
import {
isRecordFrameStartMessage,
RECORD_FRAME_PORT,
RECORD_FRAME_QUERY,
RECORD_FRAME_START,
type RecordFramePortMessage,
type RecordFrameQueryResponse,
type RecordFrameStartMessage,
} from "@/lib/recording/frame-bridge";
import { type RecordCaptureController, startRecordCapture } from "../record-capture";
import { RecordStepDelivery } from "../record-step-delivery";
interface ActiveFrameRecording {
requestId: string;
producerId: string;
port: chrome.runtime.Port;
delivery: RecordStepDelivery;
capture: RecordCaptureController;
marker: RecordingDocumentMarker;
}
const RECORD_FRAME_REGISTER_TIMEOUT_MS = 5_000;
export class RecordFrameAgent {
#active: ActiveFrameRecording | null = null;
#startPromise: Promise<{ ok: true } | { ok: false; error: string }> | null = null;
#disposed = false;
async start(
message: RecordFrameStartMessage,
): Promise<{ ok: true } | { ok: false; error: string }> {
if (this.#active?.requestId === message.requestId) return { ok: true };
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
if (this.#startPromise) {
await this.#startPromise;
return this.start(message);
}
if (this.#active) this.cancel(this.#active.requestId);
const starting = this.#start(message);
this.#startPromise = starting;
try {
return await starting;
} finally {
if (this.#startPromise === starting) this.#startPromise = null;
}
}
cancel(requestId: string): void {
const active = this.#active;
if (!active || active.requestId !== requestId) return;
active.capture.dispose();
active.marker.restore();
active.port.disconnect();
this.#active = null;
}
dispose(): void {
this.#disposed = true;
if (this.#active) this.cancel(this.#active.requestId);
}
async #start(
message: RecordFrameStartMessage,
): Promise<{ ok: true } | { ok: false; error: string }> {
const producerId = crypto.randomUUID();
const root = await waitForDocumentElement();
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
const marker = markRecordingDocument(producerId, root);
const port = chrome.runtime.connect({ name: RECORD_FRAME_PORT });
try {
await registerPort(port, message.requestId, producerId);
} catch {
marker.restore();
port.disconnect();
return { ok: false, error: "failed to register recording document" };
}
if (this.#disposed) {
marker.restore();
port.disconnect();
return { ok: false, error: "recording document is disposed" };
}
const delivery = new RecordStepDelivery(message.requestId, undefined, producerId);
const capture = startRecordCapture(
message.requestId,
(step) => {
marker.ensure();
delivery.enqueue(step);
},
{ captureNavigation: window.top === window },
);
const active: ActiveFrameRecording = {
requestId: message.requestId,
producerId,
port,
delivery,
capture,
marker,
};
this.#active = active;
port.onMessage.addListener((raw: unknown) => {
const command = raw as Partial<RecordFramePortMessage>;
if (command.type === "stop" && command.requestId === active.requestId) {
void this.#stop(active, command.commandId);
} else if (command.type === "cancel" && command.requestId === active.requestId) {
this.cancel(active.requestId);
}
});
port.onDisconnect.addListener(() => {
if (this.#active !== active) return;
active.capture.dispose();
active.marker.restore();
this.#active = null;
});
return { ok: true };
}
async #stop(active: ActiveFrameRecording, commandId: string | undefined): Promise<void> {
if (!commandId || this.#active !== active) return;
active.capture.dispose();
const succeeded = await active.delivery.flush();
try {
active.port.postMessage({
type: "stopped",
requestId: active.requestId,
commandId,
ok: succeeded,
...(succeeded ? {} : { error: "failed to deliver one or more recorded steps" }),
} satisfies RecordFramePortMessage);
} catch {
return;
}
if (!succeeded || this.#active !== active) return;
active.marker.restore();
this.#active = null;
active.port.disconnect();
}
}
function registerPort(
port: chrome.runtime.Port,
requestId: string,
producerId: string,
): Promise<void> {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => {
cleanup();
reject(new Error("recording document registration timed out"));
}, RECORD_FRAME_REGISTER_TIMEOUT_MS);
const cleanup = () => {
clearTimeout(timer);
port.onMessage.removeListener(onMessage);
port.onDisconnect.removeListener(onDisconnect);
};
const onMessage = (raw: unknown) => {
const message = raw as Partial<RecordFramePortMessage>;
if (
message.type !== "ready_ack" ||
message.requestId !== requestId ||
message.producerId !== producerId
) {
return;
}
cleanup();
resolve();
};
const onDisconnect = () => {
cleanup();
reject(new Error("recording document port disconnected"));
};
port.onMessage.addListener(onMessage);
port.onDisconnect.addListener(onDisconnect);
port.postMessage({ type: "ready", requestId, producerId } satisfies RecordFramePortMessage);
});
}
export function attachRecordFrameAgent(): () => void {
const agent = new RecordFrameAgent();
const onMessage = (
message: unknown,
_sender: chrome.runtime.MessageSender,
sendResponse: (response: { ok: true } | { ok: false; error: string }) => void,
) => {
if (!isRecordFrameStartMessage(message)) return false;
void agent.start(message).then(sendResponse);
return true;
};
chrome.runtime.onMessage.addListener(onMessage);
void chrome.runtime
.sendMessage({ type: RECORD_FRAME_QUERY })
.then((response: RecordFrameQueryResponse | undefined) => {
if (!response?.active || !response.requestId || response.startedAtMs === undefined) return;
return agent.start({
type: RECORD_FRAME_START,
requestId: response.requestId,
startedAtMs: response.startedAtMs,
});
})
.catch(() => {});
return () => {
chrome.runtime.onMessage.removeListener(onMessage);
agent.dispose();
};
}
+25 -22
View File
@@ -21,6 +21,7 @@ import {
type OverlayMode,
} from "@/lib/overlay-bridge";
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
import { recordFrameCoordinator } from "@/lib/recording/frame-coordinator";
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup";
import { attachSessionEventHandler } from "@/session-manager/event-handler";
@@ -38,6 +39,7 @@ import {
attachRecordFinishListener,
attachRecordQueryListener,
attachRecordStepListener,
type RecordRuntimeDeps,
} from "@/tools/record";
import { detectBrowserMeta } from "@/transport/handshake";
import type { Transport } from "@/transport/transport";
@@ -145,10 +147,33 @@ export default defineBackground(() => {
void sessionsLive.syncFromManager();
},
});
const recordDeps = {
tabsApi: chrome.tabs,
cdp,
frameCoordinator: recordFrameCoordinator,
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
chrome.tabs.sendMessage(tabId, msg),
bypassOverlay: async (tabId: number, enabled: boolean) => {
try {
await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
} satisfies RecordRuntimeDeps;
recordFrameCoordinator.attach();
attachRecordStepListener(recordDeps);
attachRecordFinishListener(recordDeps);
attachRecordQueryListener(recordDeps);
const dispatcher = new ToolDispatcher({
transport,
sessions,
cdp,
recording: recordDeps,
onSessionsChanged: onOverlaySessionStateChanged,
onBrowserControlResumed,
approveBorrow: (ctx) =>
@@ -170,28 +195,6 @@ export default defineBackground(() => {
}),
});
dispatcher.start();
const recordDeps = {
tabsApi: chrome.tabs,
cdp,
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
chrome.tabs.sendMessage(tabId, msg),
bypassOverlay: async (tabId: number, enabled: boolean) => {
try {
await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
};
// Message listeners stay up (cheap; fire only for record message types).
// Tab / webNavigation observation attaches lazily while a recording is
// active — see ensureBrowserObservationListeners in tools/record.ts.
attachRecordStepListener(recordDeps);
attachRecordFinishListener(recordDeps);
attachRecordQueryListener(recordDeps);
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
attachBorrowNotificationClickHandler({
onClicked: chrome.notifications.onClicked,
+34 -45
View File
@@ -11,11 +11,6 @@ import { createHelpRequestData } from "@/content/help-request";
import overlayCss from "@/content/overlay.css?inline";
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
import { RecordOverlay } from "@/content/RecordOverlay";
import {
handleRecordContentMessage,
isRecordContentMessage,
type RecordCaptureController,
} from "@/content/record-capture";
import {
type CaptureSuppressAck,
type CaptureSuppressMessage,
@@ -45,11 +40,17 @@ import {
} from "@/lib/overlay-bridge";
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
import {
isRecordCancelMessage,
isRecordStartMessage,
isRecordStopMessage,
RECORD_FINISH,
RECORD_QUERY,
type RecordCancelMessage,
type RecordQueryResponse,
type RecordStartAck,
type RecordStartMessage,
type RecordStopAck,
type RecordStopMessage,
} from "@/lib/record-bridge";
import type {
BorrowCancelMessage,
@@ -69,7 +70,6 @@ export default defineContentScript({
if (window.top !== window) return;
const overlays = new OverlayController();
let recordCapture: RecordCaptureController | null = null;
let activeRecordRequestId: string | null = null;
let reactRoot: ReactDOM.Root | null = null;
let overlayHost: HTMLElement | null = null;
@@ -228,8 +228,6 @@ export default defineContentScript({
if (previousHelp) {
void sendHelpFinish(previousHelp.id, "cancelled");
}
recordCapture?.dispose();
recordCapture = null;
activeRecordRequestId = null;
renderAll();
}
@@ -259,6 +257,9 @@ export default defineContentScript({
| HelpRequestMessage
| HelpCancelMessage
| CaptureSuppressMessage
| RecordStartMessage
| RecordStopMessage
| RecordCancelMessage
| OverlayAgentOverlayResetMessage
| OverlayAgentStateMessage
| OverlayAutomationBypassMessage,
@@ -269,41 +270,32 @@ export default defineContentScript({
return captureSuppress.handleMessage(message, sendResponse);
}
if (isRecordContentMessage(message)) {
const needsAsync = handleRecordContentMessage(
message,
{
activeRequestId: activeRecordRequestId,
capture: recordCapture,
setActiveRequestId: (id) => {
activeRecordRequestId = id;
},
setCapture: (capture) => {
recordCapture = capture;
},
onStart: (requestId, startedAtMs) => {
overlays.setAgentRecordRequest({
id: requestId,
...(typeof startedAtMs === "number" ? { startedAtMs } : {}),
onFinish: () => {
void chrome.runtime.sendMessage({
type: RECORD_FINISH,
requestId,
});
},
});
renderAll();
},
onStop: () => {
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
renderAll();
},
if (isRecordStartMessage(message)) {
activeRecordRequestId = message.requestId;
overlays.setAgentRecordRequest({
id: message.requestId,
...(typeof message.startedAtMs === "number" ? { startedAtMs: message.startedAtMs } : {}),
onFinish: () => {
void chrome.runtime.sendMessage({
type: RECORD_FINISH,
requestId: message.requestId,
});
},
sendResponse as unknown as
| ((response: RecordStartAck | RecordStopAck) => void)
| undefined,
);
return needsAsync;
});
renderAll();
(sendResponse as unknown as (response: RecordStartAck) => void)({ ok: true });
return false;
}
if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
if (activeRecordRequestId !== message.requestId) return false;
overlays.clearAgentRecordRequest(message.requestId);
activeRecordRequestId = null;
renderAll();
if (isRecordStopMessage(message)) {
(sendResponse as unknown as (response: RecordStopAck) => void)({ ok: true });
}
return false;
}
if (
@@ -526,9 +518,6 @@ export default defineContentScript({
chrome.runtime.onMessage.removeListener(onMessage);
chrome.storage.onChanged.removeListener(onStorageChange);
window.removeEventListener("pageshow", onPageShow);
// Restore history hooks / remove capture listeners before the CS unloads.
recordCapture?.dispose();
recordCapture = null;
activeRecordRequestId = null;
});
},
@@ -0,0 +1,14 @@
import { attachRecordFrameAgent } from "@/content/recording/frame-agent";
export default defineContentScript({
matches: ["<all_urls>"],
runAt: "document_start",
allFrames: true,
matchAboutBlank: true,
matchOriginAsFallback: true,
main(ctx) {
const dispose = attachRecordFrameAgent();
ctx.onInvalidated(dispose);
},
});
@@ -0,0 +1,213 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { RECORD_FRAME_PORT, RECORD_FRAME_QUERY } from "../recording/frame-bridge";
import { RecordFrameCoordinator } from "../recording/frame-coordinator";
class ListenerSet<T extends (...args: never[]) => unknown> {
readonly listeners = new Set<T>();
addListener = (listener: T) => this.listeners.add(listener);
removeListener = (listener: T) => this.listeners.delete(listener);
}
function fakePort(sender: chrome.runtime.MessageSender) {
const messages: unknown[] = [];
const onMessage = new ListenerSet<(message: unknown) => void>();
const onDisconnect = new ListenerSet<() => void>();
return {
port: {
name: RECORD_FRAME_PORT,
sender,
onMessage,
onDisconnect,
postMessage: vi.fn((message: unknown) => messages.push(message)),
disconnect: vi.fn(),
} as unknown as chrome.runtime.Port,
messages,
receive(message: unknown) {
for (const listener of onMessage.listeners) listener(message);
},
};
}
describe("RecordFrameCoordinator", () => {
const onConnect = new ListenerSet<(port: chrome.runtime.Port) => void>();
const onMessage = new ListenerSet<
(
message: unknown,
sender: chrome.runtime.MessageSender,
sendResponse: (response: unknown) => void,
) => boolean
>();
beforeEach(() => {
onConnect.listeners.clear();
onMessage.listeners.clear();
vi.stubGlobal("chrome", {
runtime: { onConnect, onMessage },
});
});
it("answers child-document queries for the initial tab before it is armed", () => {
const coordinator = new RecordFrameCoordinator({
getAllFrames: async () => [],
sendToDocument: async () => ({ ok: true }),
});
coordinator.attach();
coordinator.begin("rec-1", 10, 3);
const query = (tabId: number) => {
let response: unknown;
for (const listener of onMessage.listeners) {
listener(
{ type: RECORD_FRAME_QUERY },
{
tab: { id: tabId },
frameId: 7,
documentId: "child-document",
} as chrome.runtime.MessageSender,
(value) => {
response = value;
},
);
}
return response;
};
expect(query(3)).toEqual({ active: true, requestId: "rec-1", startedAtMs: 10 });
expect(query(4)).toEqual({ active: false });
});
it("starts a child Document that appears after the initial frame snapshot", async () => {
let notifyFrameNavigation:
| ((frame: {
tabId: number;
frameId: number;
documentId?: string;
lifecycle: "committed" | "completed";
}) => void)
| undefined;
const sendToDocument = vi.fn(async () => ({ ok: true }));
const coordinator = new RecordFrameCoordinator({
getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }],
sendToDocument,
subscribeFrameNavigation(listener) {
notifyFrameNavigation = listener;
return () => {};
},
});
coordinator.attach();
const onDocumentReady = vi.fn();
coordinator.begin("rec-1", 10, 3, onDocumentReady);
await coordinator.armTab("rec-1", 3);
sendToDocument.mockClear();
notifyFrameNavigation?.({
tabId: 3,
frameId: 7,
documentId: "late-child",
lifecycle: "completed",
});
await vi.waitFor(() => {
expect(sendToDocument).toHaveBeenCalledWith(
3,
{ type: "bsk-record-frame-start", requestId: "rec-1", startedAtMs: 10 },
{ documentId: "late-child" },
);
});
expect(onDocumentReady).not.toHaveBeenCalled();
const child = fakePort({
tab: { id: 3 },
frameId: 7,
documentId: "late-child",
} as chrome.runtime.MessageSender);
for (const listener of onConnect.listeners) listener(child.port);
child.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
notifyFrameNavigation?.({
tabId: 3,
frameId: 7,
documentId: "late-child",
lifecycle: "completed",
});
await vi.waitFor(() => {
expect(onDocumentReady).toHaveBeenCalledWith({
tabId: 3,
documentId: "late-child",
browserFrameId: 7,
producerId: "producer-1",
});
});
});
it("binds a producer to its sender Document and keeps final steps valid while stopping", async () => {
const sendToDocument = vi.fn(async () => ({ ok: true }));
const coordinator = new RecordFrameCoordinator({
getAllFrames: async () => [
{ frameId: 0, documentId: "top-document" },
{ frameId: 7, documentId: "child-document" },
],
sendToDocument,
});
coordinator.attach();
coordinator.begin("rec-1", 10, 3);
await expect(coordinator.armTab("rec-1", 3)).resolves.toBe(true);
const sender = {
tab: { id: 3 },
frameId: 7,
documentId: "child-document",
} as chrome.runtime.MessageSender;
const frame = fakePort(sender);
for (const listener of onConnect.listeners) listener(frame.port);
frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toEqual({
tabId: 3,
documentId: "child-document",
browserFrameId: 7,
producerId: "producer-1",
});
const stopping = coordinator.stop("rec-1");
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).not.toBeNull();
const stop = frame.messages.find(
(message): message is { type: "stop"; commandId: string } =>
typeof message === "object" &&
message !== null &&
(message as { type?: string }).type === "stop",
);
expect(stop).toBeDefined();
frame.receive({
type: "stopped",
requestId: "rec-1",
commandId: stop!.commandId,
ok: true,
});
await expect(stopping).resolves.toBe(true);
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toBeNull();
});
it("rejects a producer used from another Document", async () => {
const coordinator = new RecordFrameCoordinator({
getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }],
sendToDocument: async () => ({ ok: true }),
});
coordinator.attach();
coordinator.begin("rec-1", 10, 3);
await coordinator.armTab("rec-1", 3);
const sender = {
tab: { id: 3 },
frameId: 0,
documentId: "top-document",
} as chrome.runtime.MessageSender;
const frame = fakePort(sender);
for (const listener of onConnect.listeners) listener(frame.port);
frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
expect(
coordinator.sourceFor("rec-1", "producer-1", {
...sender,
documentId: "other-document",
}),
).toBeNull();
});
});
@@ -0,0 +1,70 @@
import { describe, expect, it } from "vitest";
import {
RECORD_DOCUMENT_ATTRIBUTE,
recordingDocumentMarkerValue,
} from "@/shared/recording-document-identity";
import { markRecordingDocument } from "../recording/document-marker";
import { ObservationNodeIndex } from "../recording/observation-capture";
describe("recording document marker", () => {
it("rejects a non-random Document identity", () => {
expect(() => markRecordingDocument("user@example.com")).toThrow(
"recording Document identity must be a random UUID",
);
});
it("restores the Document attribute exactly", () => {
document.documentElement.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, "page-value");
const producerId = "123e4567-e89b-42d3-a456-426614174000";
const marker = markRecordingDocument(producerId);
expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(
recordingDocumentMarkerValue(producerId),
);
marker.restore();
expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe("page-value");
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
});
it("indexes the CDP scope from the safe frame identity", () => {
const index = new ObservationNodeIndex({
rootFrameId: "root",
frames: [
{
frameId: "child",
target: { tabId: 3, sessionId: "oopif-session" },
recordingDocumentId: "producer-1",
},
],
matchNodes: [],
refs: [],
});
expect(index.documentScope("producer-1")).toEqual({
frameId: "child",
target: { tabId: 3, sessionId: "oopif-session" },
});
});
it("fails closed when a recording identity maps to multiple Documents", () => {
const index = new ObservationNodeIndex({
rootFrameId: "root",
frames: [
{
frameId: "left",
target: { tabId: 3, sessionId: "left-session" },
recordingDocumentId: "duplicate",
},
{
frameId: "right",
target: { tabId: 3, sessionId: "right-session" },
recordingDocumentId: "duplicate",
},
],
matchNodes: [],
refs: [],
});
expect(index.documentScope("duplicate")).toBeUndefined();
});
});
@@ -2,20 +2,27 @@ import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
const captureRecordingObservation = vi.hoisted(() => vi.fn());
const waitForDocumentSettled = vi.hoisted(() => vi.fn(async () => "quiet" as const));
vi.mock("../recording/observation-capture", async (importOriginal) => {
const actual = await importOriginal<typeof import("../recording/observation-capture")>();
return { ...actual, captureRecordingObservation };
});
vi.mock("../recording/document-settle", async (importOriginal) => {
const actual = await importOriginal<typeof import("../recording/document-settle")>();
return { ...actual, waitForDocumentSettled };
});
import { ObservationNodeIndex } from "../recording/observation-capture";
import { RecordingObservationRuntime } from "../recording/recording-runtime";
import type { RecordingDraftStep } from "../recording/types";
function observation() {
function observation(url = "https://example.com/") {
return {
rootFrameId: "root",
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }),
url: "https://example.com/",
url,
title: "Example",
vomText: '@vom 1\nRootWebArea "Example"',
truncated: false,
@@ -30,7 +37,10 @@ function runtime(): RecordingObservationRuntime {
}
describe("RecordingObservationRuntime", () => {
afterEach(() => captureRecordingObservation.mockReset());
afterEach(() => {
captureRecordingObservation.mockReset();
waitForDocumentSettled.mockClear();
});
it("shares one initial capture and includes it in flush", async () => {
let release!: (value: ReturnType<typeof observation>) => void;
@@ -130,4 +140,150 @@ describe("RecordingObservationRuntime", () => {
expect(dumped).not.toContain("backendNodeId");
expect(dumped).not.toContain("ObservationNodeIndex");
});
it("captures a tab transition without sharing observation cursors", async () => {
captureRecordingObservation
.mockResolvedValueOnce(observation("https://example.com/first"))
.mockResolvedValueOnce(observation("https://example.com/second"));
const recording = runtime();
await recording.captureInitial(4);
const transition = await recording.captureTabTransition(4, 5);
await recording.captureInitial(4);
await recording.captureInitial(5);
expect(transition).toEqual({
preStateId: "s1",
postStateId: "s2",
targetUrl: "https://example.com/second",
});
expect(captureRecordingObservation.mock.calls.map(([input]) => input.tabId)).toEqual([4, 5]);
});
it("binds an iframe draft to its marked CDP Document scope", async () => {
captureRecordingObservation.mockResolvedValueOnce({
...observation(),
index: new ObservationNodeIndex({
rootFrameId: "root",
frames: [
{
frameId: "child",
target: { tabId: 7, sessionId: "oopif-session" },
recordingDocumentId: "producer-1",
},
],
matchNodes: [
{
backendNodeId: 42,
frameId: "child",
tag: "button",
rect: { x: 410, y: 20, w: 100, h: 30 },
localRect: { x: 10, y: 20, w: 100, h: 30 },
},
],
refs: [{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
}),
});
const recording = runtime();
const drafts: RecordingDraftStep[] = [
{
op: "click" as const,
captureTarget: { tag: "button", role: "button", name: "Save" },
targetHint: {
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
},
];
await recording.captureInitial(7);
await recording.processDraft(7, drafts, 0, "producer-1");
recording.cancel();
const draft = drafts[0];
expect(draft?.op).toBe("click");
if (!draft || draft.op !== "click") throw new Error("expected click draft");
expect(draft.targetHint).toMatchObject({
frameId: "child",
geometrySpace: "local",
});
expect(draft.matchedTarget?.ref).toBe("e1");
});
it("refreshes a late iframe before matching its first action", async () => {
const iframeObservation = {
...observation(),
index: new ObservationNodeIndex({
rootFrameId: "root",
frames: [
{
frameId: "child",
target: { tabId: 7, sessionId: "oopif-session" },
recordingDocumentId: "producer-1",
},
],
matchNodes: [
{
backendNodeId: 42,
frameId: "child",
tag: "button",
rect: { x: 410, y: 20, w: 100, h: 30 },
localRect: { x: 10, y: 20, w: 100, h: 30 },
},
],
refs: [
{
ref: "e1",
backendNodeId: 42,
frameId: "child",
role: "button",
name: "表格视图",
line: 1,
},
],
}),
};
captureRecordingObservation
.mockResolvedValueOnce(observation())
.mockResolvedValueOnce(iframeObservation)
.mockResolvedValueOnce(iframeObservation);
const recording = runtime();
await recording.captureInitial(7);
await recording.refreshDocument(7, "producer-1");
const drafts: RecordingDraftStep[] = [
{
op: "click",
captureTarget: { tag: "button", role: "button", name: "表格视图" },
targetHint: {
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
},
];
await recording.processDraft(7, drafts, 0, "producer-1");
recording.cancel();
expect(captureRecordingObservation).toHaveBeenCalledTimes(3);
expect(waitForDocumentSettled).toHaveBeenCalledWith(
expect.anything(),
{ frameId: "child", target: { tabId: 7, sessionId: "oopif-session" } },
{ signal: expect.any(AbortSignal) },
);
const draft = drafts[0];
expect(draft?.op).toBe("click");
if (!draft || draft.op !== "click") throw new Error("expected click draft");
expect(draft.matchedTarget).toMatchObject({ ref: "e1", name: "表格视图" });
});
it("does not poison final flush when a frame readiness refresh fails", async () => {
captureRecordingObservation
.mockResolvedValueOnce(observation())
.mockRejectedValueOnce(new Error("child Document replaced"));
const recording = runtime();
await recording.captureInitial(7);
await expect(recording.refreshDocument(7, "producer-1")).rejects.toThrow(
"child Document replaced",
);
await expect(recording.flush()).resolves.toBeUndefined();
});
});
@@ -3,7 +3,7 @@ import { appendRecordedPayload, observeRecordedNavigation } from "../recording/s
describe("recording-step-buffer", () => {
it("stores semantic click without summary", () => {
const buffer = { steps: [], pendingNavigation: false };
const buffer = { steps: [], navigation: { pendingNavigation: false } };
appendRecordedPayload(buffer, {
op: "click",
target: { tag: "button", role: "button", name: "发布" },
@@ -15,11 +15,11 @@ describe("recording-step-buffer", () => {
captureTarget: { tag: "button", role: "button", name: "发布" },
},
]);
expect(buffer.pendingNavigation).toBe(true);
expect(buffer.navigation.pendingNavigation).toBe(true);
});
it("keeps the hovered element description as a capture fallback", () => {
const buffer = { steps: [], pendingNavigation: false };
const buffer = { steps: [], navigation: { pendingNavigation: false } };
appendRecordedPayload(
buffer,
{
@@ -59,9 +59,11 @@ describe("recording-step-buffer", () => {
captureTarget: { tag: "button", role: "button", name: "发布" },
},
],
currentUrl: "https://example.com/a",
pendingNavigation: true,
pendingNavigationDeadline: Date.now() + 5_000,
navigation: {
currentUrl: "https://example.com/a",
pendingNavigation: true,
pendingNavigationDeadline: Date.now() + 5_000,
},
};
observeRecordedNavigation(buffer, "https://example.com/b", true);
expect(buffer.steps).toEqual([
@@ -83,9 +85,11 @@ describe("recording-step-buffer", () => {
values: ["tech"],
},
],
currentUrl: "https://example.com/list",
pendingNavigation: true,
pendingNavigationDeadline: Date.now() + 5_000,
navigation: {
currentUrl: "https://example.com/list",
pendingNavigation: true,
pendingNavigationDeadline: Date.now() + 5_000,
},
};
observeRecordedNavigation(buffer, "https://example.com/list?cat=tech", true);
expect(buffer.steps).toEqual([
@@ -101,8 +105,7 @@ describe("recording-step-buffer", () => {
it("emits navigate for uncaused URL changes", () => {
const buffer = {
steps: [],
currentUrl: "https://example.com/a",
pendingNavigation: false,
navigation: { currentUrl: "https://example.com/a", pendingNavigation: false },
};
const result = observeRecordedNavigation(buffer, "https://example.com/b", false);
expect(result).toEqual({ kind: "appended", index: 0 });
@@ -115,8 +118,10 @@ describe("recording-step-buffer", () => {
it("asks the recorder to coalesce redirect hops instead of emitting each one", () => {
const buffer = {
steps: [],
currentUrl: "https://passport.example/login",
pendingNavigation: false,
navigation: {
currentUrl: "https://passport.example/login",
pendingNavigation: false,
},
};
const hop1 = observeRecordedNavigation(
buffer,
@@ -139,14 +144,16 @@ describe("recording-step-buffer", () => {
url: "https://app.example/dashboard",
});
expect(buffer.steps).toEqual([]);
expect(buffer.currentUrl).toBe("https://app.example/dashboard");
expect(buffer.navigation.currentUrl).toBe("https://app.example/dashboard");
});
it("does not reuse redirect metadata for a later content-observed URL change", () => {
const buffer = {
steps: [],
currentUrl: "https://example.com/redirect",
pendingNavigation: false,
navigation: {
currentUrl: "https://example.com/redirect",
pendingNavigation: false,
},
};
const result = observeRecordedNavigation(buffer, "https://example.com/spa");
@@ -0,0 +1,39 @@
import { describe, expect, it } from "vitest";
import { RecordingTabCoordinator } from "../recording/tab-coordinator";
describe("RecordingTabCoordinator", () => {
it("keeps navigation state isolated per tab", () => {
const tabs = new RecordingTabCoordinator(4, "https://example.com/first");
const first = tabs.navigation(4);
first.pendingNavigation = true;
const second = tabs.navigation(5, "https://example.com/second");
expect(second).toEqual({
currentUrl: "https://example.com/second",
pendingNavigation: false,
});
expect(tabs.navigation(4)).toBe(first);
expect(tabs.navigation(4).pendingNavigation).toBe(true);
});
it("invalidates an earlier activation when a newer tab becomes active", () => {
const tabs = new RecordingTabCoordinator(4);
const second = tabs.noteActivation(5);
const third = tabs.noteActivation(6);
expect(tabs.isLatest(second)).toBe(false);
expect(tabs.isLatest(third)).toBe(true);
tabs.commit(6);
expect(tabs.currentTabId).toBe(6);
});
it("commits the freshly observed URL when returning to an existing tab", () => {
const tabs = new RecordingTabCoordinator(4);
tabs.navigation(5, "https://example.com/old");
tabs.commit(5, "https://example.com/new");
expect(tabs.currentTabId).toBe(5);
expect(tabs.navigation(5).currentUrl).toBe("https://example.com/new");
});
});
@@ -69,6 +69,23 @@ describe("matchObservationTarget", () => {
expect(target.ref).toBe("e2");
});
it("matches iframe capture geometry in the child viewport coordinate space", () => {
const child = node(42, "child", 410);
child.localRect = { x: 10, y: 20, w: 100, h: 30 };
const target = matchObservationTarget({
observation: observation(
[child],
[{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
),
hint: {
frameId: "child",
geometrySpace: "local",
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
});
expect(target.ref).toBe("e1");
});
it("restricts a missing frame hint to the root frame", () => {
const target = matchObservationTarget({
observation: observation(
@@ -81,6 +98,21 @@ describe("matchObservationTarget", () => {
expect(target).toEqual({ name: "发布", unmatched: true });
});
it("fails closed when the source Document has no VOM frame mapping", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "root")],
[{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }],
),
hint: {
frameId: null,
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
fallback: { tag: "button", role: "button", name: "发布" },
});
expect(target).toEqual({ role: "button", name: "发布", unmatched: true });
});
it("returns unmatched for ambiguous geometry", () => {
const target = matchObservationTarget({
observation: observation(
@@ -27,6 +27,17 @@ describe("shouldRecordPress", () => {
});
describe("reduceTraceSteps", () => {
it("does not expose internal tab transitions in trace v2", () => {
const trace = reduceTraceSteps([
{
op: "switch_tab",
preStateId: "s1",
postStateId: "s2",
},
]);
expect(trace.steps).toEqual([]);
});
it("builds steps with pages dictionary and page id references", () => {
const drafts: RecordingDraftStep[] = [
{
@@ -160,7 +171,7 @@ describe("reduceTraceSteps", () => {
});
});
it("keeps hover steps before menu clicks", () => {
it("drops hover steps, which only exist in trace v3", () => {
const { steps } = reduceTraceSteps(
[
{
@@ -176,12 +187,7 @@ describe("reduceTraceSteps", () => {
],
"https://example.com/app",
);
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
expect(steps[0]).toMatchObject({
op: "hover",
page: "p1",
target: { name: "Account" },
});
expect(steps.map((s) => s.op)).toEqual(["click"]);
});
it("resolveTraceStartUrl prefers explicit start URL", () => {
@@ -33,6 +33,49 @@ describe("trace reducer v3", () => {
expect(JSON.stringify(reduced.steps)).not.toContain("private-account-id");
});
it("emits tab transitions only for callers that advertised support", () => {
const drafts: RecordingDraftStep[] = [
{
op: "switch_tab",
preStateId: "s1",
postStateId: "s2",
},
];
expect(reduceTraceStepsV3(drafts).steps).toEqual([]);
expect(reduceTraceStepsV3(drafts, { includeTabSwitches: true }).steps).toEqual([
{ op: "switch_tab", id: 1, state: "s1", result: { state: "s2" } },
]);
});
it("keeps hover steps before menu clicks", () => {
const drafts: RecordingDraftStep[] = [
{
op: "hover",
captureTarget: { tag: "span", role: "button", name: "Account" },
preStateId: "s1",
postStateId: "s1",
},
{
op: "click",
captureTarget: { tag: "a", role: "link", name: "Profile" },
preStateId: "s1",
postStateId: "s1",
},
];
expect(reduceTraceStepsV3(drafts).steps.map((s) => s.op)).toEqual(["hover", "click"]);
});
it("does not export a tab transition without both observation endpoints", () => {
const sourceOnly: RecordingDraftStep = { op: "switch_tab", preStateId: "s1" };
const targetOnly: RecordingDraftStep = { op: "switch_tab", postStateId: "s2" };
expect(
reduceTraceStepsV3([sourceOnly, targetOnly], { includeTabSwitches: true }).steps,
).toEqual([]);
});
it("collapses redirect hops while retaining draft-to-step identity", () => {
const drafts: RecordingDraftStep[] = [
{ op: "navigate", url: "https://example.com/start", preStateId: "s1", postStateId: "s2" },
@@ -0,0 +1,39 @@
import {
RECORD_DOCUMENT_ATTRIBUTE,
recordingDocumentMarkerValue,
} from "@/shared/recording-document-identity";
export interface RecordingDocumentMarker {
restore(): void;
ensure(): void;
}
export function waitForDocumentElement(): Promise<HTMLElement> {
if (document.documentElement) return Promise.resolve(document.documentElement);
return new Promise((resolve) => {
const observer = new MutationObserver(() => {
if (!document.documentElement) return;
observer.disconnect();
resolve(document.documentElement);
});
observer.observe(document, { childList: true });
});
}
export function markRecordingDocument(
producerId: string,
root: HTMLElement = document.documentElement,
): RecordingDocumentMarker {
const previous = root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE);
const markerValue = recordingDocumentMarkerValue(producerId);
const ensure = () => root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, markerValue);
ensure();
return {
ensure,
restore() {
if (root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE) !== markerValue) return;
if (previous === null) root.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
else root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, previous);
},
};
}
@@ -0,0 +1,65 @@
export const RECORD_FRAME_PORT = "bsk-record-frame";
export const RECORD_FRAME_QUERY = "bsk-record-frame-query";
export const RECORD_FRAME_START = "bsk-record-frame-start";
export interface RecordFrameQueryMessage {
type: typeof RECORD_FRAME_QUERY;
}
export interface RecordFrameQueryResponse {
active: boolean;
requestId?: string;
startedAtMs?: number;
}
export interface RecordFrameStartMessage {
type: typeof RECORD_FRAME_START;
requestId: string;
startedAtMs: number;
}
export type RecordFramePortMessage =
| {
type: "ready";
requestId: string;
producerId: string;
}
| {
type: "ready_ack";
requestId: string;
producerId: string;
}
| {
type: "stop";
requestId: string;
commandId: string;
}
| {
type: "cancel";
requestId: string;
}
| {
type: "stopped";
requestId: string;
commandId: string;
ok: boolean;
error?: string;
};
export function isRecordFrameQueryMessage(value: unknown): value is RecordFrameQueryMessage {
return (
typeof value === "object" &&
value !== null &&
(value as { type?: unknown }).type === RECORD_FRAME_QUERY
);
}
export function isRecordFrameStartMessage(value: unknown): value is RecordFrameStartMessage {
if (typeof value !== "object" || value === null) return false;
const message = value as Partial<RecordFrameStartMessage>;
return (
message.type === RECORD_FRAME_START &&
typeof message.requestId === "string" &&
typeof message.startedAtMs === "number"
);
}
@@ -0,0 +1,399 @@
import {
isRecordFrameQueryMessage,
RECORD_FRAME_PORT,
RECORD_FRAME_START,
type RecordFramePortMessage,
type RecordFrameQueryResponse,
type RecordFrameStartMessage,
} from "./frame-bridge";
export interface RecordingCaptureScope {
tabId: number;
documentId: string;
browserFrameId: number;
producerId: string;
}
interface ArmedRecording {
requestId: string;
startedAtMs: number;
tabIds: Set<number>;
agents: Map<string, FrameAgent>;
onDocumentReady?: (scope: RecordingCaptureScope) => void | Promise<void>;
finishing: boolean;
}
interface FrameAgent extends RecordingCaptureScope {
requestId: string;
port: chrome.runtime.Port;
stopWaiters: Map<string, (ok: boolean) => void>;
}
interface BrowserFrame {
frameId: number;
documentId?: string;
}
interface BrowserFrameNavigation extends BrowserFrame {
tabId: number;
lifecycle: "committed" | "completed";
}
const RECORD_FRAME_STOP_TIMEOUT_MS = 5_000;
export interface RecordFrameCoordinatorDeps {
getAllFrames(tabId: number): Promise<BrowserFrame[]>;
sendToDocument(
tabId: number,
message: RecordFrameStartMessage,
target: { documentId?: string; frameId?: number },
): Promise<unknown>;
subscribeFrameNavigation?(listener: (frame: BrowserFrameNavigation) => void): () => void;
}
function documentKey(tabId: number, documentId: string): string {
return `${tabId}:${documentId}`;
}
function senderAddress(
sender: chrome.runtime.MessageSender,
): { tabId: number; documentId: string; browserFrameId: number } | null {
const tabId = sender.tab?.id;
const documentId = sender.documentId;
const browserFrameId = sender.frameId;
if (
typeof tabId !== "number" ||
typeof documentId !== "string" ||
typeof browserFrameId !== "number"
) {
return null;
}
return { tabId, documentId, browserFrameId };
}
function defaultDeps(): RecordFrameCoordinatorDeps {
return {
async getAllFrames(tabId) {
return (await chrome.webNavigation.getAllFrames({ tabId })) ?? [];
},
sendToDocument(tabId, message, target) {
return chrome.tabs.sendMessage(tabId, message, target);
},
subscribeFrameNavigation(listener) {
const onCommitted = (details: chrome.webNavigation.WebNavigationTransitionCallbackDetails) =>
listener({ ...details, lifecycle: "committed" });
const onCompleted = (details: chrome.webNavigation.WebNavigationFramedCallbackDetails) =>
listener({ ...details, lifecycle: "completed" });
chrome.webNavigation.onCommitted.addListener(onCommitted);
chrome.webNavigation.onCompleted.addListener(onCompleted);
return () => {
chrome.webNavigation.onCommitted.removeListener(onCommitted);
chrome.webNavigation.onCompleted.removeListener(onCompleted);
};
},
};
}
export class RecordFrameCoordinator {
readonly #deps: RecordFrameCoordinatorDeps;
readonly #recordings = new Map<string, ArmedRecording>();
#attached = false;
#detachFrameNavigation: (() => void) | null = null;
constructor(deps: RecordFrameCoordinatorDeps = defaultDeps()) {
this.#deps = deps;
}
attach(): () => void {
if (this.#attached) return () => {};
this.#attached = true;
chrome.runtime.onConnect.addListener(this.#onConnect);
chrome.runtime.onMessage.addListener(this.#onMessage);
this.#detachFrameNavigation =
this.#deps.subscribeFrameNavigation?.(this.#onFrameNavigation) ?? null;
return () => {
if (!this.#attached) return;
this.#attached = false;
chrome.runtime.onConnect.removeListener(this.#onConnect);
chrome.runtime.onMessage.removeListener(this.#onMessage);
this.#detachFrameNavigation?.();
this.#detachFrameNavigation = null;
for (const recording of this.#recordings.values()) this.#cancelAgents(recording);
this.#recordings.clear();
};
}
begin(
requestId: string,
startedAtMs: number,
initialTabId: number,
onDocumentReady?: (scope: RecordingCaptureScope) => void | Promise<void>,
): void {
const previous = this.#recordings.get(requestId);
if (previous) this.#cancelAgents(previous);
this.#recordings.set(requestId, {
requestId,
startedAtMs,
// Register the initial tab before navigation begins. A newly-created
// child document can query at document_start before armTab's frame
// snapshot sees it, especially when Chromium promotes it to an OOPIF.
tabIds: new Set([initialTabId]),
agents: new Map(),
...(onDocumentReady ? { onDocumentReady } : {}),
finishing: false,
});
}
async armTab(requestId: string, tabId: number): Promise<boolean> {
const recording = this.#recordings.get(requestId);
if (!recording || recording.finishing) return false;
recording.tabIds.add(tabId);
let frames: BrowserFrame[];
try {
frames = await this.#deps.getAllFrames(tabId);
} catch {
frames = [{ frameId: 0 }];
}
if (!frames.some((frame) => frame.frameId === 0)) frames.unshift({ frameId: 0 });
const results = await Promise.all(
frames.map(async (frame) => {
const started = await this.#startFrame(recording, tabId, frame);
return { frameId: frame.frameId, started };
}),
);
return results.some((result) => result.frameId === 0 && result.started);
}
sourceFor(
requestId: string,
producerId: string,
sender: chrome.runtime.MessageSender,
): RecordingCaptureScope | null {
const address = senderAddress(sender);
const recording = this.#recordings.get(requestId);
if (!address || !recording) return null;
const agent = recording.agents.get(documentKey(address.tabId, address.documentId));
if (!agent || agent.producerId !== producerId) return null;
return {
tabId: agent.tabId,
documentId: agent.documentId,
browserFrameId: agent.browserFrameId,
producerId: agent.producerId,
};
}
async stop(requestId: string): Promise<boolean> {
const recording = this.#recordings.get(requestId);
if (!recording) return true;
recording.finishing = true;
const commandId = crypto.randomUUID();
const results = await Promise.all(
[...recording.agents.values()].map((agent) => this.#stopAgent(agent, requestId, commandId)),
);
const succeeded = results.every(Boolean);
if (succeeded) this.#recordings.delete(requestId);
else recording.finishing = false;
return succeeded;
}
cancel(requestId: string): void {
const recording = this.#recordings.get(requestId);
if (!recording) return;
this.#cancelAgents(recording);
this.#recordings.delete(requestId);
}
readonly #onMessage = (
message: unknown,
sender: chrome.runtime.MessageSender,
sendResponse: (response: RecordFrameQueryResponse) => void,
): boolean => {
if (!isRecordFrameQueryMessage(message)) return false;
const tabId = sender.tab?.id;
const recording =
typeof tabId === "number"
? [...this.#recordings.values()].find(
(candidate) => !candidate.finishing && candidate.tabIds.has(tabId),
)
: undefined;
sendResponse(
recording
? {
active: true,
requestId: recording.requestId,
startedAtMs: recording.startedAtMs,
}
: { active: false },
);
return false;
};
readonly #onFrameNavigation = (frame: BrowserFrameNavigation): void => {
if (frame.frameId === 0) return;
for (const recording of this.#recordings.values()) {
if (recording.finishing || !recording.tabIds.has(frame.tabId)) continue;
// onCommitted can race document_start, while onCompleted runs after the
// content script is available. Starting twice is safe and closes both
// the early-query and late-SPA-frame gaps.
void (async () => {
const started = await this.#startFrame(recording, frame.tabId, frame);
if (!started || frame.lifecycle !== "completed" || recording.finishing) return;
const agent = frame.documentId
? recording.agents.get(documentKey(frame.tabId, frame.documentId))
: [...recording.agents.values()].find(
(candidate) =>
candidate.tabId === frame.tabId && candidate.browserFrameId === frame.frameId,
);
if (!agent) return;
await recording.onDocumentReady?.({
tabId: agent.tabId,
documentId: agent.documentId,
browserFrameId: agent.browserFrameId,
producerId: agent.producerId,
});
})();
}
};
readonly #onConnect = (port: chrome.runtime.Port): void => {
if (port.name !== RECORD_FRAME_PORT) return;
const address = senderAddress(port.sender ?? {});
if (!address) {
port.disconnect();
return;
}
let registered: FrameAgent | null = null;
port.onMessage.addListener((raw: unknown) => {
const message = raw as Partial<RecordFramePortMessage>;
if (message.type === "ready") {
if (typeof message.requestId !== "string" || typeof message.producerId !== "string") {
port.disconnect();
return;
}
const requestId = message.requestId;
const producerId = message.producerId;
if (registered) {
if (registered.requestId === requestId && registered.producerId === producerId) {
port.postMessage({
type: "ready_ack",
requestId,
producerId,
} satisfies RecordFramePortMessage);
} else {
port.disconnect();
}
return;
}
const recording = this.#recordings.get(requestId);
if (
!recording ||
recording.finishing ||
!recording.tabIds.has(address.tabId) ||
producerId.length === 0
) {
port.disconnect();
return;
}
const agent: FrameAgent = {
...address,
requestId,
producerId,
port,
stopWaiters: new Map(),
};
registered = agent;
const key = documentKey(address.tabId, address.documentId);
const previous = recording.agents.get(key);
if (previous && previous !== agent) previous.port.disconnect();
recording.agents.set(key, agent);
port.postMessage({
type: "ready_ack",
requestId,
producerId,
} satisfies RecordFramePortMessage);
return;
}
if (
message.type === "stopped" &&
registered &&
message.requestId === registered.requestId &&
typeof message.commandId === "string"
) {
registered.stopWaiters.get(message.commandId)?.(message.ok === true);
registered.stopWaiters.delete(message.commandId);
}
});
port.onDisconnect.addListener(() => {
if (!registered) return;
for (const resolve of registered.stopWaiters.values()) resolve(true);
const recording = this.#recordings.get(registered.requestId);
const key = documentKey(registered.tabId, registered.documentId);
if (recording?.agents.get(key) === registered) recording.agents.delete(key);
});
};
#stopAgent(agent: FrameAgent, requestId: string, commandId: string): Promise<boolean> {
return new Promise((resolve) => {
const timer = setTimeout(() => {
agent.stopWaiters.delete(commandId);
resolve(false);
}, RECORD_FRAME_STOP_TIMEOUT_MS);
agent.stopWaiters.set(commandId, (ok) => {
clearTimeout(timer);
resolve(ok);
});
try {
agent.port.postMessage({
type: "stop",
requestId,
commandId,
} satisfies RecordFramePortMessage);
} catch {
clearTimeout(timer);
agent.stopWaiters.delete(commandId);
resolve(true);
}
});
}
async #startFrame(
recording: ArmedRecording,
tabId: number,
frame: BrowserFrame,
): Promise<boolean> {
const message: RecordFrameStartMessage = {
type: RECORD_FRAME_START,
requestId: recording.requestId,
startedAtMs: recording.startedAtMs,
};
try {
const response = await this.#deps.sendToDocument(tabId, message, {
...(frame.documentId ? { documentId: frame.documentId } : { frameId: frame.frameId }),
});
return isStarted(response);
} catch {
return false;
}
}
#cancelAgents(recording: ArmedRecording): void {
for (const agent of recording.agents.values()) {
try {
agent.port.postMessage({
type: "cancel",
requestId: recording.requestId,
} satisfies RecordFramePortMessage);
} catch {
// A destroyed Document has no remaining capture state to cancel.
}
}
}
}
function isStarted(value: unknown): boolean {
return typeof value === "object" && value !== null && (value as { ok?: unknown }).ok === true;
}
export const recordFrameCoordinator = new RecordFrameCoordinator();
@@ -1,4 +1,5 @@
import type { RenderedRef } from "@browser-skill/vom";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import {
type CaptureVomMatchNode,
type CaptureVomObservationResult,
@@ -28,6 +29,11 @@ export interface RegisteredObservation {
url: string;
}
export interface RecordingDocumentScope {
frameId: string;
target: CdpTarget;
}
function nodeKey(frameId: string, backendNodeId: number): string {
return `${frameId}:${backendNodeId}`;
}
@@ -40,8 +46,12 @@ export class ObservationNodeIndex {
readonly #nodesByFrameTag = new Map<string, IndexedObservationNode[]>();
readonly #refById = new Map<string, RenderedRef>();
readonly #refsByFrame = new Map<string, RenderedRef[]>();
readonly #scopeByProducer = new Map<string, RecordingDocumentScope | null>();
constructor(input: Pick<CaptureVomObservationResult, "rootFrameId" | "matchNodes" | "refs">) {
constructor(
input: Pick<CaptureVomObservationResult, "rootFrameId" | "matchNodes" | "refs"> &
Partial<Pick<CaptureVomObservationResult, "frames">>,
) {
const refByNode = new Map<string, RenderedRef>();
for (const ref of input.refs) {
const frameId = ref.frameId ?? input.rootFrameId;
@@ -51,6 +61,16 @@ export class ObservationNodeIndex {
frameRefs.push(ref);
this.#refsByFrame.set(frameId, frameRefs);
}
for (const frame of input.frames ?? []) {
const documentId = frame.recordingDocumentId;
if (!documentId) continue;
this.#scopeByProducer.set(
documentId,
this.#scopeByProducer.has(documentId)
? null
: { frameId: frame.frameId, target: frame.target },
);
}
for (const geometry of input.matchNodes) {
const { frameId } = geometry;
const entry = {
@@ -76,6 +96,10 @@ export class ObservationNodeIndex {
refs(frameId: string): readonly RenderedRef[] {
return this.#refsByFrame.get(frameId) ?? [];
}
documentScope(producerId: string): RecordingDocumentScope | undefined {
return this.#scopeByProducer.get(producerId) ?? undefined;
}
}
async function readTabMeta(
@@ -28,7 +28,13 @@ function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
}
function isTargeted(draft: RecordingDraftStep): draft is TargetedRecordingDraft {
return draft.op !== "navigate" && draft.op !== "scroll";
return (
draft.op === "click" ||
draft.op === "hover" ||
draft.op === "fill" ||
draft.op === "press" ||
draft.op === "select"
);
}
export class RecordingObservationSession {
@@ -1,6 +1,7 @@
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import type { StopReason, TraceV3 } from "@/transport/types";
import type { DocumentSettleScope } from "./document-settle";
import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle";
import type { RegisteredObservation } from "./observation-capture";
import { RecordingObservationSession } from "./observation-session";
import { inferMissingPostStates, SettleController } from "./settle-controller";
import { RecordingStateRegistry } from "./state-registry";
@@ -10,11 +11,13 @@ import type { RecordingDraftStep, StepAnnotation } from "./types";
interface TabRecordingContext {
session: RecordingObservationSession;
settle: SettleController;
initialCapture: PendingInitialCapture | null;
pendingCapture: PendingCapture | null;
frameRefresh: Promise<void>;
frameRefreshAbort: AbortController | null;
}
interface PendingInitialCapture {
promise: Promise<void>;
interface PendingCapture {
promise: Promise<RegisteredObservation>;
abort: AbortController;
}
@@ -56,37 +59,113 @@ export class RecordingObservationRuntime {
tabsApi: this.#tabsApi,
tabId,
}),
initialCapture: null,
pendingCapture: null,
frameRefresh: Promise.resolve(),
frameRefreshAbort: null,
};
this.#contexts.set(tabId, context);
return context;
}
async captureInitial(tabId: number): Promise<void> {
async #capture(tabId: number): Promise<RegisteredObservation> {
const context = this.#context(tabId);
if (context.session.cursor.lastSettled) return;
if (context.initialCapture) return context.initialCapture.promise;
if (context.pendingCapture) return context.pendingCapture.promise;
const abort = new AbortController();
let pending!: PendingInitialCapture;
let pending!: PendingCapture;
const promise = context.session.capture(this.#cdp, this.#tabsApi, tabId, abort.signal);
pending = {
abort,
promise: promise
.then(() => {})
.finally(() => {
if (context.initialCapture === pending) context.initialCapture = null;
}),
promise: promise.finally(() => {
if (context.pendingCapture === pending) context.pendingCapture = null;
}),
};
context.initialCapture = pending;
context.pendingCapture = pending;
return pending.promise;
}
async captureInitial(tabId: number): Promise<void> {
const context = this.#context(tabId);
if (context.session.cursor.lastSettled) return;
await this.#capture(tabId);
}
/** Refresh the safe observation after a late child Document becomes usable. */
async refreshDocument(tabId: number, producerId: string): Promise<void> {
const context = this.#context(tabId);
context.frameRefreshAbort?.abort();
const abort = new AbortController();
context.frameRefreshAbort = abort;
const previous = context.frameRefresh;
const refresh = (async () => {
await previous;
if (abort.signal.aborted) return;
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
if (abort.signal.aborted) return;
// The first capture discovers the safe producer -> CDP Document scope.
await this.#capture(tabId);
if (abort.signal.aborted) return;
const scope = context.session.cursor.lastSettled?.index.documentScope(producerId);
if (scope) {
await waitForDocumentSettled(this.#cdp, scope, { signal: abort.signal });
}
if (abort.signal.aborted) return;
// Capture again after the child SPA is quiet so the next action can be
// matched against refs that actually exist in its pre-action state.
await this.#capture(tabId);
})();
const tracked = refresh.catch(() => {});
context.frameRefresh = tracked;
try {
await refresh;
} finally {
if (context.frameRefresh === tracked) context.frameRefreshAbort = null;
}
}
async captureTabTransition(
fromTabId: number,
toTabId: number,
): Promise<{ preStateId?: string; postStateId?: string; targetUrl?: string }> {
const from = this.#context(fromTabId);
await this.#flushContext(from);
if (!from.session.cursor.lastSettled) {
try {
await this.captureInitial(fromTabId);
} catch {
// A target-tab observation can still preserve the forward transition.
}
}
const to = this.#context(toTabId);
let target = to.session.cursor.lastSettled;
try {
target = await this.#capture(toTabId);
} catch {
// Reuse the last settled target state when a refresh races navigation.
}
const source = from.session.cursor.lastSettled;
return {
...(source ? { preStateId: source.stateId } : {}),
...(target ? { postStateId: target.stateId, targetUrl: target.url } : {}),
};
}
bindTabTransition(
draft: Extract<RecordingDraftStep, { op: "switch_tab" }>,
draftId: number,
): void {
if (draft.preStateId) this.#registry.markStep(draft.preStateId, draftId);
}
async processDraft(
tabId: number,
drafts: RecordingDraftStep[],
draftIndex: number,
scope?: DocumentSettleScope,
producerId?: string,
): Promise<void> {
const draft = drafts[draftIndex];
if (!draft) return;
@@ -98,6 +177,23 @@ export class RecordingObservationRuntime {
// Post-action settle can still provide a usable state.
}
}
const scope = producerId
? context.session.cursor.lastSettled?.index.documentScope(producerId)
: undefined;
if (
producerId &&
(draft.op === "click" ||
draft.op === "hover" ||
draft.op === "fill" ||
draft.op === "press" ||
draft.op === "select")
) {
draft.targetHint = {
...(draft.targetHint ?? {}),
frameId: scope?.frameId ?? null,
geometrySpace: "local",
};
}
context.session.bindDraft(draft, draftIndex + 1, context.settle.hasPending);
context.settle.schedule(drafts, draftIndex, scope);
}
@@ -119,23 +215,24 @@ export class RecordingObservationRuntime {
this.#context(tabId).settle.scheduleRedirect(drafts, url);
}
async flushRedirects(): Promise<void> {
for (const context of this.#contexts.values()) await context.settle.flushRedirects();
async #flushContext(context: TabRecordingContext): Promise<void> {
await context.frameRefresh;
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
await context.settle.flushRedirects();
await context.settle.flush();
}
async flushRedirects(tabId: number): Promise<void> {
await this.#contexts.get(tabId)?.settle.flushRedirects();
}
async flush(): Promise<void> {
await Promise.allSettled(
[...this.#contexts.values()].flatMap((context) =>
context.initialCapture ? [context.initialCapture.promise] : [],
),
);
await this.flushRedirects();
for (const context of this.#contexts.values()) await context.settle.flush();
for (const context of this.#contexts.values()) await this.#flushContext(context);
}
async settleTrailing(tabId: number, drafts: RecordingDraftStep[]): Promise<void> {
const context = this.#context(tabId);
if (context.initialCapture) await Promise.allSettled([context.initialCapture.promise]);
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
if (!context.session.cursor.lastSettled) {
try {
await this.captureInitial(tabId);
@@ -149,7 +246,8 @@ export class RecordingObservationRuntime {
cancel(): void {
for (const context of this.#contexts.values()) {
context.initialCapture?.abort.abort();
context.frameRefreshAbort?.abort();
context.pendingCapture?.abort.abort();
context.settle.cancel();
}
}
@@ -161,6 +259,7 @@ export class RecordingObservationRuntime {
startUrl?: string;
stoppedBy: StopReason;
bskVersion: string;
includeTabSwitches?: boolean;
}): TraceV3 {
return buildTraceV3({
registry: this.#registry,
@@ -172,6 +271,7 @@ export class RecordingObservationRuntime {
stoppedBy: input.stoppedBy,
bskVersion: input.bskVersion,
redactValues: this.#redactValues,
includeTabSwitches: input.includeTabSwitches,
});
}
}
+17 -12
View File
@@ -4,6 +4,10 @@ import type { RecordingDraftStep, TargetMatchHint } from "./types";
export interface RecordingStepBuffer {
steps: RecordingDraftStep[];
navigation: RecordingNavigationCursor;
}
export interface RecordingNavigationCursor {
currentUrl?: string;
pendingNavigation: boolean;
pendingNavigationDeadline?: number;
@@ -87,22 +91,23 @@ export function observeRecordedNavigation(
transitionType?: string,
transitionQualifiers?: string[],
): NavigationObserveResult {
if (!url || url === buffer.currentUrl) return { kind: "noop" };
buffer.currentUrl = url;
const navigation = buffer.navigation;
if (!url || url === navigation.currentUrl) return { kind: "noop" };
navigation.currentUrl = url;
const pendingIsCurrent =
buffer.pendingNavigation &&
(buffer.pendingNavigationDeadline === undefined ||
buffer.pendingNavigationDeadline >= Date.now());
navigation.pendingNavigation &&
(navigation.pendingNavigationDeadline === undefined ||
navigation.pendingNavigationDeadline >= Date.now());
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
buffer.pendingNavigation = false;
buffer.pendingNavigationDeadline = undefined;
navigation.pendingNavigation = false;
navigation.pendingNavigationDeadline = undefined;
const annotatedIndex = annotateLastStepNavigation(buffer, url);
if (annotatedIndex >= 0) return { kind: "annotated", index: annotatedIndex };
} else {
buffer.pendingNavigation = false;
buffer.pendingNavigationDeadline = undefined;
navigation.pendingNavigation = false;
navigation.pendingNavigationDeadline = undefined;
}
if (hasRedirectQualifier(transitionQualifiers)) {
@@ -135,14 +140,14 @@ export function appendRecordedPayload(
return result.kind === "appended" ? result.index : null;
}
const step = toDraftStep(
{ ...payload, page_url: payload.page_url ?? buffer.currentUrl },
{ ...payload, page_url: payload.page_url ?? buffer.navigation.currentUrl },
targetHint,
);
if (!step) return null;
buffer.steps.push(step);
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
buffer.pendingNavigation = payload.expects_navigation === true;
buffer.pendingNavigationDeadline = buffer.pendingNavigation
buffer.navigation.pendingNavigation = payload.expects_navigation === true;
buffer.navigation.pendingNavigationDeadline = buffer.navigation.pendingNavigation
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
: undefined;
}
@@ -0,0 +1,61 @@
import type { RecordingNavigationCursor } from "./step-buffer";
export interface TabActivation {
tabId: number;
revision: number;
}
export class RecordingTabCoordinator {
readonly #navigationByTab = new Map<number, RecordingNavigationCursor>();
#activeTabId: number;
#currentTabId: number;
#activationRevision = 0;
constructor(initialTabId: number, initialUrl?: string) {
this.#activeTabId = initialTabId;
this.#currentTabId = initialTabId;
this.#navigationByTab.set(initialTabId, {
currentUrl: initialUrl,
pendingNavigation: false,
});
}
get activeTabId(): number {
return this.#activeTabId;
}
get currentTabId(): number {
return this.#currentTabId;
}
noteActivation(tabId: number): TabActivation {
this.#activeTabId = tabId;
this.#activationRevision += 1;
return { tabId, revision: this.#activationRevision };
}
isLatest(activation: TabActivation): boolean {
return (
activation.tabId === this.#activeTabId && activation.revision === this.#activationRevision
);
}
commit(tabId: number, currentUrl?: string): void {
if (currentUrl !== undefined) this.navigation(tabId).currentUrl = currentUrl;
this.#currentTabId = tabId;
}
navigation(tabId: number, fallbackUrl?: string): RecordingNavigationCursor {
const existing = this.#navigationByTab.get(tabId);
if (existing) {
if (!existing.currentUrl && fallbackUrl) existing.currentUrl = fallbackUrl;
return existing;
}
const created: RecordingNavigationCursor = {
currentUrl: fallbackUrl,
pendingNavigation: false,
};
this.#navigationByTab.set(tabId, created);
return created;
}
}
@@ -14,8 +14,13 @@ function rectMatches(a: TargetGeometry["rect"], b: TargetGeometry["rect"]): bool
return close(a.x, b.x) && close(a.y, b.y) && close(a.w, b.w) && close(a.h, b.h);
}
function candidateMatches(candidate: IndexedObservationNode, geometry: TargetGeometry): boolean {
return candidate.geometry.rect !== null && rectMatches(geometry.rect, candidate.geometry.rect);
function candidateMatches(
candidate: IndexedObservationNode,
geometry: TargetGeometry,
geometrySpace: "top" | "local",
): boolean {
const rect = geometrySpace === "local" ? candidate.geometry.localRect : candidate.geometry.rect;
return rect != null && rectMatches(geometry.rect, rect);
}
export function unmatchedTarget(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 {
@@ -53,12 +58,17 @@ export function matchObservationTarget(input: {
hint?: TargetMatchHint;
fallback?: CaptureTargetDescriptor;
}): TargetDescriptorV3 {
if (input.hint?.frameId === null) return unmatchedTarget(input.fallback);
const frameId = input.hint?.frameId ?? input.observation.rootFrameId;
const geometry = input.hint?.geometry;
if (geometry) {
const matches = input.observation.index
.candidates(frameId, geometry.tag)
.filter((candidate) => candidate.ref && candidateMatches(candidate, geometry));
.filter(
(candidate) =>
candidate.ref &&
candidateMatches(candidate, geometry, input.hint?.geometrySpace ?? "top"),
);
if (matches.length === 1) return descriptor(matches[0]!.ref!);
const semanticMatches = matches.filter(
(candidate) => candidate.ref && matchesSemantics(candidate.ref, input.fallback),
@@ -40,8 +40,12 @@ export function buildTraceV3(input: {
stoppedBy: StopReason;
bskVersion: string;
redactValues?: boolean;
includeTabSwitches?: boolean;
}): TraceV3 {
const reduced = reduceTraceStepsV3(input.drafts, { redactValues: input.redactValues });
const reduced = reduceTraceStepsV3(input.drafts, {
includeTabSwitches: input.includeTabSwitches,
redactValues: input.redactValues,
});
const entries = publishedEntries(input.registry, reduced.steps);
const publishedId = new Map(entries.map((entry, index) => [entry.id, `s${index + 1}`]));
const annotationsByState = new Map<string, StepAnnotation[]>();
@@ -119,14 +119,6 @@ function toV2Step(
},
effectForNavigation(step.navigatedTo, urlToId),
);
case "hover":
if (!step.captureTarget) return null;
return {
op: "hover",
id,
page,
target: step.captureTarget,
};
case "fill":
if (!step.captureTarget) return null;
return {
@@ -161,7 +153,11 @@ function toV2Step(
},
effectForNavigation(step.navigatedTo, urlToId),
);
// `hover` only exists in Trace v3. Peers that negotiate v2 predate the
// step variant and fail to decode the whole result if we emit it.
case "hover":
case "scroll":
case "switch_tab":
return null;
}
}
@@ -58,8 +58,26 @@ function selection(values: string[], labels?: string[]): Array<{ value: string;
}));
}
function reduceDraft(draft: RecordingDraftStep, id: number, redactValues: boolean): StepV3 | null {
interface ReduceDraftOptions {
includeTabSwitches: boolean;
redactValues: boolean;
}
function reduceDraft(
draft: RecordingDraftStep,
id: number,
options: ReduceDraftOptions,
): StepV3 | null {
if (!shouldIncludeDraft(draft)) return null;
if (draft.op === "switch_tab") {
if (!options.includeTabSwitches || !draft.preStateId || !draft.postStateId) return null;
return {
op: "switch_tab",
id,
state: draft.preStateId,
result: { state: draft.postStateId },
};
}
const state = draft.preStateId ?? draft.postStateId;
const resultState = draft.postStateId ?? draft.preStateId;
if (!state || !resultState) return null;
@@ -81,7 +99,7 @@ function reduceDraft(draft: RecordingDraftStep, id: number, redactValues: boolea
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
};
case "fill":
const fillIsRedacted = redactValues || draft.redacted === true;
const fillIsRedacted = options.redactValues || draft.redacted === true;
return {
op: "fill",
...common,
@@ -105,7 +123,7 @@ function reduceDraft(draft: RecordingDraftStep, id: number, redactValues: boolea
op: "select",
...common,
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
...(!redactValues ? { selection: selection(draft.values, draft.labels) } : {}),
...(!options.redactValues ? { selection: selection(draft.values, draft.labels) } : {}),
};
case "scroll":
return { op: "scroll", ...common };
@@ -119,12 +137,15 @@ export interface ReducedTraceV3 {
export function reduceTraceStepsV3(
steps: RecordingDraftStep[],
options: { redactValues?: boolean } = {},
options: { includeTabSwitches?: boolean; redactValues?: boolean } = {},
): ReducedTraceV3 {
const output: StepV3[] = [];
const stepIdByDraftId = new Map<number, number>();
for (const { draft, draftIds } of collapseRedirects(steps)) {
const step = reduceDraft(draft, output.length + 1, options.redactValues ?? false);
const step = reduceDraft(draft, output.length + 1, {
includeTabSwitches: options.includeTabSwitches === true,
redactValues: options.redactValues === true,
});
if (!step) continue;
output.push(step);
for (const draftId of draftIds) stepIdByDraftId.set(draftId, step.id);
+4 -2
View File
@@ -15,8 +15,9 @@ export interface TargetGeometry {
export interface TargetMatchHint {
geometry?: TargetGeometry;
/** Missing means the current top frame, never an unrestricted frame search. */
frameId?: string;
/** Missing means top frame; null means the source Document could not be resolved. */
frameId?: string | null;
geometrySpace?: "top" | "local";
}
export interface StepAnnotation {
@@ -69,6 +70,7 @@ export type RecordingDraftStep =
DraftTarget &
DraftNavigationEffect)
| ({ op: "scroll" } & DraftStateLink)
| ({ op: "switch_tab" } & DraftStateLink)
| ({
op: "navigate";
url: string;
@@ -0,0 +1,22 @@
export const RECORD_DOCUMENT_ATTRIBUTE = "data-bsk-record-document";
const RECORDING_DOCUMENT_ID_PATTERN =
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
const RECORDING_DOCUMENT_MARKER_PREFIX = "bsk:";
export function recordingDocumentMarkerValue(identity: string): string {
if (!RECORDING_DOCUMENT_ID_PATTERN.test(identity)) {
throw new TypeError("recording Document identity must be a random UUID");
}
return `${RECORDING_DOCUMENT_MARKER_PREFIX}${identity}`;
}
/** Read BrowserSkill's opaque per-Document recording identity from captured attributes. */
export function readRecordingDocumentIdentity(
attrs: Readonly<Record<string, string>>,
): string | undefined {
const marker = attrs[RECORD_DOCUMENT_ATTRIBUTE];
if (!marker?.startsWith(RECORDING_DOCUMENT_MARKER_PREFIX)) return undefined;
const identity = marker.slice(RECORDING_DOCUMENT_MARKER_PREFIX.length);
return identity && RECORDING_DOCUMENT_ID_PATTERN.test(identity) ? identity : undefined;
}
@@ -8,6 +8,7 @@ import type {
RequestFrame,
} from "@/transport/types";
import { ToolDispatcher } from "../dispatcher";
import { resetBrowserObservationForTests, setBrowserObservationAttachForTests } from "../record";
type TestDispatcherCdp = NonNullable<ConstructorParameters<typeof ToolDispatcher>[0]["cdp"]>;
@@ -44,9 +45,72 @@ function makeRequest(method: string, params: unknown): RequestFrame {
describe("ToolDispatcher", () => {
afterEach(() => {
resetBrowserObservationForTests();
vi.unstubAllGlobals();
});
it("uses the configured recording runtime for start and stop", async () => {
const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
},
});
await sessions.start("aa11");
const tab = {
id: 7,
windowId: 4242,
active: true,
status: "complete",
url: "https://example.com/start",
} as chrome.tabs.Tab;
const frameCoordinator = {
begin: vi.fn(),
armTab: vi.fn(async () => true),
sourceFor: vi.fn(() => null),
stop: vi.fn(async () => true),
cancel: vi.fn(),
};
const sendToTab = vi.fn(async () => ({ ok: true }));
setBrowserObservationAttachForTests(
() => () => {},
() => () => {},
);
const dispatcher = new ToolDispatcher({
transport,
sessions,
recording: {
tabsApi: {
get: vi.fn(async () => tab),
query: vi.fn(async () => [tab]),
},
frameCoordinator,
sendToTab,
},
});
dispatcher.start();
deliver(
makeRequest("tool.record_start", {
session_id: "aa11",
url: "https://example.com/start",
}),
);
await vi.waitFor(() => expect(sent).toHaveLength(1));
expect(sent[0]).toMatchObject({ result: { tab_id: 7, recording: true } });
expect(frameCoordinator.begin).toHaveBeenCalledOnce();
expect(frameCoordinator.armTab).toHaveBeenCalledWith(expect.any(String), 7);
deliver({ id: "r-2", method: "tool.record_stop", params: { session_id: "aa11" } });
await vi.waitFor(() => expect(sent).toHaveLength(2));
expect(frameCoordinator.stop).toHaveBeenCalledOnce();
expect(sent[1]).toMatchObject({ id: "r-2", result: { trace: { steps: [] } } });
});
it("routes tool.session_start to the SessionManager and replies with the window id", async () => {
const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({
@@ -15,7 +15,12 @@ import {
const AGENT_WINDOW_ID = 100;
const TAB_ID = 4;
const START_URL = "https://example.com/";
const RECORD_START_V3 = { session_id: "abcd", url: START_URL, trace_version: 3 as const };
const RECORD_START_V3 = {
session_id: "abcd",
url: START_URL,
trace_version: 3 as const,
supports_tab_switch_steps: true,
};
const stepSequenceByProducer = new Map<string, number>();
function asTraceV3(trace: RecordedTrace): TraceV3 {
@@ -48,6 +53,7 @@ function chromeEvent<T extends (...args: never[]) => unknown>() {
function installChrome() {
const runtimeOnMessage = chromeEvent<RuntimeListener>();
const tabsOnActivated = chromeEvent<(activeInfo: chrome.tabs.TabActiveInfo) => unknown>();
const webNavigationOnCompleted =
chromeEvent<(details: chrome.webNavigation.WebNavigationFramedCallbackDetails) => unknown>();
const webNavigationOnCommitted =
@@ -57,7 +63,7 @@ function installChrome() {
vi.stubGlobal("chrome", {
runtime: { onMessage: runtimeOnMessage },
tabs: {
onActivated: chromeEvent(),
onActivated: tabsOnActivated,
onCreated: chromeEvent(),
onUpdated: chromeEvent(),
},
@@ -66,7 +72,7 @@ function installChrome() {
onCommitted: webNavigationOnCommitted,
},
});
return { runtimeOnMessage, webNavigationOnCompleted, webNavigationOnCommitted };
return { runtimeOnMessage, tabsOnActivated, webNavigationOnCompleted, webNavigationOnCommitted };
}
function fakeManager() {
@@ -118,7 +124,10 @@ type FakeCdp = CdpRunner & {
const LONG_IDLE_MS = 10_000;
/** AX-only page: DOMSnapshot calls throw so capture falls back to the AX tree. */
function makeFakeCdp(trees?: unknown[], options?: { failCaptures?: boolean }): FakeCdp {
function makeFakeCdp(
trees?: unknown[],
options?: { failCaptures?: boolean; treesByTab?: Record<number, unknown> },
): FakeCdp {
type EventListener = (source: chrome.debugger.Debuggee, method: string, params: unknown) => void;
const events: EventListener[] = [];
const script = [...(trees ?? [])];
@@ -153,6 +162,8 @@ function makeFakeCdp(trees?: unknown[], options?: { failCaptures?: boolean }): F
"Accessibility.enable": () => ({}),
"Accessibility.getFullAXTree": (_params, _tabId) => {
if (failing) throw new Error("Execution context was destroyed");
const tabTree = options?.treesByTab?.[_tabId];
if (tabTree) return tabTree;
if (script.length === 0) return axTree("Example Domain", ["Submit"]);
return script.length === 1 ? script[0] : script.shift();
},
@@ -202,6 +213,31 @@ function makeTabsApi() {
};
}
function makeMultiTabsApi(tabs: chrome.tabs.Tab[]) {
const byId = new Map(tabs.flatMap((tab) => (typeof tab.id === "number" ? [[tab.id, tab]] : [])));
return {
get: async (tabId: number) => {
const tab = byId.get(tabId);
if (!tab) throw new Error(`tab ${tabId} closed`);
return tab;
},
query: async (query: chrome.tabs.QueryInfo) =>
[...byId.values()].filter(
(tab) =>
(query.windowId === undefined || tab.windowId === query.windowId) &&
(query.active === undefined || tab.active === query.active),
),
activate(tabId: number) {
for (const tab of byId.values()) tab.active = tab.id === tabId;
},
goTo(tabId: number, url: string, title: string) {
const tab = byId.get(tabId);
if (!tab) throw new Error(`tab ${tabId} closed`);
Object.assign(tab, { url, title });
},
};
}
/** Outlast a settle on a quiet page plus the per-tab observation cooldown. */
function settleWait(): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, 800));
@@ -212,16 +248,19 @@ function runtimeOnMessageEmit(
requestId: string,
step: unknown,
tabId = TAB_ID,
): void {
active = true,
): ReturnType<typeof vi.fn> {
const producerId = `test-producer-${tabId}`;
const producerKey = `${requestId}:${producerId}`;
const sequence = (stepSequenceByProducer.get(producerKey) ?? 0) + 1;
stepSequenceByProducer.set(producerKey, sequence);
const sendResponse = vi.fn();
chromeApi.runtimeOnMessage.emit(
{ type: RECORD_STEP, requestId, producerId, sequence, step },
{ tab: { id: tabId } } as chrome.runtime.MessageSender,
() => {},
{ tab: { id: tabId, active } } as chrome.runtime.MessageSender,
sendResponse,
);
return sendResponse;
}
describe("recorded user steps reach the exported trace", () => {
@@ -1161,7 +1200,8 @@ describe("recorded user steps reach the exported trace", () => {
const trace = (stopped as RecordStopResult).trace;
expect("version" in trace).toBe(false);
expect("pages" in trace && trace.pages.length).toBeGreaterThan(0);
expect("steps" in trace && trace.steps.map((step) => step.op)).toEqual(["hover", "click"]);
// `hover` has no v2 counterpart on peers that negotiate v2.
expect("steps" in trace && trace.steps.map((step) => step.op)).toEqual(["click"]);
});
it("rejects unsupported trace_version values", async () => {
@@ -1173,4 +1213,391 @@ describe("recorded user steps reach the exported trace", () => {
);
expect(result).toMatchObject({ code: "invalid_params" });
});
it("records tab transitions and binds actions to each tab's observation", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const secondUrl = "https://example.com/second";
const tabsApi = makeMultiTabsApi([
{
id: TAB_ID,
windowId: AGENT_WINDOW_ID,
active: true,
status: "complete",
url: START_URL,
title: "First tab",
} as chrome.tabs.Tab,
{
id: 5,
windowId: AGENT_WINDOW_ID,
active: false,
status: "complete",
url: secondUrl,
title: "Second tab",
} as chrome.tabs.Tab,
]);
const cdp = makeFakeCdp(undefined, {
treesByTab: {
[TAB_ID]: axTree("First tab", ["First action"]),
5: axTree("Second tab", ["Second action"]),
},
});
let requestId = "";
const sendToTab = vi.fn(async (_tabId: number, message: unknown) => {
const typed = message as { type?: string; requestId?: string };
if (typed.type === RECORD_START && typed.requestId) requestId = typed.requestId;
return { ok: true };
});
const deps = { tabsApi, sendToTab, cdp };
await handleRecordStart(manager, RECORD_START_V3, deps);
attachRecordStepListener(deps);
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
await settleWait();
runtimeOnMessageEmit(
chromeApi,
requestId,
{
op: "click",
page_url: secondUrl,
target: { role: "button", name: "Second action", tag: "button" },
},
5,
);
await settleWait();
tabsApi.activate(TAB_ID);
chromeApi.tabsOnActivated.emit({ tabId: TAB_ID, windowId: AGENT_WINDOW_ID });
await settleWait();
runtimeOnMessageEmit(chromeApi, requestId, {
op: "click",
page_url: START_URL,
target: { role: "button", name: "First action", tag: "button" },
});
await settleWait();
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
const trace = asTraceV3((stopped as RecordStopResult).trace);
const stateUrl = (stateId: string) => trace.states.find((state) => state.id === stateId)?.url;
const switches = trace.steps.filter((step) => step.op === "switch_tab");
const clicks = trace.steps.filter((step) => step.op === "click");
expect(switches).toHaveLength(2);
expect(clicks).toHaveLength(2);
expect(stateUrl(switches[0]!.state)).toBe(START_URL);
expect(stateUrl(switches[0]!.result.state)).toBe(secondUrl);
expect(stateUrl(clicks[0]!.state)).toBe(secondUrl);
expect(stateUrl(switches[1]!.result.state)).toBe(START_URL);
expect(stateUrl(clicks[1]!.state)).toBe(START_URL);
});
it("keeps switch_tab internal when the v3 caller did not advertise support", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const tabsApi = makeMultiTabsApi([
{
id: TAB_ID,
windowId: AGENT_WINDOW_ID,
active: true,
status: "complete",
url: START_URL,
} as chrome.tabs.Tab,
{
id: 5,
windowId: AGENT_WINDOW_ID,
active: false,
status: "complete",
url: "https://example.com/second",
} as chrome.tabs.Tab,
]);
const deps = { tabsApi, sendToTab: vi.fn(async () => ({ ok: true })), cdp: makeFakeCdp() };
await handleRecordStart(
manager,
{ session_id: "abcd", url: START_URL, trace_version: 3 },
deps,
);
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
await settleWait();
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
const trace = asTraceV3((stopped as RecordStopResult).trace);
expect(trace.steps.some((step) => step.op === "switch_tab")).toBe(false);
});
it("keeps the latest rapid tab activation when an earlier rearm acknowledges late", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const tabsApi = makeMultiTabsApi(
[TAB_ID, 5, 6].map(
(id) =>
({
id,
windowId: AGENT_WINDOW_ID,
active: id === TAB_ID,
status: "complete",
url: id === TAB_ID ? START_URL : `https://example.com/tab-${id}`,
}) as chrome.tabs.Tab,
),
);
let releaseSecond!: () => void;
const secondGate = new Promise<void>((resolve) => {
releaseSecond = resolve;
});
let announceSecond!: () => void;
const secondStarted = new Promise<void>((resolve) => {
announceSecond = resolve;
});
const sendToTab = vi.fn(async (tabId: number, message: unknown) => {
if ((message as { type?: string }).type === RECORD_START && tabId === 5) {
announceSecond();
await secondGate;
}
return { ok: true };
});
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
await handleRecordStart(manager, RECORD_START_V3, deps);
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
await secondStarted;
tabsApi.activate(6);
chromeApi.tabsOnActivated.emit({ tabId: 6, windowId: AGENT_WINDOW_ID });
await settleWait();
releaseSecond();
await new Promise((resolve) => setTimeout(resolve, 50));
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
const trace = asTraceV3((stopped as RecordStopResult).trace);
const switches = trace.steps.filter((step) => step.op === "switch_tab");
const stateUrl = (stateId: string) => trace.states.find((state) => state.id === stateId)?.url;
expect(switches).toHaveLength(1);
expect(stateUrl(switches[0]!.result.state)).toBe("https://example.com/tab-6");
});
it("does not retry an in-flight tab rearm after recording stops", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const tabsApi = makeMultiTabsApi([
{
id: TAB_ID,
windowId: AGENT_WINDOW_ID,
active: true,
status: "complete",
url: START_URL,
} as chrome.tabs.Tab,
{
id: 5,
windowId: AGENT_WINDOW_ID,
active: false,
status: "complete",
url: "https://example.com/second",
} as chrome.tabs.Tab,
]);
let failRearm!: () => void;
const rearmGate = new Promise<never>((_resolve, reject) => {
failRearm = () => reject(new Error("rearm failed after stop"));
});
let announceRearm!: () => void;
const rearmStarted = new Promise<void>((resolve) => {
announceRearm = resolve;
});
const sendToTab = vi.fn(async (tabId: number, message: unknown) => {
if ((message as { type?: string }).type === RECORD_START && tabId === 5) {
announceRearm();
await rearmGate;
}
return { ok: true };
});
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
await handleRecordStart(manager, RECORD_START_V3, deps);
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
await rearmStarted;
const stoppedPromise = handleRecordStop(manager, { session_id: "abcd" }, deps);
await vi.waitFor(() => {
expect(sendToTab).toHaveBeenCalledWith(5, expect.objectContaining({ type: RECORD_STOP }));
});
failRearm();
const stopped = await stoppedPromise;
await new Promise((resolve) => setTimeout(resolve, 600));
const startsForSecondTab = sendToTab.mock.calls.filter(
([tabId, message]) => tabId === 5 && (message as { type?: string }).type === RECORD_START,
);
expect(startsForSecondTab).toHaveLength(1);
expect(asTraceV3((stopped as RecordStopResult).trace).steps).toHaveLength(0);
});
it("keeps pending action settles scoped to their tab during an immediate switch", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const secondUrl = "https://example.com/second";
const tabsApi = makeMultiTabsApi([
{
id: TAB_ID,
windowId: AGENT_WINDOW_ID,
active: true,
status: "complete",
url: START_URL,
} as chrome.tabs.Tab,
{
id: 5,
windowId: AGENT_WINDOW_ID,
active: false,
status: "complete",
url: secondUrl,
} as chrome.tabs.Tab,
]);
let requestId = "";
const sendToTab = vi.fn(async (_tabId: number, message: unknown) => {
const typed = message as { type?: string; requestId?: string };
if (typed.type === RECORD_START && typed.requestId) requestId = typed.requestId;
return { ok: true };
});
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
await handleRecordStart(manager, RECORD_START_V3, deps);
attachRecordStepListener(deps);
runtimeOnMessageEmit(chromeApi, requestId, {
op: "click",
page_url: START_URL,
target: { role: "button", name: "First action", tag: "button" },
});
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
runtimeOnMessageEmit(
chromeApi,
requestId,
{
op: "click",
page_url: secondUrl,
target: { role: "button", name: "Second action", tag: "button" },
},
5,
);
await settleWait();
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
const trace = asTraceV3((stopped as RecordStopResult).trace);
const stateUrl = (stateId: string) => trace.states.find((state) => state.id === stateId)?.url;
expect(trace.steps.map((step) => step.op)).toEqual(["click", "switch_tab", "click"]);
expect(stateUrl(trace.steps[0]!.result.state)).toBe(START_URL);
expect(stateUrl(trace.steps[1]!.result.state)).toBe(secondUrl);
expect(stateUrl(trace.steps[2]!.state)).toBe(secondUrl);
});
it("tracks the same navigation URL independently for each tab", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const tabsApi = makeMultiTabsApi([
{
id: TAB_ID,
windowId: AGENT_WINDOW_ID,
active: true,
status: "complete",
url: START_URL,
} as chrome.tabs.Tab,
{
id: 5,
windowId: AGENT_WINDOW_ID,
active: false,
status: "complete",
url: "https://example.com/second",
} as chrome.tabs.Tab,
]);
const deps = { tabsApi, sendToTab: vi.fn(async () => ({ ok: true })), cdp: makeFakeCdp() };
await handleRecordStart(manager, RECORD_START_V3, deps);
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
await settleWait();
tabsApi.goTo(5, START_URL, "Same URL in second tab");
chromeApi.webNavigationOnCommitted.emit({
tabId: 5,
frameId: 0,
url: START_URL,
transitionType: "typed",
transitionQualifiers: ["from_address_bar"],
} as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails);
await settleWait();
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
const trace = asTraceV3((stopped as RecordStopResult).trace);
expect(
trace.steps.filter((step) => step.op === "navigate" && step.to === START_URL),
).toHaveLength(1);
});
it("drops late input and frame navigation from an inactive tab", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const secondUrl = "https://example.com/second";
const tabsApi = makeMultiTabsApi([
{
id: TAB_ID,
windowId: AGENT_WINDOW_ID,
active: true,
status: "complete",
url: START_URL,
} as chrome.tabs.Tab,
{
id: 5,
windowId: AGENT_WINDOW_ID,
active: false,
status: "complete",
url: secondUrl,
} as chrome.tabs.Tab,
]);
let requestId = "";
const sendToTab = vi.fn(async (_tabId: number, message: unknown) => {
const typed = message as { type?: string; requestId?: string };
if (typed.type === RECORD_START && typed.requestId) requestId = typed.requestId;
return { ok: true };
});
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
await handleRecordStart(manager, RECORD_START_V3, deps);
attachRecordStepListener(deps);
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
await settleWait();
const lateAck = runtimeOnMessageEmit(
chromeApi,
requestId,
{
op: "fill",
page_url: START_URL,
target: { role: "textbox", name: "Draft", tag: "input" },
value: "late",
},
TAB_ID,
false,
);
expect(lateAck).toHaveBeenCalledWith({ ok: true, sequence: 1 });
chromeApi.webNavigationOnCommitted.emit({
tabId: TAB_ID,
frameId: 0,
url: "https://example.com/late",
transitionType: "link",
transitionQualifiers: [],
} as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails);
chromeApi.webNavigationOnCommitted.emit({
tabId: 5,
frameId: 7,
url: "https://frame.example/late",
transitionType: "link",
transitionQualifiers: [],
} as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails);
await new Promise((resolve) => setTimeout(resolve, 50));
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
const trace = asTraceV3((stopped as RecordStopResult).trace);
expect(trace.steps.map((step) => step.op)).toEqual(["switch_tab"]);
});
});
+34 -37
View File
@@ -51,7 +51,12 @@ import {
handleScreenshot,
handleSnapshot,
} from "./observation";
import { handleRecordAwait, handleRecordStart, handleRecordStop } from "./record";
import {
handleRecordAwait,
handleRecordStart,
handleRecordStop,
type RecordRuntimeDeps,
} from "./record";
import {
handleSessionStart,
handleSessionStop,
@@ -99,6 +104,7 @@ export interface DispatcherDeps {
transport: Transport;
sessions: SessionManager;
cdp?: DispatcherCdpRunner;
recording?: RecordRuntimeDeps;
/**
* Invoked whenever a dispatched RPC may have changed the live
* session set (currently `tool.session_start` and
@@ -134,6 +140,7 @@ export class ToolDispatcher {
private readonly transport: Transport;
private readonly sessions: SessionManager;
private readonly cdp?: DispatcherCdpRunner;
private readonly recording?: RecordRuntimeDeps;
private readonly onSessionsChanged?: () => void;
private readonly onBrowserControlResumed?: (sessionId: string) => void;
private readonly approveBorrow?: BorrowConfirmationApprover;
@@ -153,6 +160,7 @@ export class ToolDispatcher {
this.transport = deps.transport;
this.sessions = deps.sessions;
this.cdp = deps.cdp;
this.recording = deps.recording;
this.onSessionsChanged = deps.onSessionsChanged;
this.onBrowserControlResumed = deps.onBrowserControlResumed;
this.approveBorrow = deps.approveBorrow;
@@ -531,44 +539,26 @@ export class ToolDispatcher {
signal,
});
case "tool.record_start":
return handleRecordStart(this.sessions, req.params as RecordStartParams, {
tabsApi: chromeTabsApi,
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
bypassOverlay: async (tabId, enabled) => {
try {
await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
...(this.cdp ? { cdp: this.cdp } : {}),
signal,
});
return this.recording
? handleRecordStart(this.sessions, req.params as RecordStartParams, {
...this.recording,
signal,
})
: recordingRuntimeUnavailable();
case "tool.record_stop":
return handleRecordStop(this.sessions, req.params as RecordStopParams, {
tabsApi: chromeTabsApi,
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
bypassOverlay: async (tabId, enabled) => {
try {
await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
signal,
});
return this.recording
? handleRecordStop(this.sessions, req.params as RecordStopParams, {
...this.recording,
signal,
})
: recordingRuntimeUnavailable();
case "tool.record_await":
return handleRecordAwait(this.sessions, req.params as RecordAwaitParams, {
tabsApi: chromeTabsApi,
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
signal,
});
return this.recording
? handleRecordAwait(this.sessions, req.params as RecordAwaitParams, {
...this.recording,
signal,
})
: recordingRuntimeUnavailable();
default:
return {
code: "unknown_method",
@@ -704,6 +694,13 @@ function isRpcError(v: unknown): v is RpcError {
);
}
function recordingRuntimeUnavailable(): RpcError {
return {
code: "protocol_error",
message: "recording runtime is unavailable",
};
}
function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
switch (req.method) {
case "tool.tab_create":
+181 -76
View File
@@ -18,8 +18,18 @@ import {
type RecordStepAck,
type RecordStopMessage,
} from "@/lib/record-bridge";
import {
type RecordFrameCoordinator,
type RecordingCaptureScope,
recordFrameCoordinator,
} from "@/lib/recording/frame-coordinator";
import { RecordingObservationRuntime } from "@/lib/recording/recording-runtime";
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording/step-buffer";
import {
appendRecordedPayload,
observeRecordedNavigation,
type RecordingStepBuffer,
} from "@/lib/recording/step-buffer";
import { RecordingTabCoordinator, type TabActivation } from "@/lib/recording/tab-coordinator";
import { buildTraceV2 } from "@/lib/recording/trace-reducer-v2";
import type { RecordingDraftStep } from "@/lib/recording/types";
import type { SessionManager } from "@/session-manager/manager";
@@ -48,7 +58,7 @@ import {
interface ActiveRecording {
requestId: string;
tabId: number;
tabs: RecordingTabCoordinator;
agentWindowId: number;
startUrl?: string;
purpose?: string;
@@ -56,14 +66,12 @@ interface ActiveRecording {
startedAt: string;
startedAtMs: number;
traceVersion: 2 | 3;
supportsTabSwitchSteps: boolean;
finishPromise: Promise<RecordedTrace>;
resolveFinish: (trace: RecordedTrace) => void;
rejectFinish: (err: Error) => void;
settled: boolean;
finishAttempt: Promise<RecordedTrace | null> | null;
currentUrl?: string;
pendingNavigation: boolean;
pendingNavigationDeadline?: number;
observation: RecordingObservationRuntime | null;
stoppedBy: StopReason;
/** Navigation callbacks tracked from event receipt through action enqueue. */
@@ -79,8 +87,17 @@ interface ActiveRecording {
lastStepSequenceByProducer: Map<string, number>;
}
function enqueueRecordingAction(recording: ActiveRecording, task: () => Promise<void>): void {
recording.actionQueue = recording.actionQueue.then(task, task).catch(() => {});
function enqueueRecordingAction(
recording: ActiveRecording,
task: () => Promise<void>,
): Promise<void> {
const queued = recording.actionQueue.then(task, task);
recording.actionQueue = queued.catch(() => {});
return queued;
}
function isRecordingFinishing(recording: ActiveRecording): boolean {
return recording.settled || recording.finishAttempt !== null;
}
const recordings = new Map<string, ActiveRecording>();
@@ -163,9 +180,11 @@ async function sendRecordStartWithAck(
tabId: number,
msg: RecordStartMessage,
sendToTab: RecordDeps["sendToTab"],
cancelled?: () => boolean,
): Promise<void> {
let lastError: unknown;
for (let attempt = 0; attempt < RECORD_START_RETRIES; attempt += 1) {
if (cancelled?.()) throw new Error("record start cancelled");
try {
const response = await sendToTab(tabId, msg);
if (isRecordStartAck(response)) return;
@@ -173,6 +192,7 @@ async function sendRecordStartWithAck(
} catch (err) {
lastError = err;
}
if (cancelled?.()) throw new Error("record start cancelled");
if (attempt + 1 < RECORD_START_RETRIES) {
await sleep(RECORD_START_RETRY_DELAY_MS);
}
@@ -198,6 +218,7 @@ function buildTrace(recording: ActiveRecording): RecordedTrace {
startUrl: recording.startUrl,
stoppedBy: recording.stoppedBy,
bskVersion: BSK_TRACE_VERSION,
includeTabSwitches: recording.supportsTabSwitchSteps,
});
}
if (recording.traceVersion === 3) {
@@ -211,14 +232,46 @@ function buildTrace(recording: ActiveRecording): RecordedTrace {
});
}
function stepBufferFor(
recording: ActiveRecording,
tabId: number,
fallbackUrl?: string,
): RecordingStepBuffer {
return {
steps: recording.steps,
navigation: recording.tabs.navigation(tabId, fallbackUrl),
};
}
async function activateRecordingTab(
recording: ActiveRecording,
targetTabId: number,
): Promise<void> {
const previousTabId = recording.tabs.currentTabId;
if (targetTabId === previousTabId) return;
const transition = recording.observation
? await recording.observation.captureTabTransition(previousTabId, targetTabId)
: {};
const { targetUrl, ...stateLinks } = transition;
const draft: Extract<RecordingDraftStep, { op: "switch_tab" }> = {
op: "switch_tab",
...stateLinks,
};
recording.steps.push(draft);
recording.observation?.bindTabTransition(draft, recording.steps.length);
recording.tabs.commit(targetTabId, targetUrl);
}
async function processRecordedStep(
recording: ActiveRecording,
draftIndex: number,
tabId: number,
producerId?: string,
): Promise<void> {
if (!recording.observation) return;
try {
await recording.observation.processDraft(tabId, recording.steps, draftIndex);
await recording.observation.processDraft(tabId, recording.steps, draftIndex, producerId);
} catch (err) {
console.warn(`[bsk record] observation failed for step ${draftIndex + 1}`, err);
}
@@ -231,16 +284,25 @@ export interface RecordDeps {
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
): Promise<unknown>;
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
frameCoordinator?: Pick<
RecordFrameCoordinator,
"begin" | "armTab" | "sourceFor" | "stop" | "cancel"
>;
cdp?: CdpRunner;
signal?: AbortSignal;
}
export type RecordRuntimeDeps = Omit<RecordDeps, "frameCoordinator" | "signal"> & {
frameCoordinator: NonNullable<RecordDeps["frameCoordinator"]>;
};
let defaultDeps: RecordDeps | null = null;
function getDefaultDeps(): RecordDeps {
if (!defaultDeps) {
defaultDeps = {
tabsApi: chromeTabsApi,
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
frameCoordinator: recordFrameCoordinator,
};
}
return defaultDeps;
@@ -307,8 +369,15 @@ export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): (
if (!isRecordStepMessage(message)) return false;
for (const recording of recordings.values()) {
if (recording.requestId !== message.requestId) continue;
const sourceTabId = sender.tab?.id ?? recording.tabId;
const producerKey = `${sourceTabId}:${message.producerId}`;
const source: RecordingCaptureScope | null | undefined = deps.frameCoordinator
? deps.frameCoordinator.sourceFor(message.requestId, message.producerId, sender)
: undefined;
if (deps.frameCoordinator && !source) return false;
const sourceTabId = source?.tabId ?? sender.tab?.id ?? recording.tabs.currentTabId;
const sourceWasActive = sender.tab?.active ?? sourceTabId === recording.tabs.activeTabId;
const producerKey = source
? `${source.tabId}:${source.documentId}:${source.producerId}`
: `${sourceTabId}:${message.producerId}`;
const expectedSequence = (recording.lastStepSequenceByProducer.get(producerKey) ?? 0) + 1;
if (message.sequence < expectedSequence) {
sendResponse({ ok: true, sequence: message.sequence });
@@ -325,11 +394,21 @@ export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): (
recording.lastStepSequenceByProducer.set(producerKey, message.sequence);
enqueueRecordingAction(recording, async () => {
await recording.observation?.flushRedirects();
if (sourceTabId !== recording.tabs.currentTabId) {
if (!sourceWasActive) return;
if (sourceTabId !== recording.tabs.activeTabId)
recording.tabs.noteActivation(sourceTabId);
await activateRecordingTab(recording, sourceTabId);
}
await recording.observation?.flushRedirects(sourceTabId);
const targetHint = message.step.geometry ? { geometry: message.step.geometry } : undefined;
const draftIndex = appendRecordedPayload(recording, message.step, targetHint);
const draftIndex = appendRecordedPayload(
stepBufferFor(recording, sourceTabId, message.step.page_url),
message.step,
targetHint,
);
if (draftIndex !== null) {
await processRecordedStep(recording, draftIndex, sourceTabId);
await processRecordedStep(recording, draftIndex, sourceTabId, source?.producerId);
}
});
sendResponse({ ok: true, sequence: message.sequence });
@@ -359,7 +438,7 @@ export function attachRecordFinishListener(deps: RecordDeps = getDefaultDeps()):
function findRecordingByTabId(tabId: number): ActiveRecording | null {
for (const recording of recordings.values()) {
if (recording.tabId === tabId && !recording.settled) return recording;
if (recording.tabs.currentTabId === tabId && !recording.settled) return recording;
}
return null;
}
@@ -396,7 +475,7 @@ async function clearRearmTimersForRecording(
recording: ActiveRecording,
deps: RecordDeps,
): Promise<void> {
clearRearmTimer(recording.tabId);
clearRearmTimer(recording.tabs.currentTabId);
try {
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
for (const tab of tabs) {
@@ -411,13 +490,16 @@ async function stopRecordingOnAllAgentTabs(
recording: ActiveRecording,
deps: RecordDeps,
): Promise<void> {
if (deps.frameCoordinator && !(await deps.frameCoordinator.stop(recording.requestId))) {
throw new Error("failed to flush one or more recording documents");
}
const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
let tabIds = [recording.tabId];
let tabIds = [recording.tabs.currentTabId];
try {
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
tabIds = [
...new Set([
recording.tabId,
recording.tabs.currentTabId,
...tabs.flatMap((tab) => (typeof tab.id === "number" ? [tab.id] : [])),
]),
];
@@ -428,11 +510,11 @@ async function stopRecordingOnAllAgentTabs(
for (const tabId of tabIds) {
try {
const response = await deps.sendToTab(tabId, stopMsg);
if (tabId === recording.tabId && !isRecordStartAck(response)) {
if (tabId === recording.tabs.currentTabId && !isRecordStartAck(response)) {
throw new Error("content script did not confirm recorded steps");
}
} catch {
if (tabId === recording.tabId) {
if (tabId === recording.tabs.currentTabId) {
throw new Error("failed to flush recorded steps");
}
}
@@ -450,22 +532,39 @@ async function rearmRecording(
recording: ActiveRecording,
targetTabId: number,
deps: RecordDeps,
activation?: TabActivation,
): Promise<boolean> {
// Do NOT toggle automation-bypass here: each retry used to increment the
// content-script counter, and a single stop decrement left the ControlOverlay
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
// already hides the control chrome while activeRecord is set.
const isFinishing = () => isRecordingFinishing(recording);
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
if (isFinishing()) return false;
const startMsg: RecordStartMessage = {
type: RECORD_START,
requestId: recording.requestId,
startedAtMs: recording.startedAtMs,
};
try {
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
recording.tabId = targetTabId;
if (deps.frameCoordinator) {
const frameStarted = await deps.frameCoordinator.armTab(recording.requestId, targetTabId);
if (!frameStarted) throw new Error("recording document did not start");
}
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab, isFinishing);
if (isFinishing()) return false;
if (activation) {
if (!recording.tabs.isLatest(activation)) return true;
await enqueueRecordingAction(recording, async () => {
if (isFinishing() || !recording.tabs.isLatest(activation)) {
return;
}
await activateRecordingTab(recording, targetTabId);
});
}
return true;
} catch {
if (isFinishing()) return false;
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
await sleep(RECORD_REARM_RETRY_DELAY_MS);
}
@@ -474,7 +573,7 @@ async function rearmRecording(
return false;
}
function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
function scheduleRearmForTab(tabId: number, deps: RecordDeps, activation?: TabActivation): void {
const existing = rearmTimers.get(tabId);
if (existing) clearTimeout(existing);
rearmTimers.set(
@@ -483,7 +582,7 @@ function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
rearmTimers.delete(tabId);
void (async () => {
const current = await findRecordingForTab(tabId, deps);
if (current) await rearmRecording(current, tabId, deps);
if (current) await rearmRecording(current, tabId, deps, activation);
})();
}, RECORD_REARM_DEBOUNCE_MS),
);
@@ -495,14 +594,20 @@ export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): ()
const windowId = tab.windowId;
if (tabId === undefined || windowId === undefined) return;
for (const recording of recordings.values()) {
if (recording.settled || recording.agentWindowId !== windowId) continue;
if (isRecordingFinishing(recording) || recording.agentWindowId !== windowId) continue;
scheduleRearmForTab(tabId, deps);
return;
}
};
const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => {
scheduleRearmForTab(activeInfo.tabId, deps);
for (const recording of recordings.values()) {
if (isRecordingFinishing(recording) || recording.agentWindowId !== activeInfo.windowId)
continue;
const activation = recording.tabs.noteActivation(activeInfo.tabId);
scheduleRearmForTab(activeInfo.tabId, deps, activation);
return;
}
};
chrome.tabs.onCreated.addListener(onCreated);
@@ -522,54 +627,39 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
transitionQualifiers?: string[],
) => {
if (!url) return;
const direct = findRecordingByTabId(tabId);
const candidates = direct
? direct.acceptingNavigation
? [direct]
: []
: [...recordings.values()].filter(
(recording) => !recording.settled && recording.acceptingNavigation,
const candidates = [...recordings.values()].filter(
(recording) =>
!recording.settled && recording.acceptingNavigation && recording.tabs.activeTabId === tabId,
);
for (const recording of candidates) {
const queued = enqueueRecordingAction(recording, async () => {
if (tabId !== recording.tabs.currentTabId) {
await activateRecordingTab(recording, tabId);
}
const result = observeRecordedNavigation(
stepBufferFor(recording, tabId, url),
url,
causedByAction,
transitionType,
transitionQualifiers,
);
if (candidates.length === 0) return;
let resolveTracked!: () => void;
const tracked = new Promise<void>((resolve) => {
resolveTracked = resolve;
});
for (const candidate of candidates) candidate.navigationCallbacks.add(tracked);
void (async () => {
try {
const recording = await findRecordingForTab(tabId, deps);
if (!recording || !recording.acceptingNavigation || !candidates.includes(recording)) {
if (result.kind === "coalesce_redirect") {
recording.observation?.scheduleRedirect(tabId, recording.steps, result.url);
return;
}
enqueueRecordingAction(recording, async () => {
const result = observeRecordedNavigation(
recording,
url,
causedByAction,
transitionType,
transitionQualifiers,
);
if (result.kind === "coalesce_redirect") {
recording.observation?.scheduleRedirect(tabId, recording.steps, result.url);
return;
}
if (result.kind === "noop") return;
recording.observation?.clearRedirect(tabId);
if (result.kind === "appended") {
await processRecordedStep(recording, result.index, tabId);
} else {
recording.observation?.scheduleSettle(tabId, recording.steps, result.index);
}
});
} finally {
for (const candidate of candidates) candidate.navigationCallbacks.delete(tracked);
resolveTracked();
}
})();
if (result.kind === "noop") return;
recording.observation?.clearRedirect(tabId);
if (result.kind === "appended") {
await processRecordedStep(recording, result.index, tabId);
} else {
recording.observation?.scheduleSettle(tabId, recording.steps, result.index);
}
});
const tracked = queued.catch(() => {});
recording.navigationCallbacks.add(tracked);
void tracked.finally(() => recording.navigationCallbacks.delete(tracked));
}
};
const onMainFrameComplete = (tabId: number, url?: string) => {
void (async () => {
@@ -651,7 +741,8 @@ export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()):
sendResponse({ active: false });
return;
}
await rearmRecording(recording, tabId, deps);
const activation = sender.tab?.active ? recording.tabs.noteActivation(tabId) : undefined;
await rearmRecording(recording, tabId, deps, activation);
sendResponse({
active: true,
requestId: recording.requestId,
@@ -716,7 +807,7 @@ async function finishRecordingAttempt(
if (!(await drainRecordingToStability(recording))) {
return null;
}
await recording.observation?.settleTrailing(recording.tabId, recording.steps);
await recording.observation?.settleTrailing(recording.tabs.currentTabId, recording.steps);
recording.settled = true;
recordings.delete(sessionId);
@@ -768,7 +859,7 @@ export async function handleRecordStart(
const redactValues = params.redact_values ?? false;
recordings.set(params.session_id, {
requestId,
tabId: target.tabId,
tabs: new RecordingTabCoordinator(target.tabId, navigateUrl),
agentWindowId: ctx.agentWindowId,
startUrl: navigateUrl,
...(params.purpose ? { purpose: params.purpose } : {}),
@@ -776,14 +867,12 @@ export async function handleRecordStart(
startedAt: new Date(startedAtMs).toISOString(),
startedAtMs,
traceVersion: traceVersionOrErr,
supportsTabSwitchSteps: params.supports_tab_switch_steps === true,
finishPromise,
resolveFinish,
rejectFinish,
settled: false,
finishAttempt: null,
currentUrl: navigateUrl,
pendingNavigation: false,
pendingNavigationDeadline: undefined,
observation:
traceVersionOrErr === 3 && deps.cdp
? new RecordingObservationRuntime({
@@ -799,12 +888,23 @@ export async function handleRecordStart(
actionQueue: Promise.resolve(),
lastStepSequenceByProducer: new Map(),
});
deps.frameCoordinator?.begin(requestId, startedAtMs, target.tabId, async (scope) => {
const recording = recordings.get(params.session_id);
if (!recording || recording.requestId !== requestId || recording.settled) return;
try {
await recording.observation?.refreshDocument(scope.tabId, scope.producerId);
} catch {
// The normal action-time capture remains available if a child Document
// is replaced while its readiness refresh is in flight.
}
});
// Observe navigations for the whole recording lifetime; attach before
// optional navigate so the destination load can rearm capture.
ensureBrowserObservationListeners(deps);
const abortPending = async (notifyContent: boolean) => {
recordings.get(params.session_id)?.observation?.cancel();
deps.frameCoordinator?.cancel(requestId);
recordings.delete(params.session_id);
releaseBrowserObservationListenersIfIdle();
if (notifyContent) {
@@ -886,7 +986,7 @@ export async function handleRecordStart(
return cancelledError();
}
active.startUrl = startUrl;
active.currentUrl = startUrl;
active.tabs.navigation(target.tabId, startUrl).currentUrl = startUrl;
if (isContentScriptRestrictedUrl(startUrl)) {
await abortPending(false);
@@ -921,6 +1021,10 @@ export async function handleRecordStart(
const startMsg: RecordStartMessage = { type: RECORD_START, requestId, startedAtMs };
try {
if (deps.frameCoordinator) {
const frameStarted = await deps.frameCoordinator.armTab(requestId, target.tabId);
if (!frameStarted) throw new Error("top recording document did not start");
}
await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
} catch {
await abortPending(true);
@@ -1040,6 +1144,7 @@ export function clearRecordingForSession(sessionId: string): void {
}
void clearRearmTimersForRecording(recording, getDefaultDeps());
if (!recording.settled) {
getDefaultDeps().frameCoordinator?.cancel(recording.requestId);
recording.observation?.cancel();
recording.settled = true;
recording.rejectFinish(new Error("recording cleared"));
@@ -0,0 +1,82 @@
import { describe, expect, it } from "vitest";
import {
RECORD_DOCUMENT_ATTRIBUTE,
recordingDocumentMarkerValue,
} from "@/shared/recording-document-identity";
import { projectRecordSafeObservation } from "../record-safe-observation";
describe("projectRecordSafeObservation", () => {
it("projects the opaque recording identity without exposing captured attributes", () => {
const recordingDocumentId = "123e4567-e89b-42d3-a456-426614174000";
const result = projectRecordSafeObservation({
rootFrameId: "child",
frameDocuments: [
{
frameId: "child",
target: { tabId: 4, sessionId: "oopif-session" },
contextScopeId: "scope-1",
axNodes: [],
domNodes: [
{
backendNodeId: 1,
parentBackendNodeId: null,
tag: "html",
attrs: {
[RECORD_DOCUMENT_ATTRIBUTE]: recordingDocumentMarkerValue(recordingDocumentId),
value: "user@example.com",
},
rect: null,
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
],
},
],
rendered: { text: '@vom 1\nRootWebArea "Example"', refs: [], truncated: false },
});
expect(result.frames).toEqual([
{
frameId: "child",
target: { tabId: 4, sessionId: "oopif-session" },
recordingDocumentId,
},
]);
const dumped = JSON.stringify(result);
expect(dumped).not.toContain("user@example.com");
expect(dumped).not.toContain("attrs");
expect(dumped).not.toContain("domNodes");
expect(dumped).not.toContain("axNodes");
});
it("does not project a page-controlled marker value as recording identity", () => {
const result = projectRecordSafeObservation({
rootFrameId: "root",
frameDocuments: [
{
frameId: "root",
target: { tabId: 4 },
contextScopeId: "scope-1",
axNodes: [],
domNodes: [
{
backendNodeId: 1,
parentBackendNodeId: null,
tag: "html",
attrs: { [RECORD_DOCUMENT_ATTRIBUTE]: "bsk:user@example.com" },
rect: null,
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
],
},
],
rendered: { text: '@vom 1\nRootWebArea "Example"', refs: [], truncated: false },
});
expect(result.frames).toEqual([{ frameId: "root", target: { tabId: 4 } }]);
expect(JSON.stringify(result)).not.toContain("user@example.com");
});
});
@@ -1,5 +1,6 @@
import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import { readRecordingDocumentIdentity } from "@/shared/recording-document-identity";
import type { CapturedSurfaceProbe } from "./capture";
import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture";
@@ -9,6 +10,8 @@ export interface CaptureVomFrame {
target: CdpTarget;
parentFrameId?: string;
ownerBackendNodeId?: number;
/** BrowserSkill-generated identity for the recording agent in this Document. */
recordingDocumentId?: string;
}
/** Allowlisted geometry used to match a recorded action to a rendered ref. */
@@ -37,14 +40,22 @@ export interface CaptureVomObservationResult {
}
function projectFrames(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomFrame[] {
return documents.map((document) => ({
frameId: document.frameId,
target: document.target,
...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}),
...(document.ownerBackendNodeId !== undefined
? { ownerBackendNodeId: document.ownerBackendNodeId }
: {}),
}));
return documents.map((document) => {
let recordingDocumentId: string | undefined;
for (const node of document.domNodes) {
recordingDocumentId = readRecordingDocumentIdentity(node.attrs);
if (recordingDocumentId) break;
}
return {
frameId: document.frameId,
target: document.target,
...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}),
...(document.ownerBackendNodeId !== undefined
? { ownerBackendNodeId: document.ownerBackendNodeId }
: {}),
...(recordingDocumentId ? { recordingDocumentId } : {}),
};
});
}
function projectMatchNodes(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomMatchNode[] {
+3
View File
@@ -780,6 +780,7 @@ export interface SelectedOptionV3 {
export type StepV3 =
| ({ op: "navigate" } & StepCommonV3 & { to: string; cause: NavigationCause })
| ({ op: "switch_tab" } & StepCommonV3)
| ({ op: "click" } & StepCommonV3 & { target: TargetDescriptorV3 })
| ({ op: "hover" } & StepCommonV3 & { target: TargetDescriptorV3 })
| ({ op: "fill" } & StepCommonV3 & {
@@ -823,6 +824,8 @@ export interface RecordStartParams {
redact_values?: boolean;
/** Omitted means v2; `3` requests a state-linked v3 trace. */
trace_version?: number;
/** Client can decode the v3 `switch_tab` step variant. */
supports_tab_switch_steps?: boolean;
}
export interface RecordStartResult {
+1
View File
@@ -115,6 +115,7 @@ fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError>
max_page_tokens: args.max_page_tokens,
redact_values: Some(args.redact_values),
trace_version: Some(TRACE_VERSION_V3),
supports_tab_switch_steps: Some(true),
};
let start_result = business_rpc::call::<RecordStartParams, RecordStartResult>(
info.sock_path.clone(),
@@ -480,6 +480,36 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
@@ -29,6 +29,13 @@
"session_id": {
"type": "string"
},
"supports_tab_switch_steps": {
"description": "Client can decode the v3 `switch_tab` step variant.",
"type": [
"boolean",
"null"
]
},
"tab_id": {
"type": [
"integer",
@@ -480,6 +480,36 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
+30
View File
@@ -469,6 +469,36 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
@@ -431,6 +431,36 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
@@ -41,6 +41,36 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
+30
View File
@@ -173,6 +173,36 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
+20
View File
@@ -39,6 +39,9 @@ pub struct RecordStartParams {
/// Desired trace export format. Omitted means v2; `3` requests v3.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub trace_version: Option<u32>,
/// Client can decode the v3 `switch_tab` step variant.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_tab_switch_steps: Option<bool>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
@@ -423,6 +426,19 @@ mod tests {
assert_eq!(round, step);
}
#[test]
fn step_switch_tab_round_trips() {
let step = StepV3::SwitchTab {
common: sample_common(2, "s2", "s5"),
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["op"], "switch_tab");
assert_eq!(v["state"], "s2");
assert_eq!(v["result"]["state"], "s5");
let round: StepV3 = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn trace_v3_round_trips() {
let trace = sample_trace();
@@ -463,10 +479,12 @@ mod tests {
max_page_tokens: None,
redact_values: None,
trace_version: None,
supports_tab_switch_steps: None,
};
let legacy_value = serde_json::to_value(legacy).unwrap();
assert!(legacy_value.get("trace_version").is_none());
assert!(legacy_value.get("max_page_tokens").is_none());
assert!(legacy_value.get("supports_tab_switch_steps").is_none());
let v3 = RecordStartParams {
session_id: "session".into(),
@@ -476,11 +494,13 @@ mod tests {
max_page_tokens: Some(4_000),
redact_values: Some(true),
trace_version: Some(TRACE_VERSION_V3),
supports_tab_switch_steps: Some(true),
};
let v3_value = serde_json::to_value(v3).unwrap();
assert_eq!(v3_value["trace_version"], TRACE_VERSION_V3);
assert_eq!(v3_value["max_page_tokens"], 4_000);
assert_eq!(v3_value["redact_values"], true);
assert_eq!(v3_value["supports_tab_switch_steps"], true);
}
#[test]
@@ -126,6 +126,10 @@ pub enum StepV3 {
to: String,
cause: NavigationCause,
},
SwitchTab {
#[serde(flatten)]
common: StepCommonV3,
},
Click {
#[serde(flatten)]
common: StepCommonV3,
@@ -340,6 +344,19 @@ mod tests {
assert_eq!(round, step);
}
#[test]
fn step_switch_tab_round_trips() {
let step = StepV3::SwitchTab {
common: sample_common(2, "s2", "s5"),
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["op"], "switch_tab");
assert_eq!(v["state"], "s2");
assert_eq!(v["result"]["state"], "s5");
let round: StepV3 = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn trace_v3_round_trips() {
let trace = sample_trace();