Compare commits

...
Author SHA1 Message Date
Ljy-0827 da3534d066 chore(release): bsk 0.1.11 2026-08-26 14:59:09 +08:00
BB-fat a004291848 feat(dsh-plugin): better-sidebar 集成、会话关停与归档清理、悬浮窗修复 (#130)
* better-sidebar 集成:observation 视图迁入 better-sidebar 载体,按会话作用域展示 bsk 会话
* 会话关停与归档清理:支持会话 dispose 与归档自动清理
* 悬浮窗修复:悬浮 overlay 拖拽/缩放/PiP 弹出等问题修复
* 修复 Document PiP 空白窗口:载体卸载(关 tab、切会话、插件 HMR)时主动 close() PiP 窗口,避免残留空白窗口
2026-08-25 10:33:46 +08:00
Hannah Liu 4b6cdde168 Merge pull request #115 from Tencent/fix/recorder-iframe
fix(recording): support user action recording in iframes and OOPIFs
2026-08-24 13:18:05 +08:00
Ljy-0827 4aade4b94f fix(recorder): bug fix, oopif load too late 2026-08-24 10:42:46 +08:00
Ljy-0827 616af1099f fix(record): saftey policy and rebase bug fix 2026-08-24 10:42:46 +08:00
Ljy-0827 3656f0dd3c fix(recorder): ci fix 2026-08-24 10:42:46 +08:00
Ljy-0827 9742d83213 fix(recorder): bug fix 2026-08-24 10:42:46 +08:00
Ljy-0827 615e866608 fix(recorder): add iframe support 2026-08-24 10:42:46 +08:00
Ljy-0827 c08fed5de3 fix(record): saftey policy and rebase bug fix 2026-08-24 10:42:45 +08:00
“haonan” 887d1ed0b9 fix(record): drop 'hover' in tracev2 for compatible 2026-08-24 10:42:45 +08:00
Ljy-0827 1a3d3b7de8 fix(record): bug fix 2026-08-24 10:42:45 +08:00
“haonan”andCursor 72834980d2 fix(record): record switch_tab and keep Trace v3 states per tab
Co-authored-by: Cursor <cursoragent@cursor.com>
2026-08-24 10:42:45 +08:00
67 changed files with 4464 additions and 628 deletions
Generated
+2 -2
View File
@@ -134,7 +134,7 @@ dependencies = [
[[package]]
name = "bsk"
version = "0.1.10"
version = "0.1.11"
dependencies = [
"anyhow",
"bsk-protocol",
@@ -168,7 +168,7 @@ dependencies = [
[[package]]
name = "bsk-protocol"
version = "0.1.10"
version = "0.1.11"
dependencies = [
"schemars",
"semver",
+1 -1
View File
@@ -7,7 +7,7 @@ members = [
default-members = ["crates/bsk-cli"]
[workspace.package]
version = "0.1.10"
version = "0.1.11"
edition = "2024"
rust-version = "1.85"
license = "MIT"
@@ -1,6 +1,6 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { RECORD_START, RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
import { handleRecordContentMessage, startRecordCapture } from "../record-capture";
import type { RecordStepPayload } from "@/lib/record-bridge";
import { startRecordCapture } from "../record-capture";
vi.stubGlobal("chrome", {
runtime: {
@@ -48,118 +48,6 @@ function click(el: Element): void {
el.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
}
describe("handleRecordContentMessage stop/cancel", () => {
it("ignores STOP when no recording is active", () => {
const dispose = vi.fn();
const onStop = vi.fn();
const setActiveRequestId = vi.fn();
const setCapture = vi.fn();
const sendResponse = vi.fn();
const needsAsync = handleRecordContentMessage(
{ type: RECORD_STOP, requestId: "rec-stale" },
{
activeRequestId: null,
capture: { dispose },
setActiveRequestId,
setCapture,
onStart: vi.fn(),
onStop,
},
sendResponse,
);
expect(needsAsync).toBe(false);
expect(dispose).not.toHaveBeenCalled();
expect(onStop).not.toHaveBeenCalled();
expect(setActiveRequestId).not.toHaveBeenCalled();
expect(setCapture).not.toHaveBeenCalled();
expect(sendResponse).not.toHaveBeenCalled();
});
it("ignores STOP for a mismatched requestId", () => {
const dispose = vi.fn();
const onStop = vi.fn();
const needsAsync = handleRecordContentMessage(
{ type: RECORD_STOP, requestId: "rec-other" },
{
activeRequestId: "rec-1",
capture: { dispose },
setActiveRequestId: vi.fn(),
setCapture: vi.fn(),
onStart: vi.fn(),
onStop,
},
vi.fn(),
);
expect(needsAsync).toBe(false);
expect(dispose).not.toHaveBeenCalled();
expect(onStop).not.toHaveBeenCalled();
});
it("keeps a failed STOP retryable and redelivers its recorded step", async () => {
document.body.innerHTML = `<label for="retry">Draft</label><input id="retry" />`;
const sendMessage = vi.mocked(chrome.runtime.sendMessage);
sendMessage.mockReset();
sendMessage.mockRejectedValueOnce(new Error("service worker unavailable"));
sendMessage.mockRejectedValueOnce(new Error("service worker still unavailable"));
sendMessage.mockResolvedValueOnce({ ok: true, sequence: 1 });
let activeRequestId: string | null = null;
let capture: ReturnType<typeof startRecordCapture> | null = null;
const onStop = vi.fn();
const dispatch = (
message:
| { type: typeof RECORD_START; requestId: string; startedAtMs?: number }
| { type: typeof RECORD_STOP; requestId: string },
sendResponse?: (response: unknown) => void,
) =>
handleRecordContentMessage(
message,
{
activeRequestId,
capture,
setActiveRequestId: (id) => {
activeRequestId = id;
},
setCapture: (next) => {
capture = next;
},
onStart: vi.fn(),
onStop,
},
sendResponse,
);
dispatch({ type: RECORD_START, requestId: "rec-retry" });
const input = document.querySelector("input")!;
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
input.value = "dirty final value";
input.dispatchEvent(new Event("input", { bubbles: true }));
const firstResponse = vi.fn();
expect(dispatch({ type: RECORD_STOP, requestId: "rec-retry" }, firstResponse)).toBe(true);
await vi.waitFor(() =>
expect(firstResponse).toHaveBeenCalledWith({
ok: false,
error: "failed to deliver one or more recorded steps",
}),
);
expect(activeRequestId).toBe("rec-retry");
expect(onStop).not.toHaveBeenCalled();
const retryResponse = vi.fn();
expect(dispatch({ type: RECORD_STOP, requestId: "rec-retry" }, retryResponse)).toBe(true);
await vi.waitFor(() => expect(retryResponse).toHaveBeenCalledWith({ ok: true }));
expect(sendMessage).toHaveBeenCalledTimes(3);
expect(activeRequestId).toBeNull();
expect(onStop).toHaveBeenCalledTimes(1);
});
});
describe("record-capture semantic", () => {
let steps: RecordStepPayload[];
@@ -236,6 +124,19 @@ describe("record-capture semantic", () => {
]);
});
it("does not emit page navigation from a child Document capture", () => {
const originalUrl = location.href;
const capture = startRecordCapture("rec-child", (step) => steps.push(step), {
captureNavigation: false,
});
history.pushState({}, "", "#inside-frame");
expect(steps).toEqual([]);
capture.dispose();
history.replaceState({}, "", originalUrl);
});
it("does not record clicks on anonymous layout divs", () => {
document.body.innerHTML = `
<div id="chrome">page chrome</div>
@@ -0,0 +1,102 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { RecordFramePortMessage } from "@/lib/recording/frame-bridge";
import { RECORD_FRAME_START } from "@/lib/recording/frame-bridge";
import { RECORD_DOCUMENT_ATTRIBUTE } from "@/shared/recording-document-identity";
import { RecordFrameAgent } from "../recording/frame-agent";
class PortListeners<T extends (...args: never[]) => unknown> {
readonly values = new Set<T>();
addListener = (listener: T) => this.values.add(listener);
removeListener = (listener: T) => this.values.delete(listener);
}
function portHarness() {
const onMessage = new PortListeners<(message: unknown) => void>();
const onDisconnect = new PortListeners<() => void>();
const outbound: RecordFramePortMessage[] = [];
const port = {
name: "bsk-record-frame",
onMessage,
onDisconnect,
postMessage(message: RecordFramePortMessage) {
outbound.push(message);
if (message.type === "ready") {
queueMicrotask(() => {
for (const listener of onMessage.values) {
listener({
type: "ready_ack",
requestId: message.requestId,
producerId: message.producerId,
});
}
});
}
},
disconnect: vi.fn(),
} as unknown as chrome.runtime.Port;
return {
port,
outbound,
receive(message: RecordFramePortMessage) {
for (const listener of onMessage.values) listener(message);
},
};
}
describe("RecordFrameAgent", () => {
afterEach(() => {
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
document.body.replaceChildren();
});
it("keeps a failed stop retryable and flushes the final dirty fill", async () => {
const harness = portHarness();
const sendMessage = vi
.fn<(message: { sequence: number }) => Promise<unknown>>()
.mockRejectedValueOnce(new Error("offline"))
.mockRejectedValueOnce(new Error("still offline"))
.mockImplementation(async (message) => ({ ok: true, sequence: message.sequence }));
vi.stubGlobal("chrome", {
runtime: {
connect: vi.fn(() => harness.port),
sendMessage,
},
});
document.body.innerHTML = `<label for="draft">Draft</label><input id="draft" />`;
const agent = new RecordFrameAgent();
await expect(
agent.start({ type: RECORD_FRAME_START, requestId: "rec-1", startedAtMs: 10 }),
).resolves.toEqual({ ok: true });
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true);
const input = document.querySelector("input")!;
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
input.value = "final value";
input.dispatchEvent(new Event("input", { bubbles: true }));
harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-1" });
await vi.waitFor(() =>
expect(harness.outbound).toContainEqual({
type: "stopped",
requestId: "rec-1",
commandId: "stop-1",
ok: false,
error: "failed to deliver one or more recorded steps",
}),
);
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true);
harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-2" });
await vi.waitFor(() =>
expect(harness.outbound).toContainEqual({
type: "stopped",
requestId: "rec-1",
commandId: "stop-2",
ok: true,
}),
);
expect(sendMessage).toHaveBeenCalledTimes(3);
expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(false);
});
});
+30 -131
View File
@@ -12,21 +12,7 @@ import {
hasDirectHoverInteractiveSignal,
hasStrongHoverExpansionSignal,
} from "@/lib/hover-trigger-policy";
import {
isRecordCancelMessage,
isRecordStartMessage,
isRecordStopMessage,
RECORD_CANCEL,
RECORD_START,
RECORD_STEP,
RECORD_STOP,
type RecordCancelMessage,
type RecordStartAck,
type RecordStartMessage,
type RecordStepPayload,
type RecordStopAck,
type RecordStopMessage,
} from "@/lib/record-bridge";
import type { RecordStepPayload } from "@/lib/record-bridge";
import { shouldRecordPress } from "@/lib/recording/draft-policy";
import {
closestHoverSurfaceCandidate,
@@ -36,18 +22,6 @@ import {
isHoverSurfaceCandidateElement,
isLikelyHoverSurfaceOwner,
} from "./record-hover-surface";
import { RecordStepDelivery } from "./record-step-delivery";
const stepDeliveries = new Map<string, RecordStepDelivery>();
const pendingStopFlushes = new Map<string, Promise<RecordStopAck>>();
function deliveryFor(requestId: string): RecordStepDelivery {
const existing = stepDeliveries.get(requestId);
if (existing) return existing;
const created = new RecordStepDelivery(requestId);
stepDeliveries.set(requestId, created);
return created;
}
export interface RecordCaptureController {
dispose(): void;
@@ -385,6 +359,7 @@ function scheduleInputCompletionCommit(
export function startRecordCapture(
_requestId: string,
sendStep: (step: RecordStepPayload) => void,
options: { captureNavigation?: boolean } = {},
): RecordCaptureController {
const emitStep = (step: RecordStepPayload) => {
sendStep({ page_url: location.href, ...step });
@@ -921,26 +896,33 @@ export function startRecordCapture(
document.addEventListener("change", onChange, true);
document.addEventListener("keydown", onKeyDown, true);
const urlObserver = new MutationObserver(() => emitNavigateIfChanged());
urlObserver.observe(document, { subtree: true, childList: true });
const captureNavigation = options.captureNavigation ?? true;
const urlObserver = captureNavigation
? new MutationObserver(() => emitNavigateIfChanged())
: null;
urlObserver?.observe(document, { subtree: true, childList: true });
const onUrlEvent = () => emitNavigateIfChanged();
// Wrap: passing commitFillSession directly would forward the DOM event as
// the `commit` argument and stamp it onto the recorded fill step.
const onPageHide = () => commitFillSession();
window.addEventListener("hashchange", onUrlEvent);
window.addEventListener("popstate", onUrlEvent);
if (captureNavigation) {
window.addEventListener("hashchange", onUrlEvent);
window.addEventListener("popstate", onUrlEvent);
}
window.addEventListener("pagehide", onPageHide);
const originalPushState = history.pushState;
const originalReplaceState = history.replaceState;
history.pushState = function (...args: Parameters<History["pushState"]>) {
originalPushState.apply(this, args);
emitNavigateIfChanged(navigationActionPending ? true : undefined);
};
history.replaceState = function (...args: Parameters<History["replaceState"]>) {
originalReplaceState.apply(this, args);
emitNavigateIfChanged(navigationActionPending ? true : undefined);
};
if (captureNavigation) {
history.pushState = function (...args: Parameters<History["pushState"]>) {
originalPushState.apply(this, args);
emitNavigateIfChanged(navigationActionPending ? true : undefined);
};
history.replaceState = function (...args: Parameters<History["replaceState"]>) {
originalReplaceState.apply(this, args);
emitNavigateIfChanged(navigationActionPending ? true : undefined);
};
}
return {
dispose() {
@@ -954,99 +936,16 @@ export function startRecordCapture(
document.removeEventListener("compositionend", onCompositionEnd, true);
document.removeEventListener("change", onChange, true);
document.removeEventListener("keydown", onKeyDown, true);
urlObserver.disconnect();
window.removeEventListener("hashchange", onUrlEvent);
window.removeEventListener("popstate", onUrlEvent);
urlObserver?.disconnect();
if (captureNavigation) {
window.removeEventListener("hashchange", onUrlEvent);
window.removeEventListener("popstate", onUrlEvent);
}
window.removeEventListener("pagehide", onPageHide);
history.pushState = originalPushState;
history.replaceState = originalReplaceState;
if (captureNavigation) {
history.pushState = originalPushState;
history.replaceState = originalReplaceState;
}
},
};
}
export type RecordContentMessage = RecordStartMessage | RecordStopMessage | RecordCancelMessage;
export function isRecordContentMessage(msg: unknown): msg is RecordContentMessage {
return isRecordStartMessage(msg) || isRecordStopMessage(msg) || isRecordCancelMessage(msg);
}
export function handleRecordContentMessage(
message: RecordContentMessage,
state: {
activeRequestId: string | null;
capture: RecordCaptureController | null;
setActiveRequestId(id: string | null): void;
setCapture(capture: RecordCaptureController | null): void;
onStart(requestId: string, startedAtMs?: number): void;
onStop(): void;
},
sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
): boolean {
if (isRecordStartMessage(message)) {
const delivery = deliveryFor(message.requestId);
state.capture?.dispose();
state.setCapture(
startRecordCapture(message.requestId, (step) => {
delivery.enqueue(step);
}),
);
state.setActiveRequestId(message.requestId);
state.onStart(message.requestId, message.startedAtMs);
sendResponse?.({ ok: true });
return sendResponse !== undefined;
}
if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
// Require an active recording that matches this requestId — otherwise a
// stray STOP/CANCEL (e.g. after teardown) would still run finishStop and
// clear overlay state even though nothing was capturing.
if (!state.activeRequestId || state.activeRequestId !== message.requestId) {
return false;
}
state.capture?.dispose();
state.setCapture(null);
const finishStop = () => {
state.onStop();
state.setActiveRequestId(null);
};
if (isRecordStopMessage(message) && sendResponse) {
const existingFlush = pendingStopFlushes.get(message.requestId);
if (existingFlush) {
void existingFlush.then(sendResponse);
return true;
}
const flush = deliveryFor(message.requestId)
.flush()
.then((succeeded): RecordStopAck => {
if (succeeded) {
stepDeliveries.delete(message.requestId);
finishStop();
}
return succeeded
? { ok: true }
: {
ok: false,
error: "failed to deliver one or more recorded steps",
};
});
pendingStopFlushes.set(message.requestId, flush);
void flush.then(sendResponse).finally(() => {
if (pendingStopFlushes.get(message.requestId) === flush) {
pendingStopFlushes.delete(message.requestId);
}
});
return true;
}
if (isRecordCancelMessage(message)) {
stepDeliveries.delete(message.requestId);
pendingStopFlushes.delete(message.requestId);
}
finishStop();
return false;
}
return false;
}
export { RECORD_CANCEL, RECORD_START, RECORD_STEP, RECORD_STOP };
@@ -0,0 +1,209 @@
import {
markRecordingDocument,
type RecordingDocumentMarker,
waitForDocumentElement,
} from "@/lib/recording/document-marker";
import {
isRecordFrameStartMessage,
RECORD_FRAME_PORT,
RECORD_FRAME_QUERY,
RECORD_FRAME_START,
type RecordFramePortMessage,
type RecordFrameQueryResponse,
type RecordFrameStartMessage,
} from "@/lib/recording/frame-bridge";
import { type RecordCaptureController, startRecordCapture } from "../record-capture";
import { RecordStepDelivery } from "../record-step-delivery";
interface ActiveFrameRecording {
requestId: string;
producerId: string;
port: chrome.runtime.Port;
delivery: RecordStepDelivery;
capture: RecordCaptureController;
marker: RecordingDocumentMarker;
}
const RECORD_FRAME_REGISTER_TIMEOUT_MS = 5_000;
export class RecordFrameAgent {
#active: ActiveFrameRecording | null = null;
#startPromise: Promise<{ ok: true } | { ok: false; error: string }> | null = null;
#disposed = false;
async start(
message: RecordFrameStartMessage,
): Promise<{ ok: true } | { ok: false; error: string }> {
if (this.#active?.requestId === message.requestId) return { ok: true };
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
if (this.#startPromise) {
await this.#startPromise;
return this.start(message);
}
if (this.#active) this.cancel(this.#active.requestId);
const starting = this.#start(message);
this.#startPromise = starting;
try {
return await starting;
} finally {
if (this.#startPromise === starting) this.#startPromise = null;
}
}
cancel(requestId: string): void {
const active = this.#active;
if (!active || active.requestId !== requestId) return;
active.capture.dispose();
active.marker.restore();
active.port.disconnect();
this.#active = null;
}
dispose(): void {
this.#disposed = true;
if (this.#active) this.cancel(this.#active.requestId);
}
async #start(
message: RecordFrameStartMessage,
): Promise<{ ok: true } | { ok: false; error: string }> {
const producerId = crypto.randomUUID();
const root = await waitForDocumentElement();
if (this.#disposed) return { ok: false, error: "recording document is disposed" };
const marker = markRecordingDocument(producerId, root);
const port = chrome.runtime.connect({ name: RECORD_FRAME_PORT });
try {
await registerPort(port, message.requestId, producerId);
} catch {
marker.restore();
port.disconnect();
return { ok: false, error: "failed to register recording document" };
}
if (this.#disposed) {
marker.restore();
port.disconnect();
return { ok: false, error: "recording document is disposed" };
}
const delivery = new RecordStepDelivery(message.requestId, undefined, producerId);
const capture = startRecordCapture(
message.requestId,
(step) => {
marker.ensure();
delivery.enqueue(step);
},
{ captureNavigation: window.top === window },
);
const active: ActiveFrameRecording = {
requestId: message.requestId,
producerId,
port,
delivery,
capture,
marker,
};
this.#active = active;
port.onMessage.addListener((raw: unknown) => {
const command = raw as Partial<RecordFramePortMessage>;
if (command.type === "stop" && command.requestId === active.requestId) {
void this.#stop(active, command.commandId);
} else if (command.type === "cancel" && command.requestId === active.requestId) {
this.cancel(active.requestId);
}
});
port.onDisconnect.addListener(() => {
if (this.#active !== active) return;
active.capture.dispose();
active.marker.restore();
this.#active = null;
});
return { ok: true };
}
async #stop(active: ActiveFrameRecording, commandId: string | undefined): Promise<void> {
if (!commandId || this.#active !== active) return;
active.capture.dispose();
const succeeded = await active.delivery.flush();
try {
active.port.postMessage({
type: "stopped",
requestId: active.requestId,
commandId,
ok: succeeded,
...(succeeded ? {} : { error: "failed to deliver one or more recorded steps" }),
} satisfies RecordFramePortMessage);
} catch {
return;
}
if (!succeeded || this.#active !== active) return;
active.marker.restore();
this.#active = null;
active.port.disconnect();
}
}
function registerPort(
port: chrome.runtime.Port,
requestId: string,
producerId: string,
): Promise<void> {
return new Promise((resolve, reject) => {
const timer = setTimeout(() => {
cleanup();
reject(new Error("recording document registration timed out"));
}, RECORD_FRAME_REGISTER_TIMEOUT_MS);
const cleanup = () => {
clearTimeout(timer);
port.onMessage.removeListener(onMessage);
port.onDisconnect.removeListener(onDisconnect);
};
const onMessage = (raw: unknown) => {
const message = raw as Partial<RecordFramePortMessage>;
if (
message.type !== "ready_ack" ||
message.requestId !== requestId ||
message.producerId !== producerId
) {
return;
}
cleanup();
resolve();
};
const onDisconnect = () => {
cleanup();
reject(new Error("recording document port disconnected"));
};
port.onMessage.addListener(onMessage);
port.onDisconnect.addListener(onDisconnect);
port.postMessage({ type: "ready", requestId, producerId } satisfies RecordFramePortMessage);
});
}
export function attachRecordFrameAgent(): () => void {
const agent = new RecordFrameAgent();
const onMessage = (
message: unknown,
_sender: chrome.runtime.MessageSender,
sendResponse: (response: { ok: true } | { ok: false; error: string }) => void,
) => {
if (!isRecordFrameStartMessage(message)) return false;
void agent.start(message).then(sendResponse);
return true;
};
chrome.runtime.onMessage.addListener(onMessage);
void chrome.runtime
.sendMessage({ type: RECORD_FRAME_QUERY })
.then((response: RecordFrameQueryResponse | undefined) => {
if (!response?.active || !response.requestId || response.startedAtMs === undefined) return;
return agent.start({
type: RECORD_FRAME_START,
requestId: response.requestId,
startedAtMs: response.startedAtMs,
});
})
.catch(() => {});
return () => {
chrome.runtime.onMessage.removeListener(onMessage);
agent.dispose();
};
}
+25 -22
View File
@@ -21,6 +21,7 @@ import {
type OverlayMode,
} from "@/lib/overlay-bridge";
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
import { recordFrameCoordinator } from "@/lib/recording/frame-coordinator";
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup";
import { attachSessionEventHandler } from "@/session-manager/event-handler";
@@ -38,6 +39,7 @@ import {
attachRecordFinishListener,
attachRecordQueryListener,
attachRecordStepListener,
type RecordRuntimeDeps,
} from "@/tools/record";
import { detectBrowserMeta } from "@/transport/handshake";
import type { Transport } from "@/transport/transport";
@@ -145,10 +147,33 @@ export default defineBackground(() => {
void sessionsLive.syncFromManager();
},
});
const recordDeps = {
tabsApi: chrome.tabs,
cdp,
frameCoordinator: recordFrameCoordinator,
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
chrome.tabs.sendMessage(tabId, msg),
bypassOverlay: async (tabId: number, enabled: boolean) => {
try {
await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
} satisfies RecordRuntimeDeps;
recordFrameCoordinator.attach();
attachRecordStepListener(recordDeps);
attachRecordFinishListener(recordDeps);
attachRecordQueryListener(recordDeps);
const dispatcher = new ToolDispatcher({
transport,
sessions,
cdp,
recording: recordDeps,
onSessionsChanged: onOverlaySessionStateChanged,
onBrowserControlResumed,
approveBorrow: (ctx) =>
@@ -170,28 +195,6 @@ export default defineBackground(() => {
}),
});
dispatcher.start();
const recordDeps = {
tabsApi: chrome.tabs,
cdp,
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
chrome.tabs.sendMessage(tabId, msg),
bypassOverlay: async (tabId: number, enabled: boolean) => {
try {
await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
};
// Message listeners stay up (cheap; fire only for record message types).
// Tab / webNavigation observation attaches lazily while a recording is
// active — see ensureBrowserObservationListeners in tools/record.ts.
attachRecordStepListener(recordDeps);
attachRecordFinishListener(recordDeps);
attachRecordQueryListener(recordDeps);
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
attachBorrowNotificationClickHandler({
onClicked: chrome.notifications.onClicked,
+34 -45
View File
@@ -11,11 +11,6 @@ import { createHelpRequestData } from "@/content/help-request";
import overlayCss from "@/content/overlay.css?inline";
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
import { RecordOverlay } from "@/content/RecordOverlay";
import {
handleRecordContentMessage,
isRecordContentMessage,
type RecordCaptureController,
} from "@/content/record-capture";
import {
type CaptureSuppressAck,
type CaptureSuppressMessage,
@@ -45,11 +40,17 @@ import {
} from "@/lib/overlay-bridge";
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
import {
isRecordCancelMessage,
isRecordStartMessage,
isRecordStopMessage,
RECORD_FINISH,
RECORD_QUERY,
type RecordCancelMessage,
type RecordQueryResponse,
type RecordStartAck,
type RecordStartMessage,
type RecordStopAck,
type RecordStopMessage,
} from "@/lib/record-bridge";
import type {
BorrowCancelMessage,
@@ -69,7 +70,6 @@ export default defineContentScript({
if (window.top !== window) return;
const overlays = new OverlayController();
let recordCapture: RecordCaptureController | null = null;
let activeRecordRequestId: string | null = null;
let reactRoot: ReactDOM.Root | null = null;
let overlayHost: HTMLElement | null = null;
@@ -228,8 +228,6 @@ export default defineContentScript({
if (previousHelp) {
void sendHelpFinish(previousHelp.id, "cancelled");
}
recordCapture?.dispose();
recordCapture = null;
activeRecordRequestId = null;
renderAll();
}
@@ -259,6 +257,9 @@ export default defineContentScript({
| HelpRequestMessage
| HelpCancelMessage
| CaptureSuppressMessage
| RecordStartMessage
| RecordStopMessage
| RecordCancelMessage
| OverlayAgentOverlayResetMessage
| OverlayAgentStateMessage
| OverlayAutomationBypassMessage,
@@ -269,41 +270,32 @@ export default defineContentScript({
return captureSuppress.handleMessage(message, sendResponse);
}
if (isRecordContentMessage(message)) {
const needsAsync = handleRecordContentMessage(
message,
{
activeRequestId: activeRecordRequestId,
capture: recordCapture,
setActiveRequestId: (id) => {
activeRecordRequestId = id;
},
setCapture: (capture) => {
recordCapture = capture;
},
onStart: (requestId, startedAtMs) => {
overlays.setAgentRecordRequest({
id: requestId,
...(typeof startedAtMs === "number" ? { startedAtMs } : {}),
onFinish: () => {
void chrome.runtime.sendMessage({
type: RECORD_FINISH,
requestId,
});
},
});
renderAll();
},
onStop: () => {
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
renderAll();
},
if (isRecordStartMessage(message)) {
activeRecordRequestId = message.requestId;
overlays.setAgentRecordRequest({
id: message.requestId,
...(typeof message.startedAtMs === "number" ? { startedAtMs: message.startedAtMs } : {}),
onFinish: () => {
void chrome.runtime.sendMessage({
type: RECORD_FINISH,
requestId: message.requestId,
});
},
sendResponse as unknown as
| ((response: RecordStartAck | RecordStopAck) => void)
| undefined,
);
return needsAsync;
});
renderAll();
(sendResponse as unknown as (response: RecordStartAck) => void)({ ok: true });
return false;
}
if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
if (activeRecordRequestId !== message.requestId) return false;
overlays.clearAgentRecordRequest(message.requestId);
activeRecordRequestId = null;
renderAll();
if (isRecordStopMessage(message)) {
(sendResponse as unknown as (response: RecordStopAck) => void)({ ok: true });
}
return false;
}
if (
@@ -526,9 +518,6 @@ export default defineContentScript({
chrome.runtime.onMessage.removeListener(onMessage);
chrome.storage.onChanged.removeListener(onStorageChange);
window.removeEventListener("pageshow", onPageShow);
// Restore history hooks / remove capture listeners before the CS unloads.
recordCapture?.dispose();
recordCapture = null;
activeRecordRequestId = null;
});
},
@@ -0,0 +1,14 @@
import { attachRecordFrameAgent } from "@/content/recording/frame-agent";
export default defineContentScript({
matches: ["<all_urls>"],
runAt: "document_start",
allFrames: true,
matchAboutBlank: true,
matchOriginAsFallback: true,
main(ctx) {
const dispose = attachRecordFrameAgent();
ctx.onInvalidated(dispose);
},
});
@@ -0,0 +1,213 @@
import { beforeEach, describe, expect, it, vi } from "vitest";
import { RECORD_FRAME_PORT, RECORD_FRAME_QUERY } from "../recording/frame-bridge";
import { RecordFrameCoordinator } from "../recording/frame-coordinator";
class ListenerSet<T extends (...args: never[]) => unknown> {
readonly listeners = new Set<T>();
addListener = (listener: T) => this.listeners.add(listener);
removeListener = (listener: T) => this.listeners.delete(listener);
}
function fakePort(sender: chrome.runtime.MessageSender) {
const messages: unknown[] = [];
const onMessage = new ListenerSet<(message: unknown) => void>();
const onDisconnect = new ListenerSet<() => void>();
return {
port: {
name: RECORD_FRAME_PORT,
sender,
onMessage,
onDisconnect,
postMessage: vi.fn((message: unknown) => messages.push(message)),
disconnect: vi.fn(),
} as unknown as chrome.runtime.Port,
messages,
receive(message: unknown) {
for (const listener of onMessage.listeners) listener(message);
},
};
}
describe("RecordFrameCoordinator", () => {
const onConnect = new ListenerSet<(port: chrome.runtime.Port) => void>();
const onMessage = new ListenerSet<
(
message: unknown,
sender: chrome.runtime.MessageSender,
sendResponse: (response: unknown) => void,
) => boolean
>();
beforeEach(() => {
onConnect.listeners.clear();
onMessage.listeners.clear();
vi.stubGlobal("chrome", {
runtime: { onConnect, onMessage },
});
});
it("answers child-document queries for the initial tab before it is armed", () => {
const coordinator = new RecordFrameCoordinator({
getAllFrames: async () => [],
sendToDocument: async () => ({ ok: true }),
});
coordinator.attach();
coordinator.begin("rec-1", 10, 3);
const query = (tabId: number) => {
let response: unknown;
for (const listener of onMessage.listeners) {
listener(
{ type: RECORD_FRAME_QUERY },
{
tab: { id: tabId },
frameId: 7,
documentId: "child-document",
} as chrome.runtime.MessageSender,
(value) => {
response = value;
},
);
}
return response;
};
expect(query(3)).toEqual({ active: true, requestId: "rec-1", startedAtMs: 10 });
expect(query(4)).toEqual({ active: false });
});
it("starts a child Document that appears after the initial frame snapshot", async () => {
let notifyFrameNavigation:
| ((frame: {
tabId: number;
frameId: number;
documentId?: string;
lifecycle: "committed" | "completed";
}) => void)
| undefined;
const sendToDocument = vi.fn(async () => ({ ok: true }));
const coordinator = new RecordFrameCoordinator({
getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }],
sendToDocument,
subscribeFrameNavigation(listener) {
notifyFrameNavigation = listener;
return () => {};
},
});
coordinator.attach();
const onDocumentReady = vi.fn();
coordinator.begin("rec-1", 10, 3, onDocumentReady);
await coordinator.armTab("rec-1", 3);
sendToDocument.mockClear();
notifyFrameNavigation?.({
tabId: 3,
frameId: 7,
documentId: "late-child",
lifecycle: "completed",
});
await vi.waitFor(() => {
expect(sendToDocument).toHaveBeenCalledWith(
3,
{ type: "bsk-record-frame-start", requestId: "rec-1", startedAtMs: 10 },
{ documentId: "late-child" },
);
});
expect(onDocumentReady).not.toHaveBeenCalled();
const child = fakePort({
tab: { id: 3 },
frameId: 7,
documentId: "late-child",
} as chrome.runtime.MessageSender);
for (const listener of onConnect.listeners) listener(child.port);
child.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
notifyFrameNavigation?.({
tabId: 3,
frameId: 7,
documentId: "late-child",
lifecycle: "completed",
});
await vi.waitFor(() => {
expect(onDocumentReady).toHaveBeenCalledWith({
tabId: 3,
documentId: "late-child",
browserFrameId: 7,
producerId: "producer-1",
});
});
});
it("binds a producer to its sender Document and keeps final steps valid while stopping", async () => {
const sendToDocument = vi.fn(async () => ({ ok: true }));
const coordinator = new RecordFrameCoordinator({
getAllFrames: async () => [
{ frameId: 0, documentId: "top-document" },
{ frameId: 7, documentId: "child-document" },
],
sendToDocument,
});
coordinator.attach();
coordinator.begin("rec-1", 10, 3);
await expect(coordinator.armTab("rec-1", 3)).resolves.toBe(true);
const sender = {
tab: { id: 3 },
frameId: 7,
documentId: "child-document",
} as chrome.runtime.MessageSender;
const frame = fakePort(sender);
for (const listener of onConnect.listeners) listener(frame.port);
frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toEqual({
tabId: 3,
documentId: "child-document",
browserFrameId: 7,
producerId: "producer-1",
});
const stopping = coordinator.stop("rec-1");
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).not.toBeNull();
const stop = frame.messages.find(
(message): message is { type: "stop"; commandId: string } =>
typeof message === "object" &&
message !== null &&
(message as { type?: string }).type === "stop",
);
expect(stop).toBeDefined();
frame.receive({
type: "stopped",
requestId: "rec-1",
commandId: stop!.commandId,
ok: true,
});
await expect(stopping).resolves.toBe(true);
expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toBeNull();
});
it("rejects a producer used from another Document", async () => {
const coordinator = new RecordFrameCoordinator({
getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }],
sendToDocument: async () => ({ ok: true }),
});
coordinator.attach();
coordinator.begin("rec-1", 10, 3);
await coordinator.armTab("rec-1", 3);
const sender = {
tab: { id: 3 },
frameId: 0,
documentId: "top-document",
} as chrome.runtime.MessageSender;
const frame = fakePort(sender);
for (const listener of onConnect.listeners) listener(frame.port);
frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" });
expect(
coordinator.sourceFor("rec-1", "producer-1", {
...sender,
documentId: "other-document",
}),
).toBeNull();
});
});
@@ -0,0 +1,70 @@
import { describe, expect, it } from "vitest";
import {
RECORD_DOCUMENT_ATTRIBUTE,
recordingDocumentMarkerValue,
} from "@/shared/recording-document-identity";
import { markRecordingDocument } from "../recording/document-marker";
import { ObservationNodeIndex } from "../recording/observation-capture";
describe("recording document marker", () => {
it("rejects a non-random Document identity", () => {
expect(() => markRecordingDocument("user@example.com")).toThrow(
"recording Document identity must be a random UUID",
);
});
it("restores the Document attribute exactly", () => {
document.documentElement.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, "page-value");
const producerId = "123e4567-e89b-42d3-a456-426614174000";
const marker = markRecordingDocument(producerId);
expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(
recordingDocumentMarkerValue(producerId),
);
marker.restore();
expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe("page-value");
document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
});
it("indexes the CDP scope from the safe frame identity", () => {
const index = new ObservationNodeIndex({
rootFrameId: "root",
frames: [
{
frameId: "child",
target: { tabId: 3, sessionId: "oopif-session" },
recordingDocumentId: "producer-1",
},
],
matchNodes: [],
refs: [],
});
expect(index.documentScope("producer-1")).toEqual({
frameId: "child",
target: { tabId: 3, sessionId: "oopif-session" },
});
});
it("fails closed when a recording identity maps to multiple Documents", () => {
const index = new ObservationNodeIndex({
rootFrameId: "root",
frames: [
{
frameId: "left",
target: { tabId: 3, sessionId: "left-session" },
recordingDocumentId: "duplicate",
},
{
frameId: "right",
target: { tabId: 3, sessionId: "right-session" },
recordingDocumentId: "duplicate",
},
],
matchNodes: [],
refs: [],
});
expect(index.documentScope("duplicate")).toBeUndefined();
});
});
@@ -2,20 +2,27 @@ import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
const captureRecordingObservation = vi.hoisted(() => vi.fn());
const waitForDocumentSettled = vi.hoisted(() => vi.fn(async () => "quiet" as const));
vi.mock("../recording/observation-capture", async (importOriginal) => {
const actual = await importOriginal<typeof import("../recording/observation-capture")>();
return { ...actual, captureRecordingObservation };
});
vi.mock("../recording/document-settle", async (importOriginal) => {
const actual = await importOriginal<typeof import("../recording/document-settle")>();
return { ...actual, waitForDocumentSettled };
});
import { ObservationNodeIndex } from "../recording/observation-capture";
import { RecordingObservationRuntime } from "../recording/recording-runtime";
import type { RecordingDraftStep } from "../recording/types";
function observation() {
function observation(url = "https://example.com/") {
return {
rootFrameId: "root",
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }),
url: "https://example.com/",
url,
title: "Example",
vomText: '@vom 1\nRootWebArea "Example"',
truncated: false,
@@ -30,7 +37,10 @@ function runtime(): RecordingObservationRuntime {
}
describe("RecordingObservationRuntime", () => {
afterEach(() => captureRecordingObservation.mockReset());
afterEach(() => {
captureRecordingObservation.mockReset();
waitForDocumentSettled.mockClear();
});
it("shares one initial capture and includes it in flush", async () => {
let release!: (value: ReturnType<typeof observation>) => void;
@@ -130,4 +140,150 @@ describe("RecordingObservationRuntime", () => {
expect(dumped).not.toContain("backendNodeId");
expect(dumped).not.toContain("ObservationNodeIndex");
});
it("captures a tab transition without sharing observation cursors", async () => {
captureRecordingObservation
.mockResolvedValueOnce(observation("https://example.com/first"))
.mockResolvedValueOnce(observation("https://example.com/second"));
const recording = runtime();
await recording.captureInitial(4);
const transition = await recording.captureTabTransition(4, 5);
await recording.captureInitial(4);
await recording.captureInitial(5);
expect(transition).toEqual({
preStateId: "s1",
postStateId: "s2",
targetUrl: "https://example.com/second",
});
expect(captureRecordingObservation.mock.calls.map(([input]) => input.tabId)).toEqual([4, 5]);
});
it("binds an iframe draft to its marked CDP Document scope", async () => {
captureRecordingObservation.mockResolvedValueOnce({
...observation(),
index: new ObservationNodeIndex({
rootFrameId: "root",
frames: [
{
frameId: "child",
target: { tabId: 7, sessionId: "oopif-session" },
recordingDocumentId: "producer-1",
},
],
matchNodes: [
{
backendNodeId: 42,
frameId: "child",
tag: "button",
rect: { x: 410, y: 20, w: 100, h: 30 },
localRect: { x: 10, y: 20, w: 100, h: 30 },
},
],
refs: [{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
}),
});
const recording = runtime();
const drafts: RecordingDraftStep[] = [
{
op: "click" as const,
captureTarget: { tag: "button", role: "button", name: "Save" },
targetHint: {
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
},
];
await recording.captureInitial(7);
await recording.processDraft(7, drafts, 0, "producer-1");
recording.cancel();
const draft = drafts[0];
expect(draft?.op).toBe("click");
if (!draft || draft.op !== "click") throw new Error("expected click draft");
expect(draft.targetHint).toMatchObject({
frameId: "child",
geometrySpace: "local",
});
expect(draft.matchedTarget?.ref).toBe("e1");
});
it("refreshes a late iframe before matching its first action", async () => {
const iframeObservation = {
...observation(),
index: new ObservationNodeIndex({
rootFrameId: "root",
frames: [
{
frameId: "child",
target: { tabId: 7, sessionId: "oopif-session" },
recordingDocumentId: "producer-1",
},
],
matchNodes: [
{
backendNodeId: 42,
frameId: "child",
tag: "button",
rect: { x: 410, y: 20, w: 100, h: 30 },
localRect: { x: 10, y: 20, w: 100, h: 30 },
},
],
refs: [
{
ref: "e1",
backendNodeId: 42,
frameId: "child",
role: "button",
name: "表格视图",
line: 1,
},
],
}),
};
captureRecordingObservation
.mockResolvedValueOnce(observation())
.mockResolvedValueOnce(iframeObservation)
.mockResolvedValueOnce(iframeObservation);
const recording = runtime();
await recording.captureInitial(7);
await recording.refreshDocument(7, "producer-1");
const drafts: RecordingDraftStep[] = [
{
op: "click",
captureTarget: { tag: "button", role: "button", name: "表格视图" },
targetHint: {
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
},
];
await recording.processDraft(7, drafts, 0, "producer-1");
recording.cancel();
expect(captureRecordingObservation).toHaveBeenCalledTimes(3);
expect(waitForDocumentSettled).toHaveBeenCalledWith(
expect.anything(),
{ frameId: "child", target: { tabId: 7, sessionId: "oopif-session" } },
{ signal: expect.any(AbortSignal) },
);
const draft = drafts[0];
expect(draft?.op).toBe("click");
if (!draft || draft.op !== "click") throw new Error("expected click draft");
expect(draft.matchedTarget).toMatchObject({ ref: "e1", name: "表格视图" });
});
it("does not poison final flush when a frame readiness refresh fails", async () => {
captureRecordingObservation
.mockResolvedValueOnce(observation())
.mockRejectedValueOnce(new Error("child Document replaced"));
const recording = runtime();
await recording.captureInitial(7);
await expect(recording.refreshDocument(7, "producer-1")).rejects.toThrow(
"child Document replaced",
);
await expect(recording.flush()).resolves.toBeUndefined();
});
});
@@ -3,7 +3,7 @@ import { appendRecordedPayload, observeRecordedNavigation } from "../recording/s
describe("recording-step-buffer", () => {
it("stores semantic click without summary", () => {
const buffer = { steps: [], pendingNavigation: false };
const buffer = { steps: [], navigation: { pendingNavigation: false } };
appendRecordedPayload(buffer, {
op: "click",
target: { tag: "button", role: "button", name: "发布" },
@@ -15,11 +15,11 @@ describe("recording-step-buffer", () => {
captureTarget: { tag: "button", role: "button", name: "发布" },
},
]);
expect(buffer.pendingNavigation).toBe(true);
expect(buffer.navigation.pendingNavigation).toBe(true);
});
it("keeps the hovered element description as a capture fallback", () => {
const buffer = { steps: [], pendingNavigation: false };
const buffer = { steps: [], navigation: { pendingNavigation: false } };
appendRecordedPayload(
buffer,
{
@@ -59,9 +59,11 @@ describe("recording-step-buffer", () => {
captureTarget: { tag: "button", role: "button", name: "发布" },
},
],
currentUrl: "https://example.com/a",
pendingNavigation: true,
pendingNavigationDeadline: Date.now() + 5_000,
navigation: {
currentUrl: "https://example.com/a",
pendingNavigation: true,
pendingNavigationDeadline: Date.now() + 5_000,
},
};
observeRecordedNavigation(buffer, "https://example.com/b", true);
expect(buffer.steps).toEqual([
@@ -83,9 +85,11 @@ describe("recording-step-buffer", () => {
values: ["tech"],
},
],
currentUrl: "https://example.com/list",
pendingNavigation: true,
pendingNavigationDeadline: Date.now() + 5_000,
navigation: {
currentUrl: "https://example.com/list",
pendingNavigation: true,
pendingNavigationDeadline: Date.now() + 5_000,
},
};
observeRecordedNavigation(buffer, "https://example.com/list?cat=tech", true);
expect(buffer.steps).toEqual([
@@ -101,8 +105,7 @@ describe("recording-step-buffer", () => {
it("emits navigate for uncaused URL changes", () => {
const buffer = {
steps: [],
currentUrl: "https://example.com/a",
pendingNavigation: false,
navigation: { currentUrl: "https://example.com/a", pendingNavigation: false },
};
const result = observeRecordedNavigation(buffer, "https://example.com/b", false);
expect(result).toEqual({ kind: "appended", index: 0 });
@@ -115,8 +118,10 @@ describe("recording-step-buffer", () => {
it("asks the recorder to coalesce redirect hops instead of emitting each one", () => {
const buffer = {
steps: [],
currentUrl: "https://passport.example/login",
pendingNavigation: false,
navigation: {
currentUrl: "https://passport.example/login",
pendingNavigation: false,
},
};
const hop1 = observeRecordedNavigation(
buffer,
@@ -139,14 +144,16 @@ describe("recording-step-buffer", () => {
url: "https://app.example/dashboard",
});
expect(buffer.steps).toEqual([]);
expect(buffer.currentUrl).toBe("https://app.example/dashboard");
expect(buffer.navigation.currentUrl).toBe("https://app.example/dashboard");
});
it("does not reuse redirect metadata for a later content-observed URL change", () => {
const buffer = {
steps: [],
currentUrl: "https://example.com/redirect",
pendingNavigation: false,
navigation: {
currentUrl: "https://example.com/redirect",
pendingNavigation: false,
},
};
const result = observeRecordedNavigation(buffer, "https://example.com/spa");
@@ -0,0 +1,39 @@
import { describe, expect, it } from "vitest";
import { RecordingTabCoordinator } from "../recording/tab-coordinator";
describe("RecordingTabCoordinator", () => {
it("keeps navigation state isolated per tab", () => {
const tabs = new RecordingTabCoordinator(4, "https://example.com/first");
const first = tabs.navigation(4);
first.pendingNavigation = true;
const second = tabs.navigation(5, "https://example.com/second");
expect(second).toEqual({
currentUrl: "https://example.com/second",
pendingNavigation: false,
});
expect(tabs.navigation(4)).toBe(first);
expect(tabs.navigation(4).pendingNavigation).toBe(true);
});
it("invalidates an earlier activation when a newer tab becomes active", () => {
const tabs = new RecordingTabCoordinator(4);
const second = tabs.noteActivation(5);
const third = tabs.noteActivation(6);
expect(tabs.isLatest(second)).toBe(false);
expect(tabs.isLatest(third)).toBe(true);
tabs.commit(6);
expect(tabs.currentTabId).toBe(6);
});
it("commits the freshly observed URL when returning to an existing tab", () => {
const tabs = new RecordingTabCoordinator(4);
tabs.navigation(5, "https://example.com/old");
tabs.commit(5, "https://example.com/new");
expect(tabs.currentTabId).toBe(5);
expect(tabs.navigation(5).currentUrl).toBe("https://example.com/new");
});
});
@@ -69,6 +69,23 @@ describe("matchObservationTarget", () => {
expect(target.ref).toBe("e2");
});
it("matches iframe capture geometry in the child viewport coordinate space", () => {
const child = node(42, "child", 410);
child.localRect = { x: 10, y: 20, w: 100, h: 30 };
const target = matchObservationTarget({
observation: observation(
[child],
[{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }],
),
hint: {
frameId: "child",
geometrySpace: "local",
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
});
expect(target.ref).toBe("e1");
});
it("restricts a missing frame hint to the root frame", () => {
const target = matchObservationTarget({
observation: observation(
@@ -81,6 +98,21 @@ describe("matchObservationTarget", () => {
expect(target).toEqual({ name: "发布", unmatched: true });
});
it("fails closed when the source Document has no VOM frame mapping", () => {
const target = matchObservationTarget({
observation: observation(
[node(42, "root")],
[{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }],
),
hint: {
frameId: null,
geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" },
},
fallback: { tag: "button", role: "button", name: "发布" },
});
expect(target).toEqual({ role: "button", name: "发布", unmatched: true });
});
it("returns unmatched for ambiguous geometry", () => {
const target = matchObservationTarget({
observation: observation(
@@ -27,6 +27,17 @@ describe("shouldRecordPress", () => {
});
describe("reduceTraceSteps", () => {
it("does not expose internal tab transitions in trace v2", () => {
const trace = reduceTraceSteps([
{
op: "switch_tab",
preStateId: "s1",
postStateId: "s2",
},
]);
expect(trace.steps).toEqual([]);
});
it("builds steps with pages dictionary and page id references", () => {
const drafts: RecordingDraftStep[] = [
{
@@ -160,7 +171,7 @@ describe("reduceTraceSteps", () => {
});
});
it("keeps hover steps before menu clicks", () => {
it("drops hover steps, which only exist in trace v3", () => {
const { steps } = reduceTraceSteps(
[
{
@@ -176,12 +187,7 @@ describe("reduceTraceSteps", () => {
],
"https://example.com/app",
);
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
expect(steps[0]).toMatchObject({
op: "hover",
page: "p1",
target: { name: "Account" },
});
expect(steps.map((s) => s.op)).toEqual(["click"]);
});
it("resolveTraceStartUrl prefers explicit start URL", () => {
@@ -33,6 +33,49 @@ describe("trace reducer v3", () => {
expect(JSON.stringify(reduced.steps)).not.toContain("private-account-id");
});
it("emits tab transitions only for callers that advertised support", () => {
const drafts: RecordingDraftStep[] = [
{
op: "switch_tab",
preStateId: "s1",
postStateId: "s2",
},
];
expect(reduceTraceStepsV3(drafts).steps).toEqual([]);
expect(reduceTraceStepsV3(drafts, { includeTabSwitches: true }).steps).toEqual([
{ op: "switch_tab", id: 1, state: "s1", result: { state: "s2" } },
]);
});
it("keeps hover steps before menu clicks", () => {
const drafts: RecordingDraftStep[] = [
{
op: "hover",
captureTarget: { tag: "span", role: "button", name: "Account" },
preStateId: "s1",
postStateId: "s1",
},
{
op: "click",
captureTarget: { tag: "a", role: "link", name: "Profile" },
preStateId: "s1",
postStateId: "s1",
},
];
expect(reduceTraceStepsV3(drafts).steps.map((s) => s.op)).toEqual(["hover", "click"]);
});
it("does not export a tab transition without both observation endpoints", () => {
const sourceOnly: RecordingDraftStep = { op: "switch_tab", preStateId: "s1" };
const targetOnly: RecordingDraftStep = { op: "switch_tab", postStateId: "s2" };
expect(
reduceTraceStepsV3([sourceOnly, targetOnly], { includeTabSwitches: true }).steps,
).toEqual([]);
});
it("collapses redirect hops while retaining draft-to-step identity", () => {
const drafts: RecordingDraftStep[] = [
{ op: "navigate", url: "https://example.com/start", preStateId: "s1", postStateId: "s2" },
@@ -0,0 +1,39 @@
import {
RECORD_DOCUMENT_ATTRIBUTE,
recordingDocumentMarkerValue,
} from "@/shared/recording-document-identity";
export interface RecordingDocumentMarker {
restore(): void;
ensure(): void;
}
export function waitForDocumentElement(): Promise<HTMLElement> {
if (document.documentElement) return Promise.resolve(document.documentElement);
return new Promise((resolve) => {
const observer = new MutationObserver(() => {
if (!document.documentElement) return;
observer.disconnect();
resolve(document.documentElement);
});
observer.observe(document, { childList: true });
});
}
export function markRecordingDocument(
producerId: string,
root: HTMLElement = document.documentElement,
): RecordingDocumentMarker {
const previous = root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE);
const markerValue = recordingDocumentMarkerValue(producerId);
const ensure = () => root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, markerValue);
ensure();
return {
ensure,
restore() {
if (root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE) !== markerValue) return;
if (previous === null) root.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE);
else root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, previous);
},
};
}
@@ -0,0 +1,65 @@
export const RECORD_FRAME_PORT = "bsk-record-frame";
export const RECORD_FRAME_QUERY = "bsk-record-frame-query";
export const RECORD_FRAME_START = "bsk-record-frame-start";
export interface RecordFrameQueryMessage {
type: typeof RECORD_FRAME_QUERY;
}
export interface RecordFrameQueryResponse {
active: boolean;
requestId?: string;
startedAtMs?: number;
}
export interface RecordFrameStartMessage {
type: typeof RECORD_FRAME_START;
requestId: string;
startedAtMs: number;
}
export type RecordFramePortMessage =
| {
type: "ready";
requestId: string;
producerId: string;
}
| {
type: "ready_ack";
requestId: string;
producerId: string;
}
| {
type: "stop";
requestId: string;
commandId: string;
}
| {
type: "cancel";
requestId: string;
}
| {
type: "stopped";
requestId: string;
commandId: string;
ok: boolean;
error?: string;
};
export function isRecordFrameQueryMessage(value: unknown): value is RecordFrameQueryMessage {
return (
typeof value === "object" &&
value !== null &&
(value as { type?: unknown }).type === RECORD_FRAME_QUERY
);
}
export function isRecordFrameStartMessage(value: unknown): value is RecordFrameStartMessage {
if (typeof value !== "object" || value === null) return false;
const message = value as Partial<RecordFrameStartMessage>;
return (
message.type === RECORD_FRAME_START &&
typeof message.requestId === "string" &&
typeof message.startedAtMs === "number"
);
}
@@ -0,0 +1,399 @@
import {
isRecordFrameQueryMessage,
RECORD_FRAME_PORT,
RECORD_FRAME_START,
type RecordFramePortMessage,
type RecordFrameQueryResponse,
type RecordFrameStartMessage,
} from "./frame-bridge";
export interface RecordingCaptureScope {
tabId: number;
documentId: string;
browserFrameId: number;
producerId: string;
}
interface ArmedRecording {
requestId: string;
startedAtMs: number;
tabIds: Set<number>;
agents: Map<string, FrameAgent>;
onDocumentReady?: (scope: RecordingCaptureScope) => void | Promise<void>;
finishing: boolean;
}
interface FrameAgent extends RecordingCaptureScope {
requestId: string;
port: chrome.runtime.Port;
stopWaiters: Map<string, (ok: boolean) => void>;
}
interface BrowserFrame {
frameId: number;
documentId?: string;
}
interface BrowserFrameNavigation extends BrowserFrame {
tabId: number;
lifecycle: "committed" | "completed";
}
const RECORD_FRAME_STOP_TIMEOUT_MS = 5_000;
export interface RecordFrameCoordinatorDeps {
getAllFrames(tabId: number): Promise<BrowserFrame[]>;
sendToDocument(
tabId: number,
message: RecordFrameStartMessage,
target: { documentId?: string; frameId?: number },
): Promise<unknown>;
subscribeFrameNavigation?(listener: (frame: BrowserFrameNavigation) => void): () => void;
}
function documentKey(tabId: number, documentId: string): string {
return `${tabId}:${documentId}`;
}
function senderAddress(
sender: chrome.runtime.MessageSender,
): { tabId: number; documentId: string; browserFrameId: number } | null {
const tabId = sender.tab?.id;
const documentId = sender.documentId;
const browserFrameId = sender.frameId;
if (
typeof tabId !== "number" ||
typeof documentId !== "string" ||
typeof browserFrameId !== "number"
) {
return null;
}
return { tabId, documentId, browserFrameId };
}
function defaultDeps(): RecordFrameCoordinatorDeps {
return {
async getAllFrames(tabId) {
return (await chrome.webNavigation.getAllFrames({ tabId })) ?? [];
},
sendToDocument(tabId, message, target) {
return chrome.tabs.sendMessage(tabId, message, target);
},
subscribeFrameNavigation(listener) {
const onCommitted = (details: chrome.webNavigation.WebNavigationTransitionCallbackDetails) =>
listener({ ...details, lifecycle: "committed" });
const onCompleted = (details: chrome.webNavigation.WebNavigationFramedCallbackDetails) =>
listener({ ...details, lifecycle: "completed" });
chrome.webNavigation.onCommitted.addListener(onCommitted);
chrome.webNavigation.onCompleted.addListener(onCompleted);
return () => {
chrome.webNavigation.onCommitted.removeListener(onCommitted);
chrome.webNavigation.onCompleted.removeListener(onCompleted);
};
},
};
}
export class RecordFrameCoordinator {
readonly #deps: RecordFrameCoordinatorDeps;
readonly #recordings = new Map<string, ArmedRecording>();
#attached = false;
#detachFrameNavigation: (() => void) | null = null;
constructor(deps: RecordFrameCoordinatorDeps = defaultDeps()) {
this.#deps = deps;
}
attach(): () => void {
if (this.#attached) return () => {};
this.#attached = true;
chrome.runtime.onConnect.addListener(this.#onConnect);
chrome.runtime.onMessage.addListener(this.#onMessage);
this.#detachFrameNavigation =
this.#deps.subscribeFrameNavigation?.(this.#onFrameNavigation) ?? null;
return () => {
if (!this.#attached) return;
this.#attached = false;
chrome.runtime.onConnect.removeListener(this.#onConnect);
chrome.runtime.onMessage.removeListener(this.#onMessage);
this.#detachFrameNavigation?.();
this.#detachFrameNavigation = null;
for (const recording of this.#recordings.values()) this.#cancelAgents(recording);
this.#recordings.clear();
};
}
begin(
requestId: string,
startedAtMs: number,
initialTabId: number,
onDocumentReady?: (scope: RecordingCaptureScope) => void | Promise<void>,
): void {
const previous = this.#recordings.get(requestId);
if (previous) this.#cancelAgents(previous);
this.#recordings.set(requestId, {
requestId,
startedAtMs,
// Register the initial tab before navigation begins. A newly-created
// child document can query at document_start before armTab's frame
// snapshot sees it, especially when Chromium promotes it to an OOPIF.
tabIds: new Set([initialTabId]),
agents: new Map(),
...(onDocumentReady ? { onDocumentReady } : {}),
finishing: false,
});
}
async armTab(requestId: string, tabId: number): Promise<boolean> {
const recording = this.#recordings.get(requestId);
if (!recording || recording.finishing) return false;
recording.tabIds.add(tabId);
let frames: BrowserFrame[];
try {
frames = await this.#deps.getAllFrames(tabId);
} catch {
frames = [{ frameId: 0 }];
}
if (!frames.some((frame) => frame.frameId === 0)) frames.unshift({ frameId: 0 });
const results = await Promise.all(
frames.map(async (frame) => {
const started = await this.#startFrame(recording, tabId, frame);
return { frameId: frame.frameId, started };
}),
);
return results.some((result) => result.frameId === 0 && result.started);
}
sourceFor(
requestId: string,
producerId: string,
sender: chrome.runtime.MessageSender,
): RecordingCaptureScope | null {
const address = senderAddress(sender);
const recording = this.#recordings.get(requestId);
if (!address || !recording) return null;
const agent = recording.agents.get(documentKey(address.tabId, address.documentId));
if (!agent || agent.producerId !== producerId) return null;
return {
tabId: agent.tabId,
documentId: agent.documentId,
browserFrameId: agent.browserFrameId,
producerId: agent.producerId,
};
}
async stop(requestId: string): Promise<boolean> {
const recording = this.#recordings.get(requestId);
if (!recording) return true;
recording.finishing = true;
const commandId = crypto.randomUUID();
const results = await Promise.all(
[...recording.agents.values()].map((agent) => this.#stopAgent(agent, requestId, commandId)),
);
const succeeded = results.every(Boolean);
if (succeeded) this.#recordings.delete(requestId);
else recording.finishing = false;
return succeeded;
}
cancel(requestId: string): void {
const recording = this.#recordings.get(requestId);
if (!recording) return;
this.#cancelAgents(recording);
this.#recordings.delete(requestId);
}
readonly #onMessage = (
message: unknown,
sender: chrome.runtime.MessageSender,
sendResponse: (response: RecordFrameQueryResponse) => void,
): boolean => {
if (!isRecordFrameQueryMessage(message)) return false;
const tabId = sender.tab?.id;
const recording =
typeof tabId === "number"
? [...this.#recordings.values()].find(
(candidate) => !candidate.finishing && candidate.tabIds.has(tabId),
)
: undefined;
sendResponse(
recording
? {
active: true,
requestId: recording.requestId,
startedAtMs: recording.startedAtMs,
}
: { active: false },
);
return false;
};
readonly #onFrameNavigation = (frame: BrowserFrameNavigation): void => {
if (frame.frameId === 0) return;
for (const recording of this.#recordings.values()) {
if (recording.finishing || !recording.tabIds.has(frame.tabId)) continue;
// onCommitted can race document_start, while onCompleted runs after the
// content script is available. Starting twice is safe and closes both
// the early-query and late-SPA-frame gaps.
void (async () => {
const started = await this.#startFrame(recording, frame.tabId, frame);
if (!started || frame.lifecycle !== "completed" || recording.finishing) return;
const agent = frame.documentId
? recording.agents.get(documentKey(frame.tabId, frame.documentId))
: [...recording.agents.values()].find(
(candidate) =>
candidate.tabId === frame.tabId && candidate.browserFrameId === frame.frameId,
);
if (!agent) return;
await recording.onDocumentReady?.({
tabId: agent.tabId,
documentId: agent.documentId,
browserFrameId: agent.browserFrameId,
producerId: agent.producerId,
});
})();
}
};
readonly #onConnect = (port: chrome.runtime.Port): void => {
if (port.name !== RECORD_FRAME_PORT) return;
const address = senderAddress(port.sender ?? {});
if (!address) {
port.disconnect();
return;
}
let registered: FrameAgent | null = null;
port.onMessage.addListener((raw: unknown) => {
const message = raw as Partial<RecordFramePortMessage>;
if (message.type === "ready") {
if (typeof message.requestId !== "string" || typeof message.producerId !== "string") {
port.disconnect();
return;
}
const requestId = message.requestId;
const producerId = message.producerId;
if (registered) {
if (registered.requestId === requestId && registered.producerId === producerId) {
port.postMessage({
type: "ready_ack",
requestId,
producerId,
} satisfies RecordFramePortMessage);
} else {
port.disconnect();
}
return;
}
const recording = this.#recordings.get(requestId);
if (
!recording ||
recording.finishing ||
!recording.tabIds.has(address.tabId) ||
producerId.length === 0
) {
port.disconnect();
return;
}
const agent: FrameAgent = {
...address,
requestId,
producerId,
port,
stopWaiters: new Map(),
};
registered = agent;
const key = documentKey(address.tabId, address.documentId);
const previous = recording.agents.get(key);
if (previous && previous !== agent) previous.port.disconnect();
recording.agents.set(key, agent);
port.postMessage({
type: "ready_ack",
requestId,
producerId,
} satisfies RecordFramePortMessage);
return;
}
if (
message.type === "stopped" &&
registered &&
message.requestId === registered.requestId &&
typeof message.commandId === "string"
) {
registered.stopWaiters.get(message.commandId)?.(message.ok === true);
registered.stopWaiters.delete(message.commandId);
}
});
port.onDisconnect.addListener(() => {
if (!registered) return;
for (const resolve of registered.stopWaiters.values()) resolve(true);
const recording = this.#recordings.get(registered.requestId);
const key = documentKey(registered.tabId, registered.documentId);
if (recording?.agents.get(key) === registered) recording.agents.delete(key);
});
};
#stopAgent(agent: FrameAgent, requestId: string, commandId: string): Promise<boolean> {
return new Promise((resolve) => {
const timer = setTimeout(() => {
agent.stopWaiters.delete(commandId);
resolve(false);
}, RECORD_FRAME_STOP_TIMEOUT_MS);
agent.stopWaiters.set(commandId, (ok) => {
clearTimeout(timer);
resolve(ok);
});
try {
agent.port.postMessage({
type: "stop",
requestId,
commandId,
} satisfies RecordFramePortMessage);
} catch {
clearTimeout(timer);
agent.stopWaiters.delete(commandId);
resolve(true);
}
});
}
async #startFrame(
recording: ArmedRecording,
tabId: number,
frame: BrowserFrame,
): Promise<boolean> {
const message: RecordFrameStartMessage = {
type: RECORD_FRAME_START,
requestId: recording.requestId,
startedAtMs: recording.startedAtMs,
};
try {
const response = await this.#deps.sendToDocument(tabId, message, {
...(frame.documentId ? { documentId: frame.documentId } : { frameId: frame.frameId }),
});
return isStarted(response);
} catch {
return false;
}
}
#cancelAgents(recording: ArmedRecording): void {
for (const agent of recording.agents.values()) {
try {
agent.port.postMessage({
type: "cancel",
requestId: recording.requestId,
} satisfies RecordFramePortMessage);
} catch {
// A destroyed Document has no remaining capture state to cancel.
}
}
}
}
function isStarted(value: unknown): boolean {
return typeof value === "object" && value !== null && (value as { ok?: unknown }).ok === true;
}
export const recordFrameCoordinator = new RecordFrameCoordinator();
@@ -1,4 +1,5 @@
import type { RenderedRef } from "@browser-skill/vom";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import {
type CaptureVomMatchNode,
type CaptureVomObservationResult,
@@ -28,6 +29,11 @@ export interface RegisteredObservation {
url: string;
}
export interface RecordingDocumentScope {
frameId: string;
target: CdpTarget;
}
function nodeKey(frameId: string, backendNodeId: number): string {
return `${frameId}:${backendNodeId}`;
}
@@ -40,8 +46,12 @@ export class ObservationNodeIndex {
readonly #nodesByFrameTag = new Map<string, IndexedObservationNode[]>();
readonly #refById = new Map<string, RenderedRef>();
readonly #refsByFrame = new Map<string, RenderedRef[]>();
readonly #scopeByProducer = new Map<string, RecordingDocumentScope | null>();
constructor(input: Pick<CaptureVomObservationResult, "rootFrameId" | "matchNodes" | "refs">) {
constructor(
input: Pick<CaptureVomObservationResult, "rootFrameId" | "matchNodes" | "refs"> &
Partial<Pick<CaptureVomObservationResult, "frames">>,
) {
const refByNode = new Map<string, RenderedRef>();
for (const ref of input.refs) {
const frameId = ref.frameId ?? input.rootFrameId;
@@ -51,6 +61,16 @@ export class ObservationNodeIndex {
frameRefs.push(ref);
this.#refsByFrame.set(frameId, frameRefs);
}
for (const frame of input.frames ?? []) {
const documentId = frame.recordingDocumentId;
if (!documentId) continue;
this.#scopeByProducer.set(
documentId,
this.#scopeByProducer.has(documentId)
? null
: { frameId: frame.frameId, target: frame.target },
);
}
for (const geometry of input.matchNodes) {
const { frameId } = geometry;
const entry = {
@@ -76,6 +96,10 @@ export class ObservationNodeIndex {
refs(frameId: string): readonly RenderedRef[] {
return this.#refsByFrame.get(frameId) ?? [];
}
documentScope(producerId: string): RecordingDocumentScope | undefined {
return this.#scopeByProducer.get(producerId) ?? undefined;
}
}
async function readTabMeta(
@@ -28,7 +28,13 @@ function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
}
function isTargeted(draft: RecordingDraftStep): draft is TargetedRecordingDraft {
return draft.op !== "navigate" && draft.op !== "scroll";
return (
draft.op === "click" ||
draft.op === "hover" ||
draft.op === "fill" ||
draft.op === "press" ||
draft.op === "select"
);
}
export class RecordingObservationSession {
@@ -1,6 +1,7 @@
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import type { StopReason, TraceV3 } from "@/transport/types";
import type { DocumentSettleScope } from "./document-settle";
import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle";
import type { RegisteredObservation } from "./observation-capture";
import { RecordingObservationSession } from "./observation-session";
import { inferMissingPostStates, SettleController } from "./settle-controller";
import { RecordingStateRegistry } from "./state-registry";
@@ -10,11 +11,13 @@ import type { RecordingDraftStep, StepAnnotation } from "./types";
interface TabRecordingContext {
session: RecordingObservationSession;
settle: SettleController;
initialCapture: PendingInitialCapture | null;
pendingCapture: PendingCapture | null;
frameRefresh: Promise<void>;
frameRefreshAbort: AbortController | null;
}
interface PendingInitialCapture {
promise: Promise<void>;
interface PendingCapture {
promise: Promise<RegisteredObservation>;
abort: AbortController;
}
@@ -56,37 +59,113 @@ export class RecordingObservationRuntime {
tabsApi: this.#tabsApi,
tabId,
}),
initialCapture: null,
pendingCapture: null,
frameRefresh: Promise.resolve(),
frameRefreshAbort: null,
};
this.#contexts.set(tabId, context);
return context;
}
async captureInitial(tabId: number): Promise<void> {
async #capture(tabId: number): Promise<RegisteredObservation> {
const context = this.#context(tabId);
if (context.session.cursor.lastSettled) return;
if (context.initialCapture) return context.initialCapture.promise;
if (context.pendingCapture) return context.pendingCapture.promise;
const abort = new AbortController();
let pending!: PendingInitialCapture;
let pending!: PendingCapture;
const promise = context.session.capture(this.#cdp, this.#tabsApi, tabId, abort.signal);
pending = {
abort,
promise: promise
.then(() => {})
.finally(() => {
if (context.initialCapture === pending) context.initialCapture = null;
}),
promise: promise.finally(() => {
if (context.pendingCapture === pending) context.pendingCapture = null;
}),
};
context.initialCapture = pending;
context.pendingCapture = pending;
return pending.promise;
}
async captureInitial(tabId: number): Promise<void> {
const context = this.#context(tabId);
if (context.session.cursor.lastSettled) return;
await this.#capture(tabId);
}
/** Refresh the safe observation after a late child Document becomes usable. */
async refreshDocument(tabId: number, producerId: string): Promise<void> {
const context = this.#context(tabId);
context.frameRefreshAbort?.abort();
const abort = new AbortController();
context.frameRefreshAbort = abort;
const previous = context.frameRefresh;
const refresh = (async () => {
await previous;
if (abort.signal.aborted) return;
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
if (abort.signal.aborted) return;
// The first capture discovers the safe producer -> CDP Document scope.
await this.#capture(tabId);
if (abort.signal.aborted) return;
const scope = context.session.cursor.lastSettled?.index.documentScope(producerId);
if (scope) {
await waitForDocumentSettled(this.#cdp, scope, { signal: abort.signal });
}
if (abort.signal.aborted) return;
// Capture again after the child SPA is quiet so the next action can be
// matched against refs that actually exist in its pre-action state.
await this.#capture(tabId);
})();
const tracked = refresh.catch(() => {});
context.frameRefresh = tracked;
try {
await refresh;
} finally {
if (context.frameRefresh === tracked) context.frameRefreshAbort = null;
}
}
async captureTabTransition(
fromTabId: number,
toTabId: number,
): Promise<{ preStateId?: string; postStateId?: string; targetUrl?: string }> {
const from = this.#context(fromTabId);
await this.#flushContext(from);
if (!from.session.cursor.lastSettled) {
try {
await this.captureInitial(fromTabId);
} catch {
// A target-tab observation can still preserve the forward transition.
}
}
const to = this.#context(toTabId);
let target = to.session.cursor.lastSettled;
try {
target = await this.#capture(toTabId);
} catch {
// Reuse the last settled target state when a refresh races navigation.
}
const source = from.session.cursor.lastSettled;
return {
...(source ? { preStateId: source.stateId } : {}),
...(target ? { postStateId: target.stateId, targetUrl: target.url } : {}),
};
}
bindTabTransition(
draft: Extract<RecordingDraftStep, { op: "switch_tab" }>,
draftId: number,
): void {
if (draft.preStateId) this.#registry.markStep(draft.preStateId, draftId);
}
async processDraft(
tabId: number,
drafts: RecordingDraftStep[],
draftIndex: number,
scope?: DocumentSettleScope,
producerId?: string,
): Promise<void> {
const draft = drafts[draftIndex];
if (!draft) return;
@@ -98,6 +177,23 @@ export class RecordingObservationRuntime {
// Post-action settle can still provide a usable state.
}
}
const scope = producerId
? context.session.cursor.lastSettled?.index.documentScope(producerId)
: undefined;
if (
producerId &&
(draft.op === "click" ||
draft.op === "hover" ||
draft.op === "fill" ||
draft.op === "press" ||
draft.op === "select")
) {
draft.targetHint = {
...(draft.targetHint ?? {}),
frameId: scope?.frameId ?? null,
geometrySpace: "local",
};
}
context.session.bindDraft(draft, draftIndex + 1, context.settle.hasPending);
context.settle.schedule(drafts, draftIndex, scope);
}
@@ -119,23 +215,24 @@ export class RecordingObservationRuntime {
this.#context(tabId).settle.scheduleRedirect(drafts, url);
}
async flushRedirects(): Promise<void> {
for (const context of this.#contexts.values()) await context.settle.flushRedirects();
async #flushContext(context: TabRecordingContext): Promise<void> {
await context.frameRefresh;
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
await context.settle.flushRedirects();
await context.settle.flush();
}
async flushRedirects(tabId: number): Promise<void> {
await this.#contexts.get(tabId)?.settle.flushRedirects();
}
async flush(): Promise<void> {
await Promise.allSettled(
[...this.#contexts.values()].flatMap((context) =>
context.initialCapture ? [context.initialCapture.promise] : [],
),
);
await this.flushRedirects();
for (const context of this.#contexts.values()) await context.settle.flush();
for (const context of this.#contexts.values()) await this.#flushContext(context);
}
async settleTrailing(tabId: number, drafts: RecordingDraftStep[]): Promise<void> {
const context = this.#context(tabId);
if (context.initialCapture) await Promise.allSettled([context.initialCapture.promise]);
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
if (!context.session.cursor.lastSettled) {
try {
await this.captureInitial(tabId);
@@ -149,7 +246,8 @@ export class RecordingObservationRuntime {
cancel(): void {
for (const context of this.#contexts.values()) {
context.initialCapture?.abort.abort();
context.frameRefreshAbort?.abort();
context.pendingCapture?.abort.abort();
context.settle.cancel();
}
}
@@ -161,6 +259,7 @@ export class RecordingObservationRuntime {
startUrl?: string;
stoppedBy: StopReason;
bskVersion: string;
includeTabSwitches?: boolean;
}): TraceV3 {
return buildTraceV3({
registry: this.#registry,
@@ -172,6 +271,7 @@ export class RecordingObservationRuntime {
stoppedBy: input.stoppedBy,
bskVersion: input.bskVersion,
redactValues: this.#redactValues,
includeTabSwitches: input.includeTabSwitches,
});
}
}
+17 -12
View File
@@ -4,6 +4,10 @@ import type { RecordingDraftStep, TargetMatchHint } from "./types";
export interface RecordingStepBuffer {
steps: RecordingDraftStep[];
navigation: RecordingNavigationCursor;
}
export interface RecordingNavigationCursor {
currentUrl?: string;
pendingNavigation: boolean;
pendingNavigationDeadline?: number;
@@ -87,22 +91,23 @@ export function observeRecordedNavigation(
transitionType?: string,
transitionQualifiers?: string[],
): NavigationObserveResult {
if (!url || url === buffer.currentUrl) return { kind: "noop" };
buffer.currentUrl = url;
const navigation = buffer.navigation;
if (!url || url === navigation.currentUrl) return { kind: "noop" };
navigation.currentUrl = url;
const pendingIsCurrent =
buffer.pendingNavigation &&
(buffer.pendingNavigationDeadline === undefined ||
buffer.pendingNavigationDeadline >= Date.now());
navigation.pendingNavigation &&
(navigation.pendingNavigationDeadline === undefined ||
navigation.pendingNavigationDeadline >= Date.now());
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
buffer.pendingNavigation = false;
buffer.pendingNavigationDeadline = undefined;
navigation.pendingNavigation = false;
navigation.pendingNavigationDeadline = undefined;
const annotatedIndex = annotateLastStepNavigation(buffer, url);
if (annotatedIndex >= 0) return { kind: "annotated", index: annotatedIndex };
} else {
buffer.pendingNavigation = false;
buffer.pendingNavigationDeadline = undefined;
navigation.pendingNavigation = false;
navigation.pendingNavigationDeadline = undefined;
}
if (hasRedirectQualifier(transitionQualifiers)) {
@@ -135,14 +140,14 @@ export function appendRecordedPayload(
return result.kind === "appended" ? result.index : null;
}
const step = toDraftStep(
{ ...payload, page_url: payload.page_url ?? buffer.currentUrl },
{ ...payload, page_url: payload.page_url ?? buffer.navigation.currentUrl },
targetHint,
);
if (!step) return null;
buffer.steps.push(step);
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
buffer.pendingNavigation = payload.expects_navigation === true;
buffer.pendingNavigationDeadline = buffer.pendingNavigation
buffer.navigation.pendingNavigation = payload.expects_navigation === true;
buffer.navigation.pendingNavigationDeadline = buffer.navigation.pendingNavigation
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
: undefined;
}
@@ -0,0 +1,61 @@
import type { RecordingNavigationCursor } from "./step-buffer";
export interface TabActivation {
tabId: number;
revision: number;
}
export class RecordingTabCoordinator {
readonly #navigationByTab = new Map<number, RecordingNavigationCursor>();
#activeTabId: number;
#currentTabId: number;
#activationRevision = 0;
constructor(initialTabId: number, initialUrl?: string) {
this.#activeTabId = initialTabId;
this.#currentTabId = initialTabId;
this.#navigationByTab.set(initialTabId, {
currentUrl: initialUrl,
pendingNavigation: false,
});
}
get activeTabId(): number {
return this.#activeTabId;
}
get currentTabId(): number {
return this.#currentTabId;
}
noteActivation(tabId: number): TabActivation {
this.#activeTabId = tabId;
this.#activationRevision += 1;
return { tabId, revision: this.#activationRevision };
}
isLatest(activation: TabActivation): boolean {
return (
activation.tabId === this.#activeTabId && activation.revision === this.#activationRevision
);
}
commit(tabId: number, currentUrl?: string): void {
if (currentUrl !== undefined) this.navigation(tabId).currentUrl = currentUrl;
this.#currentTabId = tabId;
}
navigation(tabId: number, fallbackUrl?: string): RecordingNavigationCursor {
const existing = this.#navigationByTab.get(tabId);
if (existing) {
if (!existing.currentUrl && fallbackUrl) existing.currentUrl = fallbackUrl;
return existing;
}
const created: RecordingNavigationCursor = {
currentUrl: fallbackUrl,
pendingNavigation: false,
};
this.#navigationByTab.set(tabId, created);
return created;
}
}
@@ -14,8 +14,13 @@ function rectMatches(a: TargetGeometry["rect"], b: TargetGeometry["rect"]): bool
return close(a.x, b.x) && close(a.y, b.y) && close(a.w, b.w) && close(a.h, b.h);
}
function candidateMatches(candidate: IndexedObservationNode, geometry: TargetGeometry): boolean {
return candidate.geometry.rect !== null && rectMatches(geometry.rect, candidate.geometry.rect);
function candidateMatches(
candidate: IndexedObservationNode,
geometry: TargetGeometry,
geometrySpace: "top" | "local",
): boolean {
const rect = geometrySpace === "local" ? candidate.geometry.localRect : candidate.geometry.rect;
return rect != null && rectMatches(geometry.rect, rect);
}
export function unmatchedTarget(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 {
@@ -53,12 +58,17 @@ export function matchObservationTarget(input: {
hint?: TargetMatchHint;
fallback?: CaptureTargetDescriptor;
}): TargetDescriptorV3 {
if (input.hint?.frameId === null) return unmatchedTarget(input.fallback);
const frameId = input.hint?.frameId ?? input.observation.rootFrameId;
const geometry = input.hint?.geometry;
if (geometry) {
const matches = input.observation.index
.candidates(frameId, geometry.tag)
.filter((candidate) => candidate.ref && candidateMatches(candidate, geometry));
.filter(
(candidate) =>
candidate.ref &&
candidateMatches(candidate, geometry, input.hint?.geometrySpace ?? "top"),
);
if (matches.length === 1) return descriptor(matches[0]!.ref!);
const semanticMatches = matches.filter(
(candidate) => candidate.ref && matchesSemantics(candidate.ref, input.fallback),
@@ -40,8 +40,12 @@ export function buildTraceV3(input: {
stoppedBy: StopReason;
bskVersion: string;
redactValues?: boolean;
includeTabSwitches?: boolean;
}): TraceV3 {
const reduced = reduceTraceStepsV3(input.drafts, { redactValues: input.redactValues });
const reduced = reduceTraceStepsV3(input.drafts, {
includeTabSwitches: input.includeTabSwitches,
redactValues: input.redactValues,
});
const entries = publishedEntries(input.registry, reduced.steps);
const publishedId = new Map(entries.map((entry, index) => [entry.id, `s${index + 1}`]));
const annotationsByState = new Map<string, StepAnnotation[]>();
@@ -119,14 +119,6 @@ function toV2Step(
},
effectForNavigation(step.navigatedTo, urlToId),
);
case "hover":
if (!step.captureTarget) return null;
return {
op: "hover",
id,
page,
target: step.captureTarget,
};
case "fill":
if (!step.captureTarget) return null;
return {
@@ -161,7 +153,11 @@ function toV2Step(
},
effectForNavigation(step.navigatedTo, urlToId),
);
// `hover` only exists in Trace v3. Peers that negotiate v2 predate the
// step variant and fail to decode the whole result if we emit it.
case "hover":
case "scroll":
case "switch_tab":
return null;
}
}
@@ -58,8 +58,26 @@ function selection(values: string[], labels?: string[]): Array<{ value: string;
}));
}
function reduceDraft(draft: RecordingDraftStep, id: number, redactValues: boolean): StepV3 | null {
interface ReduceDraftOptions {
includeTabSwitches: boolean;
redactValues: boolean;
}
function reduceDraft(
draft: RecordingDraftStep,
id: number,
options: ReduceDraftOptions,
): StepV3 | null {
if (!shouldIncludeDraft(draft)) return null;
if (draft.op === "switch_tab") {
if (!options.includeTabSwitches || !draft.preStateId || !draft.postStateId) return null;
return {
op: "switch_tab",
id,
state: draft.preStateId,
result: { state: draft.postStateId },
};
}
const state = draft.preStateId ?? draft.postStateId;
const resultState = draft.postStateId ?? draft.preStateId;
if (!state || !resultState) return null;
@@ -81,7 +99,7 @@ function reduceDraft(draft: RecordingDraftStep, id: number, redactValues: boolea
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
};
case "fill":
const fillIsRedacted = redactValues || draft.redacted === true;
const fillIsRedacted = options.redactValues || draft.redacted === true;
return {
op: "fill",
...common,
@@ -105,7 +123,7 @@ function reduceDraft(draft: RecordingDraftStep, id: number, redactValues: boolea
op: "select",
...common,
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
...(!redactValues ? { selection: selection(draft.values, draft.labels) } : {}),
...(!options.redactValues ? { selection: selection(draft.values, draft.labels) } : {}),
};
case "scroll":
return { op: "scroll", ...common };
@@ -119,12 +137,15 @@ export interface ReducedTraceV3 {
export function reduceTraceStepsV3(
steps: RecordingDraftStep[],
options: { redactValues?: boolean } = {},
options: { includeTabSwitches?: boolean; redactValues?: boolean } = {},
): ReducedTraceV3 {
const output: StepV3[] = [];
const stepIdByDraftId = new Map<number, number>();
for (const { draft, draftIds } of collapseRedirects(steps)) {
const step = reduceDraft(draft, output.length + 1, options.redactValues ?? false);
const step = reduceDraft(draft, output.length + 1, {
includeTabSwitches: options.includeTabSwitches === true,
redactValues: options.redactValues === true,
});
if (!step) continue;
output.push(step);
for (const draftId of draftIds) stepIdByDraftId.set(draftId, step.id);
+4 -2
View File
@@ -15,8 +15,9 @@ export interface TargetGeometry {
export interface TargetMatchHint {
geometry?: TargetGeometry;
/** Missing means the current top frame, never an unrestricted frame search. */
frameId?: string;
/** Missing means top frame; null means the source Document could not be resolved. */
frameId?: string | null;
geometrySpace?: "top" | "local";
}
export interface StepAnnotation {
@@ -69,6 +70,7 @@ export type RecordingDraftStep =
DraftTarget &
DraftNavigationEffect)
| ({ op: "scroll" } & DraftStateLink)
| ({ op: "switch_tab" } & DraftStateLink)
| ({
op: "navigate";
url: string;
@@ -0,0 +1,22 @@
export const RECORD_DOCUMENT_ATTRIBUTE = "data-bsk-record-document";
const RECORDING_DOCUMENT_ID_PATTERN =
/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
const RECORDING_DOCUMENT_MARKER_PREFIX = "bsk:";
export function recordingDocumentMarkerValue(identity: string): string {
if (!RECORDING_DOCUMENT_ID_PATTERN.test(identity)) {
throw new TypeError("recording Document identity must be a random UUID");
}
return `${RECORDING_DOCUMENT_MARKER_PREFIX}${identity}`;
}
/** Read BrowserSkill's opaque per-Document recording identity from captured attributes. */
export function readRecordingDocumentIdentity(
attrs: Readonly<Record<string, string>>,
): string | undefined {
const marker = attrs[RECORD_DOCUMENT_ATTRIBUTE];
if (!marker?.startsWith(RECORDING_DOCUMENT_MARKER_PREFIX)) return undefined;
const identity = marker.slice(RECORDING_DOCUMENT_MARKER_PREFIX.length);
return identity && RECORDING_DOCUMENT_ID_PATTERN.test(identity) ? identity : undefined;
}
@@ -8,6 +8,7 @@ import type {
RequestFrame,
} from "@/transport/types";
import { ToolDispatcher } from "../dispatcher";
import { resetBrowserObservationForTests, setBrowserObservationAttachForTests } from "../record";
type TestDispatcherCdp = NonNullable<ConstructorParameters<typeof ToolDispatcher>[0]["cdp"]>;
@@ -44,9 +45,72 @@ function makeRequest(method: string, params: unknown): RequestFrame {
describe("ToolDispatcher", () => {
afterEach(() => {
resetBrowserObservationForTests();
vi.unstubAllGlobals();
});
it("uses the configured recording runtime for start and stop", async () => {
const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({
agentWindow: {
create: vi.fn(async () => 4242),
remove: vi.fn(),
ensureActiveTab: vi.fn(async () => {}),
},
});
await sessions.start("aa11");
const tab = {
id: 7,
windowId: 4242,
active: true,
status: "complete",
url: "https://example.com/start",
} as chrome.tabs.Tab;
const frameCoordinator = {
begin: vi.fn(),
armTab: vi.fn(async () => true),
sourceFor: vi.fn(() => null),
stop: vi.fn(async () => true),
cancel: vi.fn(),
};
const sendToTab = vi.fn(async () => ({ ok: true }));
setBrowserObservationAttachForTests(
() => () => {},
() => () => {},
);
const dispatcher = new ToolDispatcher({
transport,
sessions,
recording: {
tabsApi: {
get: vi.fn(async () => tab),
query: vi.fn(async () => [tab]),
},
frameCoordinator,
sendToTab,
},
});
dispatcher.start();
deliver(
makeRequest("tool.record_start", {
session_id: "aa11",
url: "https://example.com/start",
}),
);
await vi.waitFor(() => expect(sent).toHaveLength(1));
expect(sent[0]).toMatchObject({ result: { tab_id: 7, recording: true } });
expect(frameCoordinator.begin).toHaveBeenCalledOnce();
expect(frameCoordinator.armTab).toHaveBeenCalledWith(expect.any(String), 7);
deliver({ id: "r-2", method: "tool.record_stop", params: { session_id: "aa11" } });
await vi.waitFor(() => expect(sent).toHaveLength(2));
expect(frameCoordinator.stop).toHaveBeenCalledOnce();
expect(sent[1]).toMatchObject({ id: "r-2", result: { trace: { steps: [] } } });
});
it("routes tool.session_start to the SessionManager and replies with the window id", async () => {
const { transport, sent, deliver } = fakeTransport();
const sessions = new SessionManager({
@@ -15,7 +15,12 @@ import {
const AGENT_WINDOW_ID = 100;
const TAB_ID = 4;
const START_URL = "https://example.com/";
const RECORD_START_V3 = { session_id: "abcd", url: START_URL, trace_version: 3 as const };
const RECORD_START_V3 = {
session_id: "abcd",
url: START_URL,
trace_version: 3 as const,
supports_tab_switch_steps: true,
};
const stepSequenceByProducer = new Map<string, number>();
function asTraceV3(trace: RecordedTrace): TraceV3 {
@@ -48,6 +53,7 @@ function chromeEvent<T extends (...args: never[]) => unknown>() {
function installChrome() {
const runtimeOnMessage = chromeEvent<RuntimeListener>();
const tabsOnActivated = chromeEvent<(activeInfo: chrome.tabs.TabActiveInfo) => unknown>();
const webNavigationOnCompleted =
chromeEvent<(details: chrome.webNavigation.WebNavigationFramedCallbackDetails) => unknown>();
const webNavigationOnCommitted =
@@ -57,7 +63,7 @@ function installChrome() {
vi.stubGlobal("chrome", {
runtime: { onMessage: runtimeOnMessage },
tabs: {
onActivated: chromeEvent(),
onActivated: tabsOnActivated,
onCreated: chromeEvent(),
onUpdated: chromeEvent(),
},
@@ -66,7 +72,7 @@ function installChrome() {
onCommitted: webNavigationOnCommitted,
},
});
return { runtimeOnMessage, webNavigationOnCompleted, webNavigationOnCommitted };
return { runtimeOnMessage, tabsOnActivated, webNavigationOnCompleted, webNavigationOnCommitted };
}
function fakeManager() {
@@ -118,7 +124,10 @@ type FakeCdp = CdpRunner & {
const LONG_IDLE_MS = 10_000;
/** AX-only page: DOMSnapshot calls throw so capture falls back to the AX tree. */
function makeFakeCdp(trees?: unknown[], options?: { failCaptures?: boolean }): FakeCdp {
function makeFakeCdp(
trees?: unknown[],
options?: { failCaptures?: boolean; treesByTab?: Record<number, unknown> },
): FakeCdp {
type EventListener = (source: chrome.debugger.Debuggee, method: string, params: unknown) => void;
const events: EventListener[] = [];
const script = [...(trees ?? [])];
@@ -153,6 +162,8 @@ function makeFakeCdp(trees?: unknown[], options?: { failCaptures?: boolean }): F
"Accessibility.enable": () => ({}),
"Accessibility.getFullAXTree": (_params, _tabId) => {
if (failing) throw new Error("Execution context was destroyed");
const tabTree = options?.treesByTab?.[_tabId];
if (tabTree) return tabTree;
if (script.length === 0) return axTree("Example Domain", ["Submit"]);
return script.length === 1 ? script[0] : script.shift();
},
@@ -202,6 +213,31 @@ function makeTabsApi() {
};
}
function makeMultiTabsApi(tabs: chrome.tabs.Tab[]) {
const byId = new Map(tabs.flatMap((tab) => (typeof tab.id === "number" ? [[tab.id, tab]] : [])));
return {
get: async (tabId: number) => {
const tab = byId.get(tabId);
if (!tab) throw new Error(`tab ${tabId} closed`);
return tab;
},
query: async (query: chrome.tabs.QueryInfo) =>
[...byId.values()].filter(
(tab) =>
(query.windowId === undefined || tab.windowId === query.windowId) &&
(query.active === undefined || tab.active === query.active),
),
activate(tabId: number) {
for (const tab of byId.values()) tab.active = tab.id === tabId;
},
goTo(tabId: number, url: string, title: string) {
const tab = byId.get(tabId);
if (!tab) throw new Error(`tab ${tabId} closed`);
Object.assign(tab, { url, title });
},
};
}
/** Outlast a settle on a quiet page plus the per-tab observation cooldown. */
function settleWait(): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, 800));
@@ -212,16 +248,19 @@ function runtimeOnMessageEmit(
requestId: string,
step: unknown,
tabId = TAB_ID,
): void {
active = true,
): ReturnType<typeof vi.fn> {
const producerId = `test-producer-${tabId}`;
const producerKey = `${requestId}:${producerId}`;
const sequence = (stepSequenceByProducer.get(producerKey) ?? 0) + 1;
stepSequenceByProducer.set(producerKey, sequence);
const sendResponse = vi.fn();
chromeApi.runtimeOnMessage.emit(
{ type: RECORD_STEP, requestId, producerId, sequence, step },
{ tab: { id: tabId } } as chrome.runtime.MessageSender,
() => {},
{ tab: { id: tabId, active } } as chrome.runtime.MessageSender,
sendResponse,
);
return sendResponse;
}
describe("recorded user steps reach the exported trace", () => {
@@ -1161,7 +1200,8 @@ describe("recorded user steps reach the exported trace", () => {
const trace = (stopped as RecordStopResult).trace;
expect("version" in trace).toBe(false);
expect("pages" in trace && trace.pages.length).toBeGreaterThan(0);
expect("steps" in trace && trace.steps.map((step) => step.op)).toEqual(["hover", "click"]);
// `hover` has no v2 counterpart on peers that negotiate v2.
expect("steps" in trace && trace.steps.map((step) => step.op)).toEqual(["click"]);
});
it("rejects unsupported trace_version values", async () => {
@@ -1173,4 +1213,391 @@ describe("recorded user steps reach the exported trace", () => {
);
expect(result).toMatchObject({ code: "invalid_params" });
});
it("records tab transitions and binds actions to each tab's observation", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const secondUrl = "https://example.com/second";
const tabsApi = makeMultiTabsApi([
{
id: TAB_ID,
windowId: AGENT_WINDOW_ID,
active: true,
status: "complete",
url: START_URL,
title: "First tab",
} as chrome.tabs.Tab,
{
id: 5,
windowId: AGENT_WINDOW_ID,
active: false,
status: "complete",
url: secondUrl,
title: "Second tab",
} as chrome.tabs.Tab,
]);
const cdp = makeFakeCdp(undefined, {
treesByTab: {
[TAB_ID]: axTree("First tab", ["First action"]),
5: axTree("Second tab", ["Second action"]),
},
});
let requestId = "";
const sendToTab = vi.fn(async (_tabId: number, message: unknown) => {
const typed = message as { type?: string; requestId?: string };
if (typed.type === RECORD_START && typed.requestId) requestId = typed.requestId;
return { ok: true };
});
const deps = { tabsApi, sendToTab, cdp };
await handleRecordStart(manager, RECORD_START_V3, deps);
attachRecordStepListener(deps);
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
await settleWait();
runtimeOnMessageEmit(
chromeApi,
requestId,
{
op: "click",
page_url: secondUrl,
target: { role: "button", name: "Second action", tag: "button" },
},
5,
);
await settleWait();
tabsApi.activate(TAB_ID);
chromeApi.tabsOnActivated.emit({ tabId: TAB_ID, windowId: AGENT_WINDOW_ID });
await settleWait();
runtimeOnMessageEmit(chromeApi, requestId, {
op: "click",
page_url: START_URL,
target: { role: "button", name: "First action", tag: "button" },
});
await settleWait();
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
const trace = asTraceV3((stopped as RecordStopResult).trace);
const stateUrl = (stateId: string) => trace.states.find((state) => state.id === stateId)?.url;
const switches = trace.steps.filter((step) => step.op === "switch_tab");
const clicks = trace.steps.filter((step) => step.op === "click");
expect(switches).toHaveLength(2);
expect(clicks).toHaveLength(2);
expect(stateUrl(switches[0]!.state)).toBe(START_URL);
expect(stateUrl(switches[0]!.result.state)).toBe(secondUrl);
expect(stateUrl(clicks[0]!.state)).toBe(secondUrl);
expect(stateUrl(switches[1]!.result.state)).toBe(START_URL);
expect(stateUrl(clicks[1]!.state)).toBe(START_URL);
});
it("keeps switch_tab internal when the v3 caller did not advertise support", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const tabsApi = makeMultiTabsApi([
{
id: TAB_ID,
windowId: AGENT_WINDOW_ID,
active: true,
status: "complete",
url: START_URL,
} as chrome.tabs.Tab,
{
id: 5,
windowId: AGENT_WINDOW_ID,
active: false,
status: "complete",
url: "https://example.com/second",
} as chrome.tabs.Tab,
]);
const deps = { tabsApi, sendToTab: vi.fn(async () => ({ ok: true })), cdp: makeFakeCdp() };
await handleRecordStart(
manager,
{ session_id: "abcd", url: START_URL, trace_version: 3 },
deps,
);
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
await settleWait();
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
const trace = asTraceV3((stopped as RecordStopResult).trace);
expect(trace.steps.some((step) => step.op === "switch_tab")).toBe(false);
});
it("keeps the latest rapid tab activation when an earlier rearm acknowledges late", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const tabsApi = makeMultiTabsApi(
[TAB_ID, 5, 6].map(
(id) =>
({
id,
windowId: AGENT_WINDOW_ID,
active: id === TAB_ID,
status: "complete",
url: id === TAB_ID ? START_URL : `https://example.com/tab-${id}`,
}) as chrome.tabs.Tab,
),
);
let releaseSecond!: () => void;
const secondGate = new Promise<void>((resolve) => {
releaseSecond = resolve;
});
let announceSecond!: () => void;
const secondStarted = new Promise<void>((resolve) => {
announceSecond = resolve;
});
const sendToTab = vi.fn(async (tabId: number, message: unknown) => {
if ((message as { type?: string }).type === RECORD_START && tabId === 5) {
announceSecond();
await secondGate;
}
return { ok: true };
});
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
await handleRecordStart(manager, RECORD_START_V3, deps);
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
await secondStarted;
tabsApi.activate(6);
chromeApi.tabsOnActivated.emit({ tabId: 6, windowId: AGENT_WINDOW_ID });
await settleWait();
releaseSecond();
await new Promise((resolve) => setTimeout(resolve, 50));
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
const trace = asTraceV3((stopped as RecordStopResult).trace);
const switches = trace.steps.filter((step) => step.op === "switch_tab");
const stateUrl = (stateId: string) => trace.states.find((state) => state.id === stateId)?.url;
expect(switches).toHaveLength(1);
expect(stateUrl(switches[0]!.result.state)).toBe("https://example.com/tab-6");
});
it("does not retry an in-flight tab rearm after recording stops", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const tabsApi = makeMultiTabsApi([
{
id: TAB_ID,
windowId: AGENT_WINDOW_ID,
active: true,
status: "complete",
url: START_URL,
} as chrome.tabs.Tab,
{
id: 5,
windowId: AGENT_WINDOW_ID,
active: false,
status: "complete",
url: "https://example.com/second",
} as chrome.tabs.Tab,
]);
let failRearm!: () => void;
const rearmGate = new Promise<never>((_resolve, reject) => {
failRearm = () => reject(new Error("rearm failed after stop"));
});
let announceRearm!: () => void;
const rearmStarted = new Promise<void>((resolve) => {
announceRearm = resolve;
});
const sendToTab = vi.fn(async (tabId: number, message: unknown) => {
if ((message as { type?: string }).type === RECORD_START && tabId === 5) {
announceRearm();
await rearmGate;
}
return { ok: true };
});
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
await handleRecordStart(manager, RECORD_START_V3, deps);
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
await rearmStarted;
const stoppedPromise = handleRecordStop(manager, { session_id: "abcd" }, deps);
await vi.waitFor(() => {
expect(sendToTab).toHaveBeenCalledWith(5, expect.objectContaining({ type: RECORD_STOP }));
});
failRearm();
const stopped = await stoppedPromise;
await new Promise((resolve) => setTimeout(resolve, 600));
const startsForSecondTab = sendToTab.mock.calls.filter(
([tabId, message]) => tabId === 5 && (message as { type?: string }).type === RECORD_START,
);
expect(startsForSecondTab).toHaveLength(1);
expect(asTraceV3((stopped as RecordStopResult).trace).steps).toHaveLength(0);
});
it("keeps pending action settles scoped to their tab during an immediate switch", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const secondUrl = "https://example.com/second";
const tabsApi = makeMultiTabsApi([
{
id: TAB_ID,
windowId: AGENT_WINDOW_ID,
active: true,
status: "complete",
url: START_URL,
} as chrome.tabs.Tab,
{
id: 5,
windowId: AGENT_WINDOW_ID,
active: false,
status: "complete",
url: secondUrl,
} as chrome.tabs.Tab,
]);
let requestId = "";
const sendToTab = vi.fn(async (_tabId: number, message: unknown) => {
const typed = message as { type?: string; requestId?: string };
if (typed.type === RECORD_START && typed.requestId) requestId = typed.requestId;
return { ok: true };
});
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
await handleRecordStart(manager, RECORD_START_V3, deps);
attachRecordStepListener(deps);
runtimeOnMessageEmit(chromeApi, requestId, {
op: "click",
page_url: START_URL,
target: { role: "button", name: "First action", tag: "button" },
});
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
runtimeOnMessageEmit(
chromeApi,
requestId,
{
op: "click",
page_url: secondUrl,
target: { role: "button", name: "Second action", tag: "button" },
},
5,
);
await settleWait();
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
const trace = asTraceV3((stopped as RecordStopResult).trace);
const stateUrl = (stateId: string) => trace.states.find((state) => state.id === stateId)?.url;
expect(trace.steps.map((step) => step.op)).toEqual(["click", "switch_tab", "click"]);
expect(stateUrl(trace.steps[0]!.result.state)).toBe(START_URL);
expect(stateUrl(trace.steps[1]!.result.state)).toBe(secondUrl);
expect(stateUrl(trace.steps[2]!.state)).toBe(secondUrl);
});
it("tracks the same navigation URL independently for each tab", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const tabsApi = makeMultiTabsApi([
{
id: TAB_ID,
windowId: AGENT_WINDOW_ID,
active: true,
status: "complete",
url: START_URL,
} as chrome.tabs.Tab,
{
id: 5,
windowId: AGENT_WINDOW_ID,
active: false,
status: "complete",
url: "https://example.com/second",
} as chrome.tabs.Tab,
]);
const deps = { tabsApi, sendToTab: vi.fn(async () => ({ ok: true })), cdp: makeFakeCdp() };
await handleRecordStart(manager, RECORD_START_V3, deps);
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
await settleWait();
tabsApi.goTo(5, START_URL, "Same URL in second tab");
chromeApi.webNavigationOnCommitted.emit({
tabId: 5,
frameId: 0,
url: START_URL,
transitionType: "typed",
transitionQualifiers: ["from_address_bar"],
} as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails);
await settleWait();
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
const trace = asTraceV3((stopped as RecordStopResult).trace);
expect(
trace.steps.filter((step) => step.op === "navigate" && step.to === START_URL),
).toHaveLength(1);
});
it("drops late input and frame navigation from an inactive tab", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const secondUrl = "https://example.com/second";
const tabsApi = makeMultiTabsApi([
{
id: TAB_ID,
windowId: AGENT_WINDOW_ID,
active: true,
status: "complete",
url: START_URL,
} as chrome.tabs.Tab,
{
id: 5,
windowId: AGENT_WINDOW_ID,
active: false,
status: "complete",
url: secondUrl,
} as chrome.tabs.Tab,
]);
let requestId = "";
const sendToTab = vi.fn(async (_tabId: number, message: unknown) => {
const typed = message as { type?: string; requestId?: string };
if (typed.type === RECORD_START && typed.requestId) requestId = typed.requestId;
return { ok: true };
});
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
await handleRecordStart(manager, RECORD_START_V3, deps);
attachRecordStepListener(deps);
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
await settleWait();
const lateAck = runtimeOnMessageEmit(
chromeApi,
requestId,
{
op: "fill",
page_url: START_URL,
target: { role: "textbox", name: "Draft", tag: "input" },
value: "late",
},
TAB_ID,
false,
);
expect(lateAck).toHaveBeenCalledWith({ ok: true, sequence: 1 });
chromeApi.webNavigationOnCommitted.emit({
tabId: TAB_ID,
frameId: 0,
url: "https://example.com/late",
transitionType: "link",
transitionQualifiers: [],
} as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails);
chromeApi.webNavigationOnCommitted.emit({
tabId: 5,
frameId: 7,
url: "https://frame.example/late",
transitionType: "link",
transitionQualifiers: [],
} as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails);
await new Promise((resolve) => setTimeout(resolve, 50));
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
const trace = asTraceV3((stopped as RecordStopResult).trace);
expect(trace.steps.map((step) => step.op)).toEqual(["switch_tab"]);
});
});
+34 -37
View File
@@ -51,7 +51,12 @@ import {
handleScreenshot,
handleSnapshot,
} from "./observation";
import { handleRecordAwait, handleRecordStart, handleRecordStop } from "./record";
import {
handleRecordAwait,
handleRecordStart,
handleRecordStop,
type RecordRuntimeDeps,
} from "./record";
import {
handleSessionStart,
handleSessionStop,
@@ -99,6 +104,7 @@ export interface DispatcherDeps {
transport: Transport;
sessions: SessionManager;
cdp?: DispatcherCdpRunner;
recording?: RecordRuntimeDeps;
/**
* Invoked whenever a dispatched RPC may have changed the live
* session set (currently `tool.session_start` and
@@ -134,6 +140,7 @@ export class ToolDispatcher {
private readonly transport: Transport;
private readonly sessions: SessionManager;
private readonly cdp?: DispatcherCdpRunner;
private readonly recording?: RecordRuntimeDeps;
private readonly onSessionsChanged?: () => void;
private readonly onBrowserControlResumed?: (sessionId: string) => void;
private readonly approveBorrow?: BorrowConfirmationApprover;
@@ -153,6 +160,7 @@ export class ToolDispatcher {
this.transport = deps.transport;
this.sessions = deps.sessions;
this.cdp = deps.cdp;
this.recording = deps.recording;
this.onSessionsChanged = deps.onSessionsChanged;
this.onBrowserControlResumed = deps.onBrowserControlResumed;
this.approveBorrow = deps.approveBorrow;
@@ -531,44 +539,26 @@ export class ToolDispatcher {
signal,
});
case "tool.record_start":
return handleRecordStart(this.sessions, req.params as RecordStartParams, {
tabsApi: chromeTabsApi,
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
bypassOverlay: async (tabId, enabled) => {
try {
await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
...(this.cdp ? { cdp: this.cdp } : {}),
signal,
});
return this.recording
? handleRecordStart(this.sessions, req.params as RecordStartParams, {
...this.recording,
signal,
})
: recordingRuntimeUnavailable();
case "tool.record_stop":
return handleRecordStop(this.sessions, req.params as RecordStopParams, {
tabsApi: chromeTabsApi,
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
bypassOverlay: async (tabId, enabled) => {
try {
await chrome.tabs.sendMessage(tabId, {
type: OVERLAY_AUTOMATION_BYPASS,
enabled,
});
} catch {
// Content script may be unavailable on restricted pages.
}
},
signal,
});
return this.recording
? handleRecordStop(this.sessions, req.params as RecordStopParams, {
...this.recording,
signal,
})
: recordingRuntimeUnavailable();
case "tool.record_await":
return handleRecordAwait(this.sessions, req.params as RecordAwaitParams, {
tabsApi: chromeTabsApi,
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
signal,
});
return this.recording
? handleRecordAwait(this.sessions, req.params as RecordAwaitParams, {
...this.recording,
signal,
})
: recordingRuntimeUnavailable();
default:
return {
code: "unknown_method",
@@ -704,6 +694,13 @@ function isRpcError(v: unknown): v is RpcError {
);
}
function recordingRuntimeUnavailable(): RpcError {
return {
code: "protocol_error",
message: "recording runtime is unavailable",
};
}
function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
switch (req.method) {
case "tool.tab_create":
+181 -76
View File
@@ -18,8 +18,18 @@ import {
type RecordStepAck,
type RecordStopMessage,
} from "@/lib/record-bridge";
import {
type RecordFrameCoordinator,
type RecordingCaptureScope,
recordFrameCoordinator,
} from "@/lib/recording/frame-coordinator";
import { RecordingObservationRuntime } from "@/lib/recording/recording-runtime";
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording/step-buffer";
import {
appendRecordedPayload,
observeRecordedNavigation,
type RecordingStepBuffer,
} from "@/lib/recording/step-buffer";
import { RecordingTabCoordinator, type TabActivation } from "@/lib/recording/tab-coordinator";
import { buildTraceV2 } from "@/lib/recording/trace-reducer-v2";
import type { RecordingDraftStep } from "@/lib/recording/types";
import type { SessionManager } from "@/session-manager/manager";
@@ -48,7 +58,7 @@ import {
interface ActiveRecording {
requestId: string;
tabId: number;
tabs: RecordingTabCoordinator;
agentWindowId: number;
startUrl?: string;
purpose?: string;
@@ -56,14 +66,12 @@ interface ActiveRecording {
startedAt: string;
startedAtMs: number;
traceVersion: 2 | 3;
supportsTabSwitchSteps: boolean;
finishPromise: Promise<RecordedTrace>;
resolveFinish: (trace: RecordedTrace) => void;
rejectFinish: (err: Error) => void;
settled: boolean;
finishAttempt: Promise<RecordedTrace | null> | null;
currentUrl?: string;
pendingNavigation: boolean;
pendingNavigationDeadline?: number;
observation: RecordingObservationRuntime | null;
stoppedBy: StopReason;
/** Navigation callbacks tracked from event receipt through action enqueue. */
@@ -79,8 +87,17 @@ interface ActiveRecording {
lastStepSequenceByProducer: Map<string, number>;
}
function enqueueRecordingAction(recording: ActiveRecording, task: () => Promise<void>): void {
recording.actionQueue = recording.actionQueue.then(task, task).catch(() => {});
function enqueueRecordingAction(
recording: ActiveRecording,
task: () => Promise<void>,
): Promise<void> {
const queued = recording.actionQueue.then(task, task);
recording.actionQueue = queued.catch(() => {});
return queued;
}
function isRecordingFinishing(recording: ActiveRecording): boolean {
return recording.settled || recording.finishAttempt !== null;
}
const recordings = new Map<string, ActiveRecording>();
@@ -163,9 +180,11 @@ async function sendRecordStartWithAck(
tabId: number,
msg: RecordStartMessage,
sendToTab: RecordDeps["sendToTab"],
cancelled?: () => boolean,
): Promise<void> {
let lastError: unknown;
for (let attempt = 0; attempt < RECORD_START_RETRIES; attempt += 1) {
if (cancelled?.()) throw new Error("record start cancelled");
try {
const response = await sendToTab(tabId, msg);
if (isRecordStartAck(response)) return;
@@ -173,6 +192,7 @@ async function sendRecordStartWithAck(
} catch (err) {
lastError = err;
}
if (cancelled?.()) throw new Error("record start cancelled");
if (attempt + 1 < RECORD_START_RETRIES) {
await sleep(RECORD_START_RETRY_DELAY_MS);
}
@@ -198,6 +218,7 @@ function buildTrace(recording: ActiveRecording): RecordedTrace {
startUrl: recording.startUrl,
stoppedBy: recording.stoppedBy,
bskVersion: BSK_TRACE_VERSION,
includeTabSwitches: recording.supportsTabSwitchSteps,
});
}
if (recording.traceVersion === 3) {
@@ -211,14 +232,46 @@ function buildTrace(recording: ActiveRecording): RecordedTrace {
});
}
function stepBufferFor(
recording: ActiveRecording,
tabId: number,
fallbackUrl?: string,
): RecordingStepBuffer {
return {
steps: recording.steps,
navigation: recording.tabs.navigation(tabId, fallbackUrl),
};
}
async function activateRecordingTab(
recording: ActiveRecording,
targetTabId: number,
): Promise<void> {
const previousTabId = recording.tabs.currentTabId;
if (targetTabId === previousTabId) return;
const transition = recording.observation
? await recording.observation.captureTabTransition(previousTabId, targetTabId)
: {};
const { targetUrl, ...stateLinks } = transition;
const draft: Extract<RecordingDraftStep, { op: "switch_tab" }> = {
op: "switch_tab",
...stateLinks,
};
recording.steps.push(draft);
recording.observation?.bindTabTransition(draft, recording.steps.length);
recording.tabs.commit(targetTabId, targetUrl);
}
async function processRecordedStep(
recording: ActiveRecording,
draftIndex: number,
tabId: number,
producerId?: string,
): Promise<void> {
if (!recording.observation) return;
try {
await recording.observation.processDraft(tabId, recording.steps, draftIndex);
await recording.observation.processDraft(tabId, recording.steps, draftIndex, producerId);
} catch (err) {
console.warn(`[bsk record] observation failed for step ${draftIndex + 1}`, err);
}
@@ -231,16 +284,25 @@ export interface RecordDeps {
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
): Promise<unknown>;
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
frameCoordinator?: Pick<
RecordFrameCoordinator,
"begin" | "armTab" | "sourceFor" | "stop" | "cancel"
>;
cdp?: CdpRunner;
signal?: AbortSignal;
}
export type RecordRuntimeDeps = Omit<RecordDeps, "frameCoordinator" | "signal"> & {
frameCoordinator: NonNullable<RecordDeps["frameCoordinator"]>;
};
let defaultDeps: RecordDeps | null = null;
function getDefaultDeps(): RecordDeps {
if (!defaultDeps) {
defaultDeps = {
tabsApi: chromeTabsApi,
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
frameCoordinator: recordFrameCoordinator,
};
}
return defaultDeps;
@@ -307,8 +369,15 @@ export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): (
if (!isRecordStepMessage(message)) return false;
for (const recording of recordings.values()) {
if (recording.requestId !== message.requestId) continue;
const sourceTabId = sender.tab?.id ?? recording.tabId;
const producerKey = `${sourceTabId}:${message.producerId}`;
const source: RecordingCaptureScope | null | undefined = deps.frameCoordinator
? deps.frameCoordinator.sourceFor(message.requestId, message.producerId, sender)
: undefined;
if (deps.frameCoordinator && !source) return false;
const sourceTabId = source?.tabId ?? sender.tab?.id ?? recording.tabs.currentTabId;
const sourceWasActive = sender.tab?.active ?? sourceTabId === recording.tabs.activeTabId;
const producerKey = source
? `${source.tabId}:${source.documentId}:${source.producerId}`
: `${sourceTabId}:${message.producerId}`;
const expectedSequence = (recording.lastStepSequenceByProducer.get(producerKey) ?? 0) + 1;
if (message.sequence < expectedSequence) {
sendResponse({ ok: true, sequence: message.sequence });
@@ -325,11 +394,21 @@ export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): (
recording.lastStepSequenceByProducer.set(producerKey, message.sequence);
enqueueRecordingAction(recording, async () => {
await recording.observation?.flushRedirects();
if (sourceTabId !== recording.tabs.currentTabId) {
if (!sourceWasActive) return;
if (sourceTabId !== recording.tabs.activeTabId)
recording.tabs.noteActivation(sourceTabId);
await activateRecordingTab(recording, sourceTabId);
}
await recording.observation?.flushRedirects(sourceTabId);
const targetHint = message.step.geometry ? { geometry: message.step.geometry } : undefined;
const draftIndex = appendRecordedPayload(recording, message.step, targetHint);
const draftIndex = appendRecordedPayload(
stepBufferFor(recording, sourceTabId, message.step.page_url),
message.step,
targetHint,
);
if (draftIndex !== null) {
await processRecordedStep(recording, draftIndex, sourceTabId);
await processRecordedStep(recording, draftIndex, sourceTabId, source?.producerId);
}
});
sendResponse({ ok: true, sequence: message.sequence });
@@ -359,7 +438,7 @@ export function attachRecordFinishListener(deps: RecordDeps = getDefaultDeps()):
function findRecordingByTabId(tabId: number): ActiveRecording | null {
for (const recording of recordings.values()) {
if (recording.tabId === tabId && !recording.settled) return recording;
if (recording.tabs.currentTabId === tabId && !recording.settled) return recording;
}
return null;
}
@@ -396,7 +475,7 @@ async function clearRearmTimersForRecording(
recording: ActiveRecording,
deps: RecordDeps,
): Promise<void> {
clearRearmTimer(recording.tabId);
clearRearmTimer(recording.tabs.currentTabId);
try {
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
for (const tab of tabs) {
@@ -411,13 +490,16 @@ async function stopRecordingOnAllAgentTabs(
recording: ActiveRecording,
deps: RecordDeps,
): Promise<void> {
if (deps.frameCoordinator && !(await deps.frameCoordinator.stop(recording.requestId))) {
throw new Error("failed to flush one or more recording documents");
}
const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
let tabIds = [recording.tabId];
let tabIds = [recording.tabs.currentTabId];
try {
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
tabIds = [
...new Set([
recording.tabId,
recording.tabs.currentTabId,
...tabs.flatMap((tab) => (typeof tab.id === "number" ? [tab.id] : [])),
]),
];
@@ -428,11 +510,11 @@ async function stopRecordingOnAllAgentTabs(
for (const tabId of tabIds) {
try {
const response = await deps.sendToTab(tabId, stopMsg);
if (tabId === recording.tabId && !isRecordStartAck(response)) {
if (tabId === recording.tabs.currentTabId && !isRecordStartAck(response)) {
throw new Error("content script did not confirm recorded steps");
}
} catch {
if (tabId === recording.tabId) {
if (tabId === recording.tabs.currentTabId) {
throw new Error("failed to flush recorded steps");
}
}
@@ -450,22 +532,39 @@ async function rearmRecording(
recording: ActiveRecording,
targetTabId: number,
deps: RecordDeps,
activation?: TabActivation,
): Promise<boolean> {
// Do NOT toggle automation-bypass here: each retry used to increment the
// content-script counter, and a single stop decrement left the ControlOverlay
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
// already hides the control chrome while activeRecord is set.
const isFinishing = () => isRecordingFinishing(recording);
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
if (isFinishing()) return false;
const startMsg: RecordStartMessage = {
type: RECORD_START,
requestId: recording.requestId,
startedAtMs: recording.startedAtMs,
};
try {
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
recording.tabId = targetTabId;
if (deps.frameCoordinator) {
const frameStarted = await deps.frameCoordinator.armTab(recording.requestId, targetTabId);
if (!frameStarted) throw new Error("recording document did not start");
}
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab, isFinishing);
if (isFinishing()) return false;
if (activation) {
if (!recording.tabs.isLatest(activation)) return true;
await enqueueRecordingAction(recording, async () => {
if (isFinishing() || !recording.tabs.isLatest(activation)) {
return;
}
await activateRecordingTab(recording, targetTabId);
});
}
return true;
} catch {
if (isFinishing()) return false;
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
await sleep(RECORD_REARM_RETRY_DELAY_MS);
}
@@ -474,7 +573,7 @@ async function rearmRecording(
return false;
}
function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
function scheduleRearmForTab(tabId: number, deps: RecordDeps, activation?: TabActivation): void {
const existing = rearmTimers.get(tabId);
if (existing) clearTimeout(existing);
rearmTimers.set(
@@ -483,7 +582,7 @@ function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
rearmTimers.delete(tabId);
void (async () => {
const current = await findRecordingForTab(tabId, deps);
if (current) await rearmRecording(current, tabId, deps);
if (current) await rearmRecording(current, tabId, deps, activation);
})();
}, RECORD_REARM_DEBOUNCE_MS),
);
@@ -495,14 +594,20 @@ export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): ()
const windowId = tab.windowId;
if (tabId === undefined || windowId === undefined) return;
for (const recording of recordings.values()) {
if (recording.settled || recording.agentWindowId !== windowId) continue;
if (isRecordingFinishing(recording) || recording.agentWindowId !== windowId) continue;
scheduleRearmForTab(tabId, deps);
return;
}
};
const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => {
scheduleRearmForTab(activeInfo.tabId, deps);
for (const recording of recordings.values()) {
if (isRecordingFinishing(recording) || recording.agentWindowId !== activeInfo.windowId)
continue;
const activation = recording.tabs.noteActivation(activeInfo.tabId);
scheduleRearmForTab(activeInfo.tabId, deps, activation);
return;
}
};
chrome.tabs.onCreated.addListener(onCreated);
@@ -522,54 +627,39 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
transitionQualifiers?: string[],
) => {
if (!url) return;
const direct = findRecordingByTabId(tabId);
const candidates = direct
? direct.acceptingNavigation
? [direct]
: []
: [...recordings.values()].filter(
(recording) => !recording.settled && recording.acceptingNavigation,
const candidates = [...recordings.values()].filter(
(recording) =>
!recording.settled && recording.acceptingNavigation && recording.tabs.activeTabId === tabId,
);
for (const recording of candidates) {
const queued = enqueueRecordingAction(recording, async () => {
if (tabId !== recording.tabs.currentTabId) {
await activateRecordingTab(recording, tabId);
}
const result = observeRecordedNavigation(
stepBufferFor(recording, tabId, url),
url,
causedByAction,
transitionType,
transitionQualifiers,
);
if (candidates.length === 0) return;
let resolveTracked!: () => void;
const tracked = new Promise<void>((resolve) => {
resolveTracked = resolve;
});
for (const candidate of candidates) candidate.navigationCallbacks.add(tracked);
void (async () => {
try {
const recording = await findRecordingForTab(tabId, deps);
if (!recording || !recording.acceptingNavigation || !candidates.includes(recording)) {
if (result.kind === "coalesce_redirect") {
recording.observation?.scheduleRedirect(tabId, recording.steps, result.url);
return;
}
enqueueRecordingAction(recording, async () => {
const result = observeRecordedNavigation(
recording,
url,
causedByAction,
transitionType,
transitionQualifiers,
);
if (result.kind === "coalesce_redirect") {
recording.observation?.scheduleRedirect(tabId, recording.steps, result.url);
return;
}
if (result.kind === "noop") return;
recording.observation?.clearRedirect(tabId);
if (result.kind === "appended") {
await processRecordedStep(recording, result.index, tabId);
} else {
recording.observation?.scheduleSettle(tabId, recording.steps, result.index);
}
});
} finally {
for (const candidate of candidates) candidate.navigationCallbacks.delete(tracked);
resolveTracked();
}
})();
if (result.kind === "noop") return;
recording.observation?.clearRedirect(tabId);
if (result.kind === "appended") {
await processRecordedStep(recording, result.index, tabId);
} else {
recording.observation?.scheduleSettle(tabId, recording.steps, result.index);
}
});
const tracked = queued.catch(() => {});
recording.navigationCallbacks.add(tracked);
void tracked.finally(() => recording.navigationCallbacks.delete(tracked));
}
};
const onMainFrameComplete = (tabId: number, url?: string) => {
void (async () => {
@@ -651,7 +741,8 @@ export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()):
sendResponse({ active: false });
return;
}
await rearmRecording(recording, tabId, deps);
const activation = sender.tab?.active ? recording.tabs.noteActivation(tabId) : undefined;
await rearmRecording(recording, tabId, deps, activation);
sendResponse({
active: true,
requestId: recording.requestId,
@@ -716,7 +807,7 @@ async function finishRecordingAttempt(
if (!(await drainRecordingToStability(recording))) {
return null;
}
await recording.observation?.settleTrailing(recording.tabId, recording.steps);
await recording.observation?.settleTrailing(recording.tabs.currentTabId, recording.steps);
recording.settled = true;
recordings.delete(sessionId);
@@ -768,7 +859,7 @@ export async function handleRecordStart(
const redactValues = params.redact_values ?? false;
recordings.set(params.session_id, {
requestId,
tabId: target.tabId,
tabs: new RecordingTabCoordinator(target.tabId, navigateUrl),
agentWindowId: ctx.agentWindowId,
startUrl: navigateUrl,
...(params.purpose ? { purpose: params.purpose } : {}),
@@ -776,14 +867,12 @@ export async function handleRecordStart(
startedAt: new Date(startedAtMs).toISOString(),
startedAtMs,
traceVersion: traceVersionOrErr,
supportsTabSwitchSteps: params.supports_tab_switch_steps === true,
finishPromise,
resolveFinish,
rejectFinish,
settled: false,
finishAttempt: null,
currentUrl: navigateUrl,
pendingNavigation: false,
pendingNavigationDeadline: undefined,
observation:
traceVersionOrErr === 3 && deps.cdp
? new RecordingObservationRuntime({
@@ -799,12 +888,23 @@ export async function handleRecordStart(
actionQueue: Promise.resolve(),
lastStepSequenceByProducer: new Map(),
});
deps.frameCoordinator?.begin(requestId, startedAtMs, target.tabId, async (scope) => {
const recording = recordings.get(params.session_id);
if (!recording || recording.requestId !== requestId || recording.settled) return;
try {
await recording.observation?.refreshDocument(scope.tabId, scope.producerId);
} catch {
// The normal action-time capture remains available if a child Document
// is replaced while its readiness refresh is in flight.
}
});
// Observe navigations for the whole recording lifetime; attach before
// optional navigate so the destination load can rearm capture.
ensureBrowserObservationListeners(deps);
const abortPending = async (notifyContent: boolean) => {
recordings.get(params.session_id)?.observation?.cancel();
deps.frameCoordinator?.cancel(requestId);
recordings.delete(params.session_id);
releaseBrowserObservationListenersIfIdle();
if (notifyContent) {
@@ -886,7 +986,7 @@ export async function handleRecordStart(
return cancelledError();
}
active.startUrl = startUrl;
active.currentUrl = startUrl;
active.tabs.navigation(target.tabId, startUrl).currentUrl = startUrl;
if (isContentScriptRestrictedUrl(startUrl)) {
await abortPending(false);
@@ -921,6 +1021,10 @@ export async function handleRecordStart(
const startMsg: RecordStartMessage = { type: RECORD_START, requestId, startedAtMs };
try {
if (deps.frameCoordinator) {
const frameStarted = await deps.frameCoordinator.armTab(requestId, target.tabId);
if (!frameStarted) throw new Error("top recording document did not start");
}
await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
} catch {
await abortPending(true);
@@ -1040,6 +1144,7 @@ export function clearRecordingForSession(sessionId: string): void {
}
void clearRearmTimersForRecording(recording, getDefaultDeps());
if (!recording.settled) {
getDefaultDeps().frameCoordinator?.cancel(recording.requestId);
recording.observation?.cancel();
recording.settled = true;
recording.rejectFinish(new Error("recording cleared"));
@@ -0,0 +1,82 @@
import { describe, expect, it } from "vitest";
import {
RECORD_DOCUMENT_ATTRIBUTE,
recordingDocumentMarkerValue,
} from "@/shared/recording-document-identity";
import { projectRecordSafeObservation } from "../record-safe-observation";
describe("projectRecordSafeObservation", () => {
it("projects the opaque recording identity without exposing captured attributes", () => {
const recordingDocumentId = "123e4567-e89b-42d3-a456-426614174000";
const result = projectRecordSafeObservation({
rootFrameId: "child",
frameDocuments: [
{
frameId: "child",
target: { tabId: 4, sessionId: "oopif-session" },
contextScopeId: "scope-1",
axNodes: [],
domNodes: [
{
backendNodeId: 1,
parentBackendNodeId: null,
tag: "html",
attrs: {
[RECORD_DOCUMENT_ATTRIBUTE]: recordingDocumentMarkerValue(recordingDocumentId),
value: "user@example.com",
},
rect: null,
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
],
},
],
rendered: { text: '@vom 1\nRootWebArea "Example"', refs: [], truncated: false },
});
expect(result.frames).toEqual([
{
frameId: "child",
target: { tabId: 4, sessionId: "oopif-session" },
recordingDocumentId,
},
]);
const dumped = JSON.stringify(result);
expect(dumped).not.toContain("user@example.com");
expect(dumped).not.toContain("attrs");
expect(dumped).not.toContain("domNodes");
expect(dumped).not.toContain("axNodes");
});
it("does not project a page-controlled marker value as recording identity", () => {
const result = projectRecordSafeObservation({
rootFrameId: "root",
frameDocuments: [
{
frameId: "root",
target: { tabId: 4 },
contextScopeId: "scope-1",
axNodes: [],
domNodes: [
{
backendNodeId: 1,
parentBackendNodeId: null,
tag: "html",
attrs: { [RECORD_DOCUMENT_ATTRIBUTE]: "bsk:user@example.com" },
rect: null,
paintOrder: 0,
position: "static",
pointerEvents: "auto",
},
],
},
],
rendered: { text: '@vom 1\nRootWebArea "Example"', refs: [], truncated: false },
});
expect(result.frames).toEqual([{ frameId: "root", target: { tabId: 4 } }]);
expect(JSON.stringify(result)).not.toContain("user@example.com");
});
});
@@ -1,5 +1,6 @@
import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import { readRecordingDocumentIdentity } from "@/shared/recording-document-identity";
import type { CapturedSurfaceProbe } from "./capture";
import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture";
@@ -9,6 +10,8 @@ export interface CaptureVomFrame {
target: CdpTarget;
parentFrameId?: string;
ownerBackendNodeId?: number;
/** BrowserSkill-generated identity for the recording agent in this Document. */
recordingDocumentId?: string;
}
/** Allowlisted geometry used to match a recorded action to a rendered ref. */
@@ -37,14 +40,22 @@ export interface CaptureVomObservationResult {
}
function projectFrames(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomFrame[] {
return documents.map((document) => ({
frameId: document.frameId,
target: document.target,
...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}),
...(document.ownerBackendNodeId !== undefined
? { ownerBackendNodeId: document.ownerBackendNodeId }
: {}),
}));
return documents.map((document) => {
let recordingDocumentId: string | undefined;
for (const node of document.domNodes) {
recordingDocumentId = readRecordingDocumentIdentity(node.attrs);
if (recordingDocumentId) break;
}
return {
frameId: document.frameId,
target: document.target,
...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}),
...(document.ownerBackendNodeId !== undefined
? { ownerBackendNodeId: document.ownerBackendNodeId }
: {}),
...(recordingDocumentId ? { recordingDocumentId } : {}),
};
});
}
function projectMatchNodes(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomMatchNode[] {
+3
View File
@@ -780,6 +780,7 @@ export interface SelectedOptionV3 {
export type StepV3 =
| ({ op: "navigate" } & StepCommonV3 & { to: string; cause: NavigationCause })
| ({ op: "switch_tab" } & StepCommonV3)
| ({ op: "click" } & StepCommonV3 & { target: TargetDescriptorV3 })
| ({ op: "hover" } & StepCommonV3 & { target: TargetDescriptorV3 })
| ({ op: "fill" } & StepCommonV3 & {
@@ -823,6 +824,8 @@ export interface RecordStartParams {
redact_values?: boolean;
/** Omitted means v2; `3` requests a state-linked v3 trace. */
trace_version?: number;
/** Client can decode the v3 `switch_tab` step variant. */
supports_tab_switch_steps?: boolean;
}
export interface RecordStartResult {
+1
View File
@@ -115,6 +115,7 @@ fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError>
max_page_tokens: args.max_page_tokens,
redact_values: Some(args.redact_values),
trace_version: Some(TRACE_VERSION_V3),
supports_tab_switch_steps: Some(true),
};
let start_result = business_rpc::call::<RecordStartParams, RecordStartResult>(
info.sock_path.clone(),
@@ -480,6 +480,36 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
@@ -29,6 +29,13 @@
"session_id": {
"type": "string"
},
"supports_tab_switch_steps": {
"description": "Client can decode the v3 `switch_tab` step variant.",
"type": [
"boolean",
"null"
]
},
"tab_id": {
"type": [
"integer",
@@ -480,6 +480,36 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
+30
View File
@@ -469,6 +469,36 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
@@ -431,6 +431,36 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
@@ -41,6 +41,36 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
+30
View File
@@ -173,6 +173,36 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
+20
View File
@@ -39,6 +39,9 @@ pub struct RecordStartParams {
/// Desired trace export format. Omitted means v2; `3` requests v3.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub trace_version: Option<u32>,
/// Client can decode the v3 `switch_tab` step variant.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_tab_switch_steps: Option<bool>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
@@ -423,6 +426,19 @@ mod tests {
assert_eq!(round, step);
}
#[test]
fn step_switch_tab_round_trips() {
let step = StepV3::SwitchTab {
common: sample_common(2, "s2", "s5"),
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["op"], "switch_tab");
assert_eq!(v["state"], "s2");
assert_eq!(v["result"]["state"], "s5");
let round: StepV3 = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn trace_v3_round_trips() {
let trace = sample_trace();
@@ -463,10 +479,12 @@ mod tests {
max_page_tokens: None,
redact_values: None,
trace_version: None,
supports_tab_switch_steps: None,
};
let legacy_value = serde_json::to_value(legacy).unwrap();
assert!(legacy_value.get("trace_version").is_none());
assert!(legacy_value.get("max_page_tokens").is_none());
assert!(legacy_value.get("supports_tab_switch_steps").is_none());
let v3 = RecordStartParams {
session_id: "session".into(),
@@ -476,11 +494,13 @@ mod tests {
max_page_tokens: Some(4_000),
redact_values: Some(true),
trace_version: Some(TRACE_VERSION_V3),
supports_tab_switch_steps: Some(true),
};
let v3_value = serde_json::to_value(v3).unwrap();
assert_eq!(v3_value["trace_version"], TRACE_VERSION_V3);
assert_eq!(v3_value["max_page_tokens"], 4_000);
assert_eq!(v3_value["redact_values"], true);
assert_eq!(v3_value["supports_tab_switch_steps"], true);
}
#[test]
@@ -126,6 +126,10 @@ pub enum StepV3 {
to: String,
cause: NavigationCause,
},
SwitchTab {
#[serde(flatten)]
common: StepCommonV3,
},
Click {
#[serde(flatten)]
common: StepCommonV3,
@@ -340,6 +344,19 @@ mod tests {
assert_eq!(round, step);
}
#[test]
fn step_switch_tab_round_trips() {
let step = StepV3::SwitchTab {
common: sample_common(2, "s2", "s5"),
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["op"], "switch_tab");
assert_eq!(v["state"], "s2");
assert_eq!(v["result"]["state"], "s5");
let round: StepV3 = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn trace_v3_round_trips() {
let trace = sample_trace();
@@ -0,0 +1,108 @@
/**
* Archive-triggered bsk session cleanup. DSH conversation archival writes
* the workspace registry's durable global state, which the storage domain
* broadcasts as `domain/changed` ({domain: 'workspace', table: ''}) with
* the new `archivedSessionIds`. The watcher diffs that set and stops every
* bsk session owned by a freshly archived conversation — archived sessions
* are hidden from every surface, so their Agent Windows would otherwise
* linger with no way back to them.
*
* Ownership lineage: browser_session_start records the calling agent's
* session id PLUS every ancestor along `header.parentSession` (a subagent's
* browsers are reaped when any ancestor is archived, the root conversation
* included). Lineage is resolved through the host session store; an
* unloaded ancestor simply ends the walk.
*/
import type { Context } from "@deepseek-ai/cordis";
import type { ObservationService } from "./observation";
import type { SessionRegistry } from "./sessions";
/** The wire shape of a `domain/changed` frame (see dsh-storage-domain). */
interface DomainChange {
domain?: string;
table?: string;
value?: unknown;
}
/** The workspace registry's durable global singleton slice we read. */
interface WorkspaceGlobal {
archivedSessionIds?: unknown;
}
/** Structural host session store face: only the lineage read is needed. */
interface SessionStoreLike {
get(id: string): { header: { parentSession?: string } } | undefined;
}
/** Cap on lineage walks — defensive against a malformed parent chain. */
const MAX_LINEAGE_DEPTH = 16;
/**
* The DSH session ids that own a tool call's browser sessions: the calling
* agent's own session plus every ancestor along the seed lineage. Empty
* when the call carried no agent identity (those sessions outlive any
* archive cleanup by design — nothing can name their owner).
*/
export function ownerSessionIds(ctx: Context, agentId: string | undefined): string[] {
if (agentId === undefined) return [];
const store = ctx.get("sessions") as SessionStoreLike | undefined;
const ids: string[] = [];
let current: string | undefined = agentId;
for (let depth = 0; current !== undefined && depth < MAX_LINEAGE_DEPTH; depth += 1) {
if (ids.includes(current)) break; // a cycle in stored headers must not loop
ids.push(current);
current = store?.get(current)?.header.parentSession;
}
return ids;
}
/**
* Watch conversation archival and stop the bsk sessions it opened. Returns
* the disposer (plugin unload). In compositions without the workspace
* domain (headless), the event simply never fires — and a context without
* the events mixin degrades to a no-op like the other optional seams.
*/
export function armArchiveCleanup(
ctx: Context,
registry: SessionRegistry,
observation: ObservationService,
): () => void {
// 'domain/changed' lives outside the vendored Events type map, so the
// listener goes through a structural view of the events mixin.
const on = (
ctx as { on?: (event: string, listener: (change: DomainChange) => void) => () => void }
).on;
if (typeof on !== "function") return () => {};
/** Archived ids already accounted for; lazily seeded from the registry. */
let seen: Set<string> | undefined;
const initialize = (): Set<string> => {
if (seen === undefined) {
const registryService = ctx.get("workspaceRegistry") as
| { archivedSessionIds?: readonly string[] }
| undefined;
seen = new Set(registryService?.archivedSessionIds ?? []);
}
return seen;
};
return on.call(ctx, "domain/changed", (change: DomainChange) => {
if (change?.domain !== "workspace" || change?.table !== "") return;
const archived = (change.value as WorkspaceGlobal | undefined)?.archivedSessionIds;
if (!Array.isArray(archived)) return;
const previous = initialize();
const fresh = archived.filter(
(id): id is string => typeof id === "string" && !previous.has(id),
);
seen = new Set(archived.filter((id): id is string => typeof id === "string"));
for (const dshSessionId of fresh) {
for (const sessionId of registry.ownedByDsh(dshSessionId)) {
// stopSession owns the full teardown (kill in-flight tools, queue
// the daemon stop, drop registry + observation entries); a failure
// just leaves the session for idle timeout or unload cleanup.
void observation.stopSession(sessionId).catch(() => {});
}
}
});
}
@@ -30,17 +30,51 @@
color: var(--card-foreground);
}
/* Sidebar carrier: fill the better-sidebar tab area and let the body flex
(the tab content container owns the outer sizing). */
.sidebar-tab {
display: flex;
flex-direction: column;
height: 100%;
min-height: 0;
}
.sidebar-tab > * {
flex: 1;
min-height: 0;
}
/* The BSK product mark in the sidebar tab strip (the source artwork already
carries rounded corners — just soften the square to match). */
.brand-icon {
display: block;
border-radius: 22%;
}
.header {
display: flex;
align-items: center;
gap: 6px;
padding: 6px 8px;
cursor: move;
user-select: none;
touch-action: none;
border-bottom: 1px solid var(--border);
}
/* Only the floating card drags by its header — the sidebar tab does not. */
.header[data-draggable] {
cursor: move;
}
/* In the sidebar the status row is chrome, not a card edge: drop its bottom
border. A fixed-height bordered row can never track the shell header's
hairline across the panel boundary (its height varies per window), and
the BSK warm border tint clashes with the sidebar's neutral hairlines —
borderless, the stage's own background provides the separation. */
.sidebar-tab .header {
border-bottom: none;
}
.status-text {
flex: 1;
min-width: 0;
@@ -131,6 +165,12 @@
border-top: 1px solid var(--border);
}
.actions-group {
display: inline-flex;
align-items: center;
gap: 4px;
}
.tool-wrap {
position: relative;
display: inline-flex;
@@ -165,8 +205,14 @@
color: var(--destructive);
}
/* Hover the wrap (not just :hover on the button) so the tooltip target and
the left-corner resize overlay still light the icon up. */
/* Stop session: danger-tinted like interrupt. */
.tool-stop {
color: var(--destructive);
}
/* Hover the wrap (not just :hover on the button) so the tooltip target
lights the icon up too. The corner resize handles sit above this bar, so
the outer sliver of the corner-most buttons starts a resize instead. */
.tool-wrap:hover .tool-button:not(:disabled) {
color: var(--foreground);
background: var(--accent);
@@ -181,6 +227,15 @@
background: var(--accent);
}
/* The armed (confirm) state of the stop button: solid destructive fill so
the second click reads as final. Written at the wrap-hover specificity so
the hover rules above never wash it out. */
.tool-button.tool-stop-armed,
.tool-wrap:hover .tool-button.tool-stop-armed {
color: var(--destructive-foreground);
background: var(--destructive);
}
.hint {
position: absolute;
bottom: calc(100% + 4px);
@@ -203,13 +258,16 @@
right: 0;
}
/* Invisible hit targets — no grip glyph; all four corners resize. */
/* Invisible hit targets — no grip glyph; all four corners resize. Must sit
above the bottom .actions bar (z-index 3): it spans the full card width
and would otherwise swallow every pointer event aimed at the sw/se
corners. The top corners already win because .header is not positioned. */
.resize-handle {
position: absolute;
width: 16px;
height: 16px;
touch-action: none;
z-index: 2;
z-index: 4;
}
.resize-handle[data-corner="nw"] {
@@ -3,13 +3,18 @@
// composer's way); collapsed = a status capsule; "pop out" upgrades the same
// content to a native Document PiP window (user gesture required by the
// browser). Multi-session renders a meeting-style strip under the focus view.
// Visuals follow the BrowserSkill product family: @browser-skill/ui components
// and oklch tokens on a .bsk-obs scope root, so the card reads as BSK's own
// surface without leaking styles into (or inheriting themes from) the shell.
// When the dsh-better-sidebar plugin is installed the tracking view moves
// into a sidebar tab instead (see observation-sidebar.tsx) and this floating
// card hides itself via the sidebar-mode flag. Visuals follow the
// BrowserSkill product family: @browser-skill/ui components and oklch tokens
// on a .bsk-obs scope root, so the card reads as BSK's own surface without
// leaking styles into (or inheriting themes from) the shell.
import { cn } from "@browser-skill/ui";
import {
RiArrowDownSLine,
RiCheckLine,
RiCloseCircleLine,
RiErrorWarningLine,
RiPictureInPicture2Line,
RiPushpinFill,
@@ -25,6 +30,8 @@ const IconPip = asIcon(RiPictureInPicture2Line);
const IconDown = asIcon(RiArrowDownSLine);
const IconWarn = asIcon(RiErrorWarningLine);
const IconPin = asIcon(RiPushpinFill);
const IconCloseSession = asIcon(RiCloseCircleLine);
const IconCheck = asIcon(RiCheckLine);
import {
type ReactNode,
@@ -39,11 +46,12 @@ import { createPortal } from "react-dom";
import type { SessionObservation } from "../observation";
import css from "./ObservationOverlay.module.css";
import type { ObservationClientStore } from "./observation-store";
import { focusOf, statusOf, useObservationView, usePip } from "./observation-view";
import { getSidebarMode, subscribeSidebarMode } from "./sidebar-mode";
/** Minimal Document PiP surface (TS lib.dom lacks it). */
interface DocumentPip {
requestWindow(options?: { width?: number; height?: number }): Promise<Window>;
}
// The pure view helpers live in observation-view (shared with the sidebar
// tab); re-export so existing imports of this module keep working.
export { focusOf, statusOf };
interface Point {
x: number;
@@ -61,11 +69,6 @@ const EDGE_MARGIN = 16;
/** Default dock: top-right, just under the shell's top bar (no spacing tokens exist in dsh 0.1). */
const TOP_OFFSET = 64;
function pipApi(): DocumentPip | undefined {
if (typeof window === "undefined") return undefined;
return (window as unknown as { documentPictureInPicture?: DocumentPip }).documentPictureInPicture;
}
function clampSize(size: Size, viewport: Size): Size {
const maxW = viewport.w * 0.8;
const maxH = viewport.h * 0.8;
@@ -114,23 +117,6 @@ function formatElapsed(sinceMs: number, nowMs: number): string {
return `${mm}:${ss}`;
}
/**
* Auto-follow focus: the most recently touched session, but never yank focus
* to a session whose latest action failed (errors flag the strip item, they
* do not steal the stage) or one already reported dead.
*/
export function focusOf(sessions: readonly SessionObservation[]): SessionObservation | undefined {
if (sessions.length === 0) return undefined;
const byRecency = [...sessions].sort((a, b) => b.since - a.since);
const healthy = byRecency.find((s) => s.lastError === undefined && s.dead !== true);
return healthy ?? byRecency[0];
}
export function statusOf(obs: SessionObservation): "active" | "idle" | "error" {
if (obs.lastError !== undefined && obs.action === "idle") return "error";
return obs.action === "idle" ? "idle" : "active";
}
/** Compact toolbar icon: no label, hover bubble for the name / semantics. */
function IconAction(props: {
label: string;
@@ -166,6 +152,83 @@ function IconAction(props: {
);
}
/** How long the stop button stays armed before the confirm click expires. */
const STOP_ARM_MS = 3000;
/**
* Stop-session button with a lightweight two-click confirm: the first click
* arms the button (solid red, check icon, short expiry), the second executes
* the stop. No dialog, no layout shift — and a stray single click can never
* close an Agent Window.
*/
function StopSessionAction(props: {
sessionId: string | undefined;
onStop: (sessionId: string) => Promise<boolean>;
}) {
const { sessionId, onStop } = props;
const [hover, setHover] = useState(false);
const [armed, setArmed] = useState(false);
const [stopping, setStopping] = useState(false);
useEffect(() => {
if (!armed) return;
const timer = setTimeout(() => setArmed(false), STOP_ARM_MS);
return () => clearTimeout(timer);
}, [armed]);
// The focused session can vanish mid-arm (stopped elsewhere): disarm.
useEffect(() => {
if (sessionId === undefined) setArmed(false);
}, [sessionId]);
const disabled = sessionId === undefined || stopping;
const label = stopping
? "Stopping session…"
: armed
? `Confirm stop session ${sessionId ?? ""}`
: `Stop session ${sessionId ?? ""}`;
const hint = stopping
? "Closing the Agent Window…"
: armed
? `Click again to stop session ${sessionId ?? ""} and close its Agent Window.`
: `Stop session ${sessionId ?? ""} and close its Agent Window.`;
return (
<span
className={css["tool-wrap"]}
onPointerEnter={() => setHover(true)}
onPointerLeave={() => setHover(false)}
>
<button
type="button"
className={cn(css["tool-button"], css["tool-stop"], armed && css["tool-stop-armed"])}
disabled={disabled}
aria-label={label}
aria-pressed={armed}
data-armed={armed || undefined}
onClick={() => {
if (sessionId === undefined || stopping) return;
if (!armed) {
setArmed(true);
return;
}
setArmed(false);
setStopping(true);
const target = sessionId;
void onStop(target).finally(() => setStopping(false));
}}
>
{armed ? <IconCheck size={16} /> : <IconCloseSession size={16} />}
</button>
{hover ? (
<span className={css.hint} role="tooltip">
{hint}
</span>
) : null}
</span>
);
}
/** Flat status dot, specced after the BSK popup's ConnectionStatusIndicator. */
function StatusDot({ state }: { state: "active" | "idle" | "error" | "dead" }) {
const color =
@@ -246,8 +309,8 @@ function StripItem(props: {
);
}
/** The floating card / PiP shared content. */
function OverlayBody(props: {
/** The floating card / PiP / sidebar-tab shared content. */
export function OverlayBody(props: {
store: ObservationClientStore;
focus: SessionObservation | undefined;
sessions: readonly SessionObservation[];
@@ -310,6 +373,7 @@ function OverlayBody(props: {
<div
className={css.header}
data-testid="obs-header"
data-draggable={onHeaderPointerDown !== undefined || undefined}
onPointerDown={onHeaderPointerDown}
role="presentation"
>
@@ -363,15 +427,21 @@ function OverlayBody(props: {
</div>
) : null}
<div className={css.actions}>
<IconAction
label={interrupting ? "Interrupting…" : "Interrupt the current browser action"}
hint="Stop the current browser action."
disabled={!canInterrupt}
danger
onClick={onInterrupt}
>
<IconStop size={16} />
</IconAction>
<div className={css["actions-group"]}>
<IconAction
label={interrupting ? "Interrupting…" : "Interrupt the current browser action"}
hint="Stop the current browser action."
disabled={!canInterrupt}
danger
onClick={onInterrupt}
>
<IconStop size={16} />
</IconAction>
<StopSessionAction
sessionId={focus?.sessionId}
onStop={(sessionId) => store.stopSession(sessionId)}
/>
</div>
{onPopOut !== undefined ? (
<IconAction
label="Pop out into a mini window"
@@ -387,25 +457,17 @@ function OverlayBody(props: {
);
}
/** Clone the host document's style/link nodes into a PiP window. */
function cloneStylesInto(pipWindow: Window): void {
for (const node of document.querySelectorAll('link[rel="stylesheet"], style')) {
pipWindow.document.head.appendChild(node.cloneNode(true));
}
}
export function ObservationOverlay({ store }: { store: ObservationClientStore }) {
const snapshot = useSyncExternalStore(store.subscribe, store.getSnapshot);
useEffect(() => {
store.start();
return () => store.stop();
}, [store]);
const { snapshot, focus, pinnedId, onTogglePin, now } = useObservationView(store);
// While the better-sidebar plugin carries the tracking view, this floating
// card (and its capsule) stays out of the way; an already-open PiP window
// keeps its portal until the user closes it.
const sidebarMode = useSyncExternalStore(subscribeSidebarMode, getSidebarMode);
const [collapsed, setCollapsed] = useState(false);
const [pos, setPos] = useState<Point | null>(null);
const [size, setSize] = useState<Size>(DEFAULT_SIZE);
const [pipWindow, setPipWindow] = useState<Window | null>(null);
const [pinnedId, setPinnedId] = useState<string | null>(null);
const { pipWindow, pipSupported, popOut } = usePip();
const dragRef = useRef<{
kind: "move" | "resize";
corner?: ResizeCorner;
@@ -414,23 +476,6 @@ export function ObservationOverlay({ store }: { store: ObservationClientStore })
base: Point & Size;
} | null>(null);
// Elapsed-time ticker: 1s while anything is active.
const anyActive = snapshot.sessions.some((s) => s.action !== "idle");
const [now, setNow] = useState(() => Date.now());
useEffect(() => {
if (!anyActive) return;
const timer = setInterval(() => setNow(Date.now()), 1000);
return () => clearInterval(timer);
}, [anyActive]);
// Focus: the pinned session wins while it still exists; otherwise auto-follow.
const pinned =
pinnedId !== null ? snapshot.sessions.find((s) => s.sessionId === pinnedId) : undefined;
const focus = pinned ?? focusOf(snapshot.sessions);
const onTogglePin = useCallback((sessionId: string) => {
setPinnedId((current) => (current === sessionId ? null : sessionId));
}, []);
const viewport = (): Size => ({ w: window.innerWidth, h: window.innerHeight });
const onPointerMove = useCallback((event: PointerEvent) => {
@@ -511,34 +556,21 @@ export function ObservationOverlay({ store }: { store: ObservationClientStore })
document.addEventListener("pointerup", onPointerUp);
};
const popOut = async (): Promise<void> => {
const pip = pipApi();
if (pip === undefined) return;
try {
const win = await pip.requestWindow({ width: size.w, height: size.h });
cloneStylesInto(win);
win.addEventListener("pagehide", () => setPipWindow(null));
setPipWindow(win);
} catch {
// requestWindow rejects without a user gesture (or when the window was
// closed mid-request): stay on the in-page card, no state change.
}
};
// Hidden while no owned session exists (and no PiP is up).
if (snapshot.sessions.length === 0 && pipWindow === null) return null;
const body = (
<OverlayBody
store={store}
focus={focus}
sessions={snapshot.sessions}
available={snapshot.available}
pinnedId={pinned !== undefined ? pinnedId : null}
pinnedId={pinnedId}
onTogglePin={onTogglePin}
now={now}
inPip={pipWindow !== null}
onPopOut={pipWindow === null && pipApi() !== undefined ? () => void popOut() : undefined}
onPopOut={
pipWindow === null && pipSupported
? () => popOut({ width: size.w, height: size.h })
: undefined
}
onCollapse={pipWindow === null ? () => setCollapsed(true) : undefined}
onHeaderPointerDown={pipWindow === null ? beginMove : undefined}
/>
@@ -548,6 +580,12 @@ export function ObservationOverlay({ store }: { store: ObservationClientStore })
return createPortal(body, pipWindow.document.body);
}
// The sidebar tab is the carrier now — no floating card, no capsule.
if (sidebarMode) return null;
// Hidden while no owned session exists (and no PiP is up).
if (snapshot.sessions.length === 0) return null;
if (collapsed) {
const state = focus !== undefined ? statusOf(focus) : "idle";
return (
@@ -0,0 +1,8 @@
/**
* The BrowserSkill product mark (apps/extension/assets/logo.png, downscaled
* to 32px and inlined): the sidebar tab icon, so the tracking view reads as
* BSK's own surface next to better-sidebar's built-in "browser" tab.
* Regenerate with: resize the source to 32x32 PNG and replace the payload.
*/
export const BSK_LOGO_URL =
"data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACAAAAAgCAYAAABzenr0AAAJGklEQVR42m2X248c2V3HP79zTlV19/TcPOOZWY+vsdmQFdngTQJswgs3hQ15iCJslAey+8xDAIHgH+CBNxIpj0gkEXnAKyKQWITEJlKWXSssEMfJRmuMHa/W9njHMz3Tl+nprsv5/XiomlugpdN16lR1n9/1e75fAbAbeLlO3Pz2ly4vjO7+sRbF58oiP49FEREOPwaIYQjS3IvUy0fP66kAZoYJIM7SpPVY0uy14cyzXzn9u9+4c+MG/vp1otiNa16uvxr7X/+1L3bK3lcTyU9P80ilR/uKCGYHOxxd6s2kmdaGIVYb1qyY1e8HD63UUblstwirf9J95fW/sRvXvAD0v/6Z6/O68Xf5/pS8kigiTsTkuCfIwfzAxQN37chjaAyt594JIXE4gbKMhog6Nd/qthi7lVe6L3/nG7Jz4/fPJ9u3b4U4WSwiKog/EcsTLh+PSuO3GXYs/D44QnBgyniQs7OV0+8bOxPPM6c9l660VEyF0Nmrlp9/IYT+/S/PUJ4a5C46EW8HnjVhBKjXaHJfz9Xq95xzhMThHZR5xXBrwtPNnP6eRzvzhJWzrF1d5UI759//9iYra5lrt1ycDeXsaOf+n4Y4HX8+r9SciasLS/6P12LHoyF47wipYKpM9gqePpyyvVMxjBn+1DILH17l0rkZlhYcLetT3n/A5FGfTtvTCh4ibjpVI81/J0il62V1rNStLp6j4jowxwiJh2iMevtsbU3pjYQim6Nz9hJLH13i8mrGbJgSxjuUvQfEjT2GOw4Zddh0BakPtFPH/jRKqaCSrwetKi9mKIIYhyFuOg6TurCcF7ae5ty5t48/vcTiz3+Ei+szLC06WnGA7TykfNCnmObk5nAtj+53yDcTZteMwZOChbkMVa3/14DCJBBBFBxyopndUUBqwwxu/XjAJ778Bc4ul+R3fgyj9yg/GDMpFZxHgoe0hQ9QDRzFI48zqHzJeKhcvNKiLAyiQ1BwgsOcqNZ9Xo96rgaVgmqdiBACH740y/1/+j4b7w1x80tUkwLM4VptJAQMcN6o+p7iYQIqhBQmsUIrR7cd0NI12ODAwJnWTpvZoRGqdRrS1CECeR7Z2NzHRBjeecgbX/kut39UUlx5ATe3gFUlAM5Tb/4oHCJl0oadUcFsK8E5sFh7ZaqgEFBr3KyhxAySxDMplDv3BuztlmhhZC6QqONDrXkW28Kbf/1d7A9+i6vPr7N/e5vQDZQ7jvxxUqdPwFRwHaP3sGR1boZYGabWwHdd7MHMMBUMh5mRJp6tnZx33tllJbS53J1jZj6BKpKPpkhZ8e5mj3Nf/FV+8ZOnmNz6Pr6TUvUcxeOAc0cN6xwUoWIyUhZXMqqq6bKmrQ0hWBQzdXX+nGOwV3LnJ30+trjE8mxGUUVG/Qn7owkt7/hpb5fwmat84qXz5P95EwmOqh8oNkKTSkFqICTJYDgpCOppZYGyjIfgdoApQRScCtEgDZ5b93dxuWd3OuWD8T5pqbSLyEyWcHdrh+S3r/Kpz15i/z/eQhIh9hOKjQRTyBYgFhAndSpDW+gNC+ZbGR6jjM3BVsNsXQNqhqpiOPIicuXcLFvdKT7xtNXTG+X0Bopt9Wi9dJUXX7rI5O23kNQRB4F8I0CEzgr0/B7t2YB/2iJOQDrK8HHJ+dkZqqI+JfUAYRXMGc6pM1MBBa2Ubttz5ews51c6XFjr8OJHT5N0HPIbz/Ppz11h/+03IXVUg4T8ccAidFaNp7LHzdd2uXdvj866gQq5L8nHMD+TUJUG0aAyJBqidZQcUfFqeK2vlEo1rYh5iUXl6eMB5eoKv37tOUb/9q+4LBD7gXKj3ry7Ak/iHrdeH/ArC7P07ykPekMWzkFvkNMmkHkgagNoDciogYKzaGg0YgSNYLEJj0ndfiGj6o/Z3i5or5+j2BTKJwlawcwKbMQ9fvK9Ib+QtgitQMg8d384wVYLdocls1mCad1+Fg1TDgeV4UxFTB2ow44NjQ5rwcwZ+MjqmLe/dZOBLOHHCeUUZteED+Ie73xvyHNpmyx1TERozXieW1/mO/+yzWhTWVtsU5aCqKvRT4+GmRDEHGJ6yPPMBHNGeqbCdwqs02X9l14g/a8H3PzWm/zyp1ZZXgn8tDfgnbeGPN/qIEDSyhh7w2OcPZWRsESWCq3EoVFPksZDMuUIps7qLjDUQBxkZ0tcu8KSGdIXPo3OLbOSdfj4YMIPbg6ZXxux9wRmWimFKe0k0J7v8H5vQHchpSoiC536bKhKEDlCJzM7JLJqhqtxoEYoUcEFw/Yduh8Jl38OP7eAlBOm29ssziV87JlluqMOL15ZZWVlhu0yMrfYxQQmZUU3TbCmnog1mRE1xOp9xOoaEwVRI9QnUGOECLoPNlcRlucJ6xeIgx3K/3kX23pINWnRdnBxtYNWkbkkMJ5JSLKEwaSgRFlsJWh1mNBDfsEBzZGaYIiBIgRtImCmmArhlBFOR1Q9xZ3b6OOHWFnifEq1U/84TpXgHL4hUh7hv3t91pY6eCeUpTYs2o6RisYEO6B5hpoSRCWaanNAGP5UVZ+r+YB4dxcbZ4QVx/Q9wSYg/uDYNvKyoorwgyfbtDuBC6c65HmFa5Jsh56f3NgO4iGeYC68n8Ty2WkphAUVv2hoLuggoXgUcJkRh0rVF7wHi3Vw1YxuSMh1zMpcmw8tdynzCmmCL1JjzQG3lpNU37xAdNnjYC79xywUfzbJNboEX214iqeeuN/IrkKw6BAvNUiJ1a2K0HKOT144jQBFoTWRayrc9LhWa1TTUTNqJ038wCf/LO//xZfWWw9u/zCt8uWpSiTixZ3UeJzQPkerdkyq1RryZwVN87CJBALR0ExULMtG4/VnPy4AG3/4m19YGG/9fT7NmZpEhzg5lCNHOo//774u7UYp2WEt1XrSftYMTcF32ynbrfmXz3ztjW86u3bNn/nq69/eaZ/6PUlbm/PB+YSapUozMG2uB0M5fI7WSKoKasjBu6qHczEjwWQ2iJc03XmaLb5y5mtvfNOuNeLUruHlVeK7f/7yxbXBu39U5sXnq7I8L2pH6rwRgCZyUpeKnZSOB5K2EbT1l5hPk0c+zV7bXXzmry7/5T/cvXENf/1V4v8CSYIrURx3kHgAAAAASUVORK5CYII=";
@@ -1,7 +1,9 @@
/**
* dsh-plugin-browserskill browser half: the `browser_screenshot` keyed
* toolview plus the observation overlay (live thumbnails + interrupt) floating
* over the shell via the `shell.overlay` seat.
* toolview plus the observation overlay (live thumbnails + interrupt). The
* default carrier is a floating card on the `shell.overlay` seat; when the
* dsh-better-sidebar plugin provides its `betterSidebar` service, the view
* moves into a sidebar tab instead (see observation-sidebar.tsx).
*/
import type { ImageAttachmentRef } from "@deepseek-ai/dsh-attachment";
@@ -15,6 +17,7 @@ import { createElement } from "react";
import "./bsk-tokens.nomodule.css";
import "./bsk-ui.nomodule.css";
import { ObservationOverlay } from "./ObservationOverlay";
import { type BetterSidebarLike, registerObservationSidebar } from "./observation-sidebar";
import { type EventSourceLike, ObservationClientStore } from "./observation-store";
import { ScreenshotToolView } from "./ScreenshotToolView";
@@ -81,4 +84,14 @@ export function apply(ctx: ClientContext): void {
createElement(ObservationOverlay, { store }),
),
);
// Optional carrier upgrade: when the dsh-better-sidebar plugin is installed,
// its service moves the tracking view into a sidebar tab (the floating
// overlay hides itself through the sidebar-mode flag). In profiles without
// the sidebar plugin this fiber never runs and nothing changes.
ctx.inject(["betterSidebar"], (injected) =>
registerObservationSidebar(
(injected as unknown as { betterSidebar: BetterSidebarLike }).betterSidebar,
store,
),
);
}
@@ -0,0 +1,256 @@
// better-sidebar carrier for the observation view. When the
// dsh-better-sidebar plugin is installed its client publishes a
// `betterSidebar` cordis service; the client entry then runs the
// registration below and the tracking view moves from the floating card
// into a single-instance sidebar tab (Document PiP pop-out stays available
// from inside the tab). Detection is purely service-based — profiles
// without the sidebar plugin never start this fiber and keep the floating
// overlay.
import { createElement, type ReactNode } from "react";
import { createPortal } from "react-dom";
import { BSK_LOGO_URL } from "./brand-icon";
import { OverlayBody } from "./ObservationOverlay";
import css from "./ObservationOverlay.module.css";
import type { ObservationClientStore } from "./observation-store";
import { useObservationView, usePip, visibleToScope } from "./observation-view";
import { setSidebarMode } from "./sidebar-mode";
/** The tab title — "Browser Skill", distinct from the sidebar's built-in "browser" tab. */
const TAB_TITLE = "Browser Skill";
/** Tab strip icon: the BrowserSkill product mark at the requested size. */
function TabIcon({ size }: { size: number }) {
return (
<img
src={BSK_LOGO_URL}
width={size}
height={size}
alt=""
aria-hidden
className={css["brand-icon"]}
/>
);
}
/**
* Structural mirrors of dsh-better-sidebar's client service surface (only
* the slices this integration touches — see the upstream
* lib/types/client/service.d.ts). Declared locally so the plugin carries no
* dependency on the sidebar package; the service contract has been stable
* since v0.4.0 and newer capabilities arrive behind its `features` list.
*/
export interface SidebarTabLike {
id: string;
type: string;
title: string;
}
export interface SidebarLeafLike {
kind: "leaf";
id: string;
tabs: SidebarTabLike[];
active: string | null;
}
export interface SidebarSplitLike {
kind: "split";
id: string;
dir: "row" | "col";
sizes: number[];
children: SidebarNodeLike[];
}
export type SidebarNodeLike = SidebarLeafLike | SidebarSplitLike;
export interface SidebarStateLike {
splits: SidebarNodeLike;
bottomSplits: SidebarNodeLike;
/** Whether the right panel is expanded (the merged drawer on narrow screens). */
panelOpen?: boolean;
}
export interface SidebarSnapshotLike {
sessionId?: string;
state?: SidebarStateLike;
}
/** Props every tab component receives (the slices we read). */
export interface TabComponentPropsLike {
/** The conversation this sidebar instance belongs to. */
scope: { sessionId: string };
}
export interface TabDescriptorLike {
id: string;
title: string | (() => string);
icon?: ReactNode | ((size: number) => ReactNode);
order?: number;
/** Single-instance: opening focuses the existing tab instead of duplicating. */
single?: boolean;
/** Small pill on the tab strip; null/undefined hides it. */
badge?: (
ctx: unknown,
scope: { sessionId: string },
state: unknown,
) => string | number | null | undefined;
component: (props: TabComponentPropsLike) => ReactNode;
}
export interface BetterSidebarLike {
registerTab(descriptor: TabDescriptorLike): () => void;
openTab(seed: {
type: string;
title?: string;
/** Content seed (lands on tab.path); content opens expand the panel. */
path?: string;
}): void;
getSnapshot(): SidebarSnapshotLike;
/** Sidebar state feed (v0.12+): session switches, state and prefs changes. */
subscribeState?: (listener: () => void) => () => void;
isTabEnabled(id: string): boolean;
}
/** The registered tab type id (also the SidebarTab.type value). */
export const OBSERVATION_TAB_TYPE = "browserskill:observation";
/**
* Inert content seed carried on auto-opened tabs: its mere presence makes
* the sidebar treat the open as a content open (expanding the hosting panel
* so the tracking view lands in sight). Never read by our component.
*/
export const OBSERVATION_TAB_PATH = "browser-skill:observation";
/**
* The observation tab body: the same OverlayBody the floating card renders,
* minus the card chrome (no drag header, no collapse — the sidebar tab bar
* owns those), plus the PiP pop-out upgrade. The view is scoped to the
* sidebar's conversation: only browser sessions started by it (or its
* descendants) show here — the floating card keeps the global view.
*/
export function ObservationSidebarTab({
store,
scopeId,
}: {
store: ObservationClientStore;
scopeId: string;
}) {
const { snapshot, focus, pinnedId, onTogglePin, now } = useObservationView(store, scopeId);
const { pipWindow, pipSupported, popOut } = usePip();
const body = (
<OverlayBody
store={store}
focus={focus}
sessions={snapshot.sessions}
available={snapshot.available}
pinnedId={pinnedId}
onTogglePin={onTogglePin}
now={now}
inPip={pipWindow !== null}
onPopOut={pipWindow === null && pipSupported ? () => popOut() : undefined}
/>
);
if (pipWindow !== null) {
return createPortal(body, pipWindow.document.body);
}
return <div className={css["sidebar-tab"]}>{body}</div>;
}
function* leafNodes(node: SidebarNodeLike): Generator<SidebarLeafLike> {
if (node.kind === "leaf") {
yield node;
return;
}
for (const child of node.children) yield* leafNodes(child);
}
/** Whether a tab of our type is already open in either sidebar workbench. */
export function observationTabOpen(state: SidebarStateLike | undefined): boolean {
if (state === undefined) return false;
for (const root of [state.splits, state.bottomSplits]) {
for (const leaf of leafNodes(root)) {
if (leaf.tabs.some((tab) => tab.type === OBSERVATION_TAB_TYPE)) return true;
}
}
return false;
}
/**
* Register the observation sidebar tab and flip the carrier flag. Returns
* the disposer the cordis fiber invokes when the sidebar service goes away
* (plugin unload/HMR): the floating overlay then resumes as the carrier.
*/
export function registerObservationSidebar(
service: BetterSidebarLike,
store: ObservationClientStore,
): () => void {
setSidebarMode(true);
// Hold the feed for the whole sidebar lifetime so the auto-open watcher
// sees new sessions even while the tab itself is closed.
store.acquire();
const disposeTab = service.registerTab({
id: OBSERVATION_TAB_TYPE,
title: TAB_TITLE,
icon: (size: number) => createElement(TabIcon, { size }),
single: true,
// The badge counts only sessions visible to the tab strip's own
// conversation (global sessions from other conversations stay hidden,
// mirroring the tab body's scoped view).
badge: (_ctx, scope) => {
const count = store
.getSnapshot()
.sessions.filter((s) => visibleToScope(s, scope.sessionId)).length;
return count > 0 ? count : null;
},
component: (props) =>
createElement(ObservationSidebarTab, { store, scopeId: props.scope.sessionId }),
});
// Auto-open the tab when a browser session VISIBLE TO the active
// conversation appears (and right away when one is already live). The
// evaluation runs on observation publishes AND on sidebar state changes:
// the latter covers the sidebar store's async init (its state is
// undefined for the first beats after page load) and conversation
// switches. The open carries a content seed (`path`): only content opens
// land in sight — the sidebar expands the hosting panel for them. While
// the panel is OPEN an existing tab is never re-focused — the user may
// be reading another page on purpose; while it is collapsed a NEW
// session (0→N) nudges the tab back into sight, mirroring how the
// floating card used to reappear.
const activeVisibleCount = (): number => {
const activeId = service.getSnapshot().sessionId;
if (activeId === undefined) return 0;
return store.getSnapshot().sessions.filter((s) => visibleToScope(s, activeId)).length;
};
let previousVisible = activeVisibleCount();
const evaluate = (): void => {
const count = activeVisibleCount();
const { state, sessionId } = service.getSnapshot();
if (
state !== undefined &&
sessionId !== undefined &&
service.isTabEnabled(OBSERVATION_TAB_TYPE)
) {
const open = observationTabOpen(state);
if (count > 0 && !open) {
service.openTab({ type: OBSERVATION_TAB_TYPE, path: OBSERVATION_TAB_PATH });
} else if (previousVisible === 0 && count > 0 && open && state.panelOpen === false) {
service.openTab({ type: OBSERVATION_TAB_TYPE, path: OBSERVATION_TAB_PATH });
}
}
previousVisible = count;
};
evaluate();
const unsubscribe = store.subscribe(evaluate);
const unsubscribeState = service.subscribeState?.(evaluate);
return () => {
unsubscribe();
unsubscribeState?.();
disposeTab();
store.release();
setSidebarMode(false);
};
}
@@ -53,6 +53,7 @@ export interface OverlaySnapshot {
const STATE_URL = "/bsk-observation/state";
const EVENTS_URL = "/bsk-observation/events";
const INTERRUPT_URL = "/bsk-observation/interrupt";
const STOP_URL = "/bsk-observation/stop";
function revoke(url: string | undefined): void {
if (url !== undefined && typeof URL.revokeObjectURL === "function") {
@@ -78,6 +79,8 @@ export class ObservationClientStore {
};
private available = true;
private started = false;
/** Refcount of mounted consumers (overlay card, sidebar tab, sidebar fiber). */
private consumers = 0;
constructor(private readonly deps: ObservationClientDeps) {}
@@ -189,6 +192,23 @@ export class ObservationClientStore {
this.publish();
}
/**
* Hold the feed for one consumer's lifetime: the stream starts with the
* first holder and stops with the last release. Several carriers can share
* the store (the floating card, the better-sidebar tab, and the sidebar
* integration fiber) without one unmount killing the others' updates.
*/
acquire(): void {
this.consumers += 1;
if (this.consumers === 1) this.start();
}
release(): void {
if (this.consumers === 0) return;
this.consumers -= 1;
if (this.consumers === 0) this.stop();
}
/** Forget one session's tracked + held frames, revoking their blob URLs. */
private dropThumb(sessionId: string): void {
const attachmentId = this.thumbBySession.get(sessionId);
@@ -292,4 +312,23 @@ export class ObservationClientStore {
return false;
}
}
/**
* Stop one session and close its Agent Window. The session's removal
* arrives through the SSE remove event — no local state is touched here.
*/
async stopSession(sessionId: string): Promise<boolean> {
try {
const res = await this.deps.fetchFn(STOP_URL, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ sessionId }),
});
if (!res.ok) return false;
const body = (await res.json()) as { stopped?: boolean };
return body.stopped === true;
} catch {
return false;
}
}
}
@@ -0,0 +1,148 @@
/**
* Shared view logic for the observation carriers (the floating overlay card
* and the better-sidebar tab): the store-backed view model (snapshot, focus
* pinning, elapsed ticker) and the Document PiP pop-out. Extracted from
* ObservationOverlay so both carriers run the same focus/interrupt behavior
* without duplicating hooks.
*/
import { useCallback, useEffect, useState, useSyncExternalStore } from "react";
import type { SessionObservation } from "../observation";
import type { ObservationClientStore, OverlaySnapshot } from "./observation-store";
/**
* Auto-follow focus: the most recently touched session, but never yank focus
* to a session whose latest action failed (errors flag the strip item, they
* do not steal the stage) or one already reported dead.
*/
export function focusOf(sessions: readonly SessionObservation[]): SessionObservation | undefined {
if (sessions.length === 0) return undefined;
const byRecency = [...sessions].sort((a, b) => b.since - a.since);
const healthy = byRecency.find((s) => s.lastError === undefined && s.dead !== true);
return healthy ?? byRecency[0];
}
export function statusOf(obs: SessionObservation): "active" | "idle" | "error" {
if (obs.lastError !== undefined && obs.action === "idle") return "error";
return obs.action === "idle" ? "idle" : "active";
}
/** Minimal Document PiP surface (TS lib.dom lacks it). */
export interface DocumentPip {
requestWindow(options?: { width?: number; height?: number }): Promise<Window>;
}
export function pipApi(): DocumentPip | undefined {
if (typeof window === "undefined") return undefined;
return (window as unknown as { documentPictureInPicture?: DocumentPip }).documentPictureInPicture;
}
/** Clone the host document's style/link nodes into a PiP window. */
export function cloneStylesInto(pipWindow: Window): void {
for (const node of document.querySelectorAll('link[rel="stylesheet"], style')) {
pipWindow.document.head.appendChild(node.cloneNode(true));
}
}
export interface ObservationView {
readonly snapshot: OverlaySnapshot;
/** Pinned session wins while it still exists; otherwise auto-follow. */
readonly focus: SessionObservation | undefined;
readonly pinnedId: string | null;
readonly onTogglePin: (sessionId: string) => void;
/** 1s ticker while anything is active (drives the elapsed readout). */
readonly now: number;
}
/**
* Whether one session is visible on a surface scoped to one DSH
* conversation: sessions started by that conversation or its descendants
* (the lineage ancestors ride along on the entry). Untracked sessions (no
* owner recorded) are hidden from scoped surfaces but stay in global views.
*/
export function visibleToScope(obs: SessionObservation, scopeId: string): boolean {
return obs.dshSessionIds?.includes(scopeId) === true;
}
/**
* The store-backed observation view model. Holds the feed for the component
* lifetime (refcounted — overlapping carriers never kill each other's
* stream). `scopeId` narrows the view to one DSH conversation's sessions
* (the better-sidebar tab); undefined keeps the global view (floating
* card, PiP).
*/
export function useObservationView(
store: ObservationClientStore,
scopeId?: string,
): ObservationView {
const snapshot = useSyncExternalStore(store.subscribe, store.getSnapshot);
useEffect(() => {
store.acquire();
return () => store.release();
}, [store]);
const scopedSnapshot: OverlaySnapshot =
scopeId === undefined
? snapshot
: {
...snapshot,
sessions: snapshot.sessions.filter((s) => visibleToScope(s, scopeId)),
};
const [pinnedId, setPinnedId] = useState<string | null>(null);
const pinned =
pinnedId !== null ? scopedSnapshot.sessions.find((s) => s.sessionId === pinnedId) : undefined;
const focus = pinned ?? focusOf(scopedSnapshot.sessions);
const onTogglePin = useCallback((sessionId: string) => {
setPinnedId((current) => (current === sessionId ? null : sessionId));
}, []);
const anyActive = scopedSnapshot.sessions.some((s) => s.action !== "idle");
const [now, setNow] = useState(() => Date.now());
useEffect(() => {
if (!anyActive) return;
const timer = setInterval(() => setNow(Date.now()), 1000);
return () => clearInterval(timer);
}, [anyActive]);
return {
snapshot: scopedSnapshot,
focus,
pinnedId: pinned !== undefined ? pinnedId : null,
onTogglePin,
now,
};
}
export interface PipHandle {
/** The open PiP window, or null while the content renders in its carrier. */
readonly pipWindow: Window | null;
/** Whether the browser exposes Document PiP (Chrome-only today). */
readonly pipSupported: boolean;
/**
* Upgrade the content into a native PiP window. Rejects silently without a
* user gesture (or when closed mid-request): the carrier keeps the content.
*/
readonly popOut: (size?: { width: number; height: number }) => void;
}
export function usePip(): PipHandle {
const [pipWindow, setPipWindow] = useState<Window | null>(null);
const popOut = useCallback((size?: { width: number; height: number }): void => {
const pip = pipApi();
if (pip === undefined) return;
void pip
.requestWindow(size)
.then((win) => {
cloneStylesInto(win);
win.addEventListener("pagehide", () => setPipWindow(null));
setPipWindow(win);
})
.catch(() => {});
}, []);
// Unmounting the carrier (tab close, conversation switch, plugin HMR) does
// not destroy the PiP browsing context: close it ourselves or it survives
// as a blank window. close() on an already-closed window is a no-op.
useEffect(() => () => pipWindow?.close(), [pipWindow]);
return { pipWindow, pipSupported: pipApi() !== undefined, popOut };
}
@@ -0,0 +1,26 @@
/**
* Carrier switch for the observation view: while the better-sidebar
* integration fiber is alive (the dsh-better-sidebar plugin provides its
* `betterSidebar` service), the tracking view lives in a sidebar tab and the
* floating overlay card/capsule hides itself. The flag is a tiny external
* store so the overlay can read it through useSyncExternalStore and flip
* without a remount when the sidebar plugin (un)loads.
*/
let active = false;
const listeners = new Set<() => void>();
export function getSidebarMode(): boolean {
return active;
}
export function subscribeSidebarMode(listener: () => void): () => void {
listeners.add(listener);
return () => listeners.delete(listener);
}
export function setSidebarMode(next: boolean): void {
if (active === next) return;
active = next;
for (const listener of [...listeners]) listener();
}
@@ -12,6 +12,7 @@
import type { Context } from "@deepseek-ai/cordis";
import Schema from "@deepseek-ai/schemastery";
import { armArchiveCleanup } from "./archive-cleanup";
import { armLazyTools } from "./lazy-tools";
import { ObservationService } from "./observation";
import { registerObservationRoutes } from "./observation-http";
@@ -107,6 +108,10 @@ export function apply(
removeRoutes = registerObservationRoutes(injected, observation);
return () => removeRoutes();
});
// Reap a conversation's browsers when the conversation itself is archived:
// archived sessions are hidden from every surface, so their Agent Windows
// would otherwise linger unreachable until idle timeout or unload.
const disarmArchiveCleanup = armArchiveCleanup(ctx, registry, observation);
// Non-blocking install probe: warn early when bsk is missing instead of
// failing the first tool call with a bare spawn error. Uses --version on
@@ -132,6 +137,7 @@ export function apply(
removeSuite();
unregisterSkill();
removeRoutes();
disarmArchiveCleanup();
observation.dispose();
runner.killAll();
const stops = registry
@@ -144,6 +150,7 @@ export function apply(
});
}
export { armArchiveCleanup, ownerSessionIds } from "./archive-cleanup";
export type { ObservationEvent, ObservationOptions, SessionObservation } from "./observation";
export { ObservationService } from "./observation";
export { registerObservationRoutes } from "./observation-http";
@@ -8,6 +8,7 @@
* GET /bsk-observation/state → { sessions, available }
* GET /bsk-observation/events → SSE stream of ObservationEvent
* POST /bsk-observation/interrupt → body {sessionId?} → {interrupted: boolean}
* POST /bsk-observation/stop → body {sessionId} → {stopped: boolean}
* GET /bsk-observation/thumbnail/<attachmentId> → image bytes
*
* Routes exist only when a webServer service is mounted (web composition);
@@ -166,6 +167,42 @@ export function registerObservationRoutes(
});
},
}),
webServer.register({
kind: "exact",
path: `${ROUTE_BASE}/stop`,
handler: (req, res) => {
if (req.method !== "POST") {
sendJson(res, 405, { error: "method not allowed" });
return;
}
if (fenceRejected(req, res)) return;
let body = "";
req.on("data", (chunk: Buffer | string) => {
body += chunk;
});
req.on("end", () => {
let sessionId: string | undefined;
try {
const parsed = JSON.parse(body || "{}") as { sessionId?: unknown };
if (typeof parsed.sessionId === "string" && parsed.sessionId !== "") {
sessionId = parsed.sessionId;
}
} catch {
sendJson(res, 400, { error: "invalid JSON body" });
return;
}
// A destructive call always names its target (never "current").
if (sessionId === undefined) {
sendJson(res, 400, { error: "sessionId required" });
return;
}
void observation.stopSession(sessionId).then(
(stopped) => sendJson(res, 200, { stopped }),
() => sendJson(res, 500, { error: "stop failed" }),
);
});
},
}),
webServer.register({
kind: "prefix",
path: `${ROUTE_BASE}/thumbnail`,
@@ -39,6 +39,13 @@ export interface SessionObservation {
* greys it out until it is stopped/removed. No more frames are requested.
*/
dead?: boolean;
/**
* The DSH conversations this session belongs to (the starting agent's
* session plus its seed-lineage ancestors), recorded at start. Scoped
* surfaces (the better-sidebar tab) filter by it; absent means untracked
* ownership — visible only in the global (unscoped) view.
*/
dshSessionIds?: string[];
}
/** Incremental event carried to subscribers (SSE on the wire). */
@@ -153,11 +160,13 @@ export class ObservationService {
/** Register a fresh owned session (called from browser_session_start). */
addSession(sessionId: string, url?: string): void {
if (!this.deps.options.enabled || this.disposed) return;
const dshSessionIds = this.deps.registry.dshOwnersOf(sessionId);
this.put({
sessionId,
...(url !== undefined ? { url } : {}),
action: "idle",
since: this.scheduler.now(),
...(dshSessionIds.length > 0 ? { dshSessionIds } : {}),
});
this.lastActivity.set(sessionId, this.scheduler.now());
this.scheduleCapture(sessionId, 0);
@@ -228,6 +237,36 @@ export class ObservationService {
return this.deps.runner.killFor(target) > 0;
}
/**
* Stop one owned session and close its Agent Window (the overlay's stop
* button — same end state as `browser_session_stop`). Never waits behind a
* hung in-flight command: tool children are killed first so the session's
* keyed queue drains immediately, and no further captures queue up. A
* session the daemon already forgot (dead strip entries) stops
* idempotently — the goal state (entry gone) is identical.
* @returns whether the session ended up stopped/removed.
*/
async stopSession(sessionId: string): Promise<boolean> {
if (!this.deps.registry.isOwned(sessionId)) return false;
this.cancelCapture(sessionId);
this.deps.runner.killFor(sessionId);
const result = await this.deps.queue.run(sessionId, () =>
this.deps.runner.run(["session", "stop", sessionId], { timeoutMs: 30_000 }),
);
if (result.code !== 0) {
let code: string | undefined;
try {
code = (JSON.parse(result.stdout) as { code?: string }).code;
} catch {
code = undefined;
}
if (code !== "session_not_found") return false;
}
this.deps.registry.remove(sessionId);
this.removeSession(sessionId);
return true;
}
/**
* Read one captured thumbnail from the in-process ring. Powers the plugin's
* own HTTP thumbnail route — frames are plugin-owned runtime data, never
@@ -24,6 +24,13 @@ export interface TrackedSession {
export class SessionRegistry {
private readonly sessions = new Map<string, TrackedSession>();
private currentId: string | undefined;
/**
* bsk session id → the DSH session ids that may clean it up: the agent
* session that started it plus every ancestor along the seed lineage, so
* archiving a conversation at ANY level of the chain reaps its browsers
* (see archive-cleanup.ts).
*/
private readonly dshOwners = new Map<string, ReadonlySet<string>>();
/**
* Slots reserved by in-flight starts. reserveStart/completeStart/abandonStart
* run synchronously around the async spawn, so concurrent starts can never
@@ -72,12 +79,37 @@ export class SessionRegistry {
/** Forget a session; falls back to the most recent remaining one. */
remove(sessionId: string): void {
this.sessions.delete(sessionId);
this.dshOwners.delete(sessionId);
if (this.currentId === sessionId) {
const rest = [...this.sessions.values()];
this.currentId = rest.length > 0 ? rest[rest.length - 1].sessionId : undefined;
}
}
/**
* Record which DSH conversation(s) a freshly started bsk session belongs
* to (the starting agent's session plus its ancestors). No-op without ids
* — e.g. a start whose caller carried no agent identity.
*/
trackOwner(sessionId: string, dshSessionIds: readonly string[]): void {
if (dshSessionIds.length === 0 || !this.sessions.has(sessionId)) return;
this.dshOwners.set(sessionId, new Set(dshSessionIds));
}
/** bsk session ids owned by the given DSH conversation (or its descendants). */
ownedByDsh(dshSessionId: string): string[] {
const owned: string[] = [];
for (const [sessionId, owners] of this.dshOwners) {
if (owners.has(dshSessionId)) owned.push(sessionId);
}
return owned;
}
/** The DSH conversation ids owning one bsk session (empty when untracked). */
dshOwnersOf(sessionId: string): string[] {
return [...(this.dshOwners.get(sessionId) ?? [])];
}
/** Mark an owned session as most recently used (recency order refresh). */
private touch(sessionId: string): void {
const existing = this.sessions.get(sessionId);
@@ -17,6 +17,7 @@ import {
type ToolResult,
type ToolRunContext,
} from "@deepseek-ai/dsh-tools";
import { ownerSessionIds } from "./archive-cleanup";
import { trySaveScreenshot } from "./image";
import { actionForLabel, type ObservationService } from "./observation";
import type { KeyedExecutor } from "./queue";
@@ -245,6 +246,9 @@ export function registerTools(deps: ToolDeps): () => void {
browserInstanceId: reply.browser_instance_id,
startedAtMs: Date.now(),
});
// Ownership for archive cleanup: the calling conversation and its
// ancestors reap this session when any of them is archived.
registry.trackOwner(reply.session_id, ownerSessionIds(deps.ctx, exec.agent?.id));
deps.observation.addSession(reply.session_id, args.url);
try {
if (args.device !== undefined) {
@@ -0,0 +1,157 @@
// Archive-triggered cleanup: lineage resolution for browser-session
// ownership, the registry's owner index, and the domain/changed watcher
// that reaps a freshly archived conversation's bsk sessions.
import { describe, expect, it, vi } from "vitest";
import { armArchiveCleanup, ownerSessionIds } from "../src/archive-cleanup";
import type { ObservationService } from "../src/observation";
import { SessionRegistry } from "../src/sessions";
/** A ctx stub carrying a session store with the given lineage headers. */
function ctxWithSessions(headers: Record<string, { parentSession?: string }>) {
return {
get: (key: string) =>
key === "sessions"
? {
get: (id: string) => (headers[id] === undefined ? undefined : { header: headers[id] }),
}
: undefined,
} as never;
}
describe("ownerSessionIds", () => {
it("walks the seed lineage to the root; empty without an agent identity", () => {
expect(ownerSessionIds(ctxWithSessions({}), undefined)).toEqual([]);
const ctx = ctxWithSessions({
child: { parentSession: "parent" },
parent: { parentSession: "root" },
root: {},
});
expect(ownerSessionIds(ctx, "child")).toEqual(["child", "parent", "root"]);
});
it("stops the walk at an unloaded ancestor", () => {
const ctx = ctxWithSessions({ child: { parentSession: "gone" } });
expect(ownerSessionIds(ctx, "child")).toEqual(["child", "gone"]);
});
it("never loops on a malformed parent cycle", () => {
const ctx = ctxWithSessions({
a: { parentSession: "b" },
b: { parentSession: "a" },
});
expect(ownerSessionIds(ctx, "a")).toEqual(["a", "b"]);
});
});
describe("SessionRegistry owner tracking", () => {
function start(registry: SessionRegistry, sessionId: string): void {
registry.reserveStart();
registry.completeStart({ sessionId, startedAtMs: 1 });
}
it("indexes owners and forgets them on remove; ignores empty/unknown ownership", () => {
const registry = new SessionRegistry(5);
start(registry, "bsk1");
start(registry, "bsk2");
registry.trackOwner("bsk1", ["conv-a", "root"]);
registry.trackOwner("bsk2", ["conv-b"]);
expect(registry.ownedByDsh("root")).toEqual(["bsk1"]);
expect(registry.ownedByDsh("conv-a")).toEqual(["bsk1"]);
expect(registry.ownedByDsh("conv-b")).toEqual(["bsk2"]);
expect(registry.ownedByDsh("nobody")).toEqual([]);
registry.remove("bsk1");
expect(registry.ownedByDsh("root")).toEqual([]);
// Empty owner lists and unknown session ids record nothing.
registry.trackOwner("bsk2", []);
registry.trackOwner("ghost", ["conv-c"]);
expect(registry.ownedByDsh("conv-c")).toEqual([]);
});
});
describe("armArchiveCleanup", () => {
function harness(opts: { archived?: string[] } = {}) {
const registry = new SessionRegistry(5);
const observation = { stopSession: vi.fn(async () => true) };
const listeners = new Set<(change: unknown) => void>();
const ctx = {
get: (key: string) =>
key === "workspaceRegistry" ? { archivedSessionIds: opts.archived ?? [] } : undefined,
on: (event: string, listener: (change: unknown) => void) => {
if (event === "domain/changed") listeners.add(listener);
return () => listeners.delete(listener);
},
};
const emit = (change: unknown) => {
for (const listener of [...listeners]) listener(change);
};
return {
registry,
observation: observation as unknown as ObservationService,
stopSession: observation.stopSession,
emit,
arm: () =>
armArchiveCleanup(ctx as never, registry, observation as unknown as ObservationService),
};
}
function startOwned(registry: SessionRegistry, bskId: string, owners: string[]): void {
registry.reserveStart();
registry.completeStart({ sessionId: bskId, startedAtMs: 1 });
registry.trackOwner(bskId, owners);
}
it("stops every bsk session owned by a freshly archived conversation, until disarmed", () => {
const h = harness();
startOwned(h.registry, "bsk1", ["conv-a", "root"]);
startOwned(h.registry, "bsk2", ["conv-b"]);
const disarm = h.arm();
h.emit({
domain: "workspace",
table: "",
value: { archivedSessionIds: ["root"] },
});
expect(h.stopSession).toHaveBeenCalledTimes(1);
expect(h.stopSession).toHaveBeenCalledWith("bsk1");
// After the disposer runs the watcher is silent again.
disarm();
h.emit({
domain: "workspace",
table: "",
value: { archivedSessionIds: ["root", "conv-b"] },
});
expect(h.stopSession).toHaveBeenCalledTimes(1);
});
it("ignores pre-archived ids, foreign domains, and malformed frames", () => {
const h = harness({ archived: ["old-conv"] });
startOwned(h.registry, "bsk1", ["old-conv"]);
startOwned(h.registry, "bsk2", ["conv-a"]);
h.arm();
// Seeded from the registry: the pre-archived id must not retro-fire.
h.emit({ domain: "workspace", table: "", value: { archivedSessionIds: ["old-conv"] } });
h.emit({ domain: "settings", table: "", value: { archivedSessionIds: ["conv-a"] } });
h.emit({ domain: "workspace", table: "rows", value: { archivedSessionIds: ["conv-a"] } });
h.emit({ domain: "workspace", table: "", value: {} });
expect(h.stopSession).not.toHaveBeenCalled();
// A genuinely new archive still fires.
h.emit({
domain: "workspace",
table: "",
value: { archivedSessionIds: ["old-conv", "conv-a"] },
});
expect(h.stopSession).toHaveBeenCalledWith("bsk2");
});
it("treats a re-archived session as fresh again after unarchive", () => {
const h = harness();
startOwned(h.registry, "bsk1", ["conv-a"]);
h.arm();
h.emit({ domain: "workspace", table: "", value: { archivedSessionIds: ["conv-a"] } });
expect(h.stopSession).toHaveBeenCalledTimes(1);
// Unarchive, then re-archive: the second archival cleans up again.
h.emit({ domain: "workspace", table: "", value: { archivedSessionIds: [] } });
h.emit({ domain: "workspace", table: "", value: { archivedSessionIds: ["conv-a"] } });
expect(h.stopSession).toHaveBeenCalledTimes(2);
});
});
@@ -4,7 +4,7 @@
// upgrade/fallback (mocked documentPictureInPicture). NOTE: use RTL's waitFor
// (act-flushing), not vi.waitFor, when asserting store-driven UI updates.
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
import { act, cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import { applyResize, ObservationOverlay } from "../../src/client/ObservationOverlay";
import { type EventSourceLike, ObservationClientStore } from "../../src/client/observation-store";
@@ -157,6 +157,45 @@ describe("ObservationOverlay", () => {
await screen.findByText(/s1 · clicking/);
});
it("stop session: the first click only arms the confirm, the second posts", async () => {
const h = makeHarness([{ sessionId: "s1", action: "idle", since: Date.now() }]);
render(<ObservationOverlay store={h.store} />);
await screen.findByText(/s1 · idle/);
const stop = screen.getByRole("button", { name: "Stop session s1" });
fireEvent.click(stop);
// Armed: a plain click must never close an Agent Window by itself.
expect(h.fetches.some((f) => f.url === "/bsk-observation/stop")).toBe(false);
const confirm = await screen.findByRole("button", { name: "Confirm stop session s1" });
expect(confirm.getAttribute("aria-pressed")).toBe("true");
fireEvent.click(confirm);
await waitFor(() =>
expect(
h.fetches.some(
(f) => f.url === "/bsk-observation/stop" && f.init?.body === '{"sessionId":"s1"}',
),
).toBe(true),
);
});
it("disarms the stop confirm after the expiry window", async () => {
const h = makeHarness([{ sessionId: "s1", action: "idle", since: Date.now() }]);
render(<ObservationOverlay store={h.store} />);
const stop = await screen.findByRole("button", { name: "Stop session s1" });
vi.useFakeTimers();
try {
fireEvent.click(stop);
// Synchronous queries only: waitFor-style finds hang under fake timers.
expect(screen.getByRole("button", { name: "Confirm stop session s1" })).toBeTruthy();
act(() => {
vi.advanceTimersByTime(3100);
});
expect(screen.getByRole("button", { name: "Stop session s1" })).toBeTruthy();
expect(h.fetches.some((f) => f.url === "/bsk-observation/stop")).toBe(false);
} finally {
vi.useRealTimers();
}
});
it("resizes within min/max clamps", async () => {
const h = makeHarness([BUSY]);
render(<ObservationOverlay store={h.store} />);
@@ -222,6 +261,7 @@ describe("ObservationOverlay", () => {
addEventListener: (name: string, fn: () => void) => {
listeners.set(name, [...(listeners.get(name) ?? []), fn]);
},
close: vi.fn(),
} as unknown as Window;
});
(window as unknown as Record<string, unknown>).documentPictureInPicture = { requestWindow };
@@ -236,6 +276,28 @@ describe("ObservationOverlay", () => {
await screen.findByTestId("obs-card");
});
it("closes the PiP window when the carrier unmounts", async () => {
const h = makeHarness([BUSY]);
const pipDoc = document.implementation.createHTMLDocument("pip");
const close = vi.fn();
const requestWindow = vi.fn(async () => {
return {
document: pipDoc,
addEventListener: () => {},
close,
} as unknown as Window;
});
(window as unknown as Record<string, unknown>).documentPictureInPicture = { requestWindow };
const { unmount } = render(<ObservationOverlay store={h.store} />);
const popout = await screen.findByRole("button", { name: /Pop out/ });
fireEvent.click(popout);
await waitFor(() => expect(pipDoc.body.textContent).toContain("s1"));
// Unmounting (tab close, conversation switch, plugin HMR) closes the PiP
// instead of leaving a blank window behind.
unmount();
expect(close).toHaveBeenCalledTimes(1);
});
it("renders the strip for two sessions and pins focus on click", async () => {
const older: SessionObservation = {
sessionId: "s1",
@@ -0,0 +1,331 @@
// @vitest-environment happy-dom
// better-sidebar carrier: tab registration + auto-open, the sidebar-mode
// flag hiding the floating overlay, PiP pop-out from the tab, and the
// store's refcounted acquire/release across overlapping carriers.
import { cleanup, render, screen, waitFor } from "@testing-library/react";
import { afterEach, describe, expect, it, vi } from "vitest";
import { ObservationOverlay } from "../../src/client/ObservationOverlay";
import {
type BetterSidebarLike,
OBSERVATION_TAB_PATH,
OBSERVATION_TAB_TYPE,
ObservationSidebarTab,
observationTabOpen,
registerObservationSidebar,
type SidebarNodeLike,
type SidebarStateLike,
type TabDescriptorLike,
} from "../../src/client/observation-sidebar";
import { type EventSourceLike, ObservationClientStore } from "../../src/client/observation-store";
import { getSidebarMode, setSidebarMode } from "../../src/client/sidebar-mode";
import type { SessionObservation } from "../../src/observation";
// Owned by the sidebar mock's active conversation ("conv-1") so the scoped
// visibility logic lets it through; FOREIGN belongs to another conversation.
const BUSY: SessionObservation = {
sessionId: "s1",
action: "clicking",
since: Date.now() - 7000,
dshSessionIds: ["conv-1"],
};
const FOREIGN: SessionObservation = {
sessionId: "s9",
action: "clicking",
since: Date.now() - 5000,
dshSessionIds: ["other-conv"],
};
interface Harness {
store: ObservationClientStore;
es: () => EventSourceLike | undefined;
push: (sessions: SessionObservation[]) => void;
}
function makeHarness(initial: SessionObservation[]): Harness {
let es: EventSourceLike | undefined;
let current = initial;
const store = new ObservationClientStore({
fetchFn: async (url: string) => {
if (url === "/bsk-observation/state") {
return { ok: true, json: async () => ({ sessions: current }) };
}
return { ok: true, json: async () => ({ interrupted: true }) };
},
eventSourceFactory: () => {
es = { onmessage: null, close: vi.fn() };
return es;
},
loadImage: async (id: string) => `blob:${id}`,
});
return {
store,
es: () => es,
push: (sessions) => {
current = sessions;
for (const s of sessions) {
es?.onmessage?.({ data: JSON.stringify({ type: "upsert", session: s }) });
}
if (sessions.length === 0) es?.onmessage?.({ data: JSON.stringify({ type: "reset" }) });
},
};
}
function leafWith(type: string, panelOpen = true): SidebarStateLike {
return {
panelOpen,
splits: {
kind: "split",
id: "root",
dir: "row",
sizes: [1],
children: [
{
kind: "leaf",
id: "pane-1",
active: type,
tabs: [{ id: type, type, title: type }],
},
],
},
bottomSplits: { kind: "leaf", id: "pane-2", active: null, tabs: [] },
};
}
interface SidebarMock {
service: BetterSidebarLike;
descriptor: () => TabDescriptorLike;
openTab: ReturnType<typeof vi.fn>;
disposeTab: ReturnType<typeof vi.fn>;
setState: (state: SidebarStateLike | undefined) => void;
}
/** Insert a tab into the first leaf of the main tree (mirrors a real open). */
function withTabOpened(state: SidebarStateLike, type: string): SidebarStateLike {
let done = false;
const walk = (node: SidebarNodeLike): SidebarNodeLike => {
if (done) return node;
if (node.kind === "leaf") {
done = true;
return { ...node, active: type, tabs: [...node.tabs, { id: type, type, title: type }] };
}
return { ...node, children: node.children.map(walk) };
};
return { ...state, splits: walk(state.splits) };
}
function makeSidebar(
initialState: SidebarStateLike | undefined = leafWith("explorer"),
): SidebarMock {
let descriptor: TabDescriptorLike | undefined;
let state: SidebarStateLike | undefined = initialState;
const openTab = vi.fn((seed: { type: string }) => {
// Mirror the real feedback loop: a created tab lands in the state, so
// later evaluations see it open (single-instance dedupe re-focuses).
if (state !== undefined && !observationTabOpen(state)) {
state = withTabOpened(state, seed.type);
}
});
const disposeTab = vi.fn();
const service: BetterSidebarLike = {
registerTab: (next) => {
descriptor = next;
return disposeTab;
},
openTab,
getSnapshot: () => ({ sessionId: "conv-1", state }),
isTabEnabled: () => true,
};
return {
service,
descriptor: () => {
if (descriptor === undefined) throw new Error("no tab registered");
return descriptor;
},
openTab,
disposeTab,
setState: (next) => {
state = next;
},
};
}
afterEach(() => {
cleanup();
setSidebarMode(false);
});
describe("registerObservationSidebar", () => {
it("registers a single-instance tab and holds the feed for its lifetime", () => {
const h = makeHarness([]);
const sidebar = makeSidebar();
expect(getSidebarMode()).toBe(false);
const dispose = registerObservationSidebar(sidebar.service, h.store);
expect(getSidebarMode()).toBe(true);
// The fiber holds the observation feed itself, so the auto-open watcher
// sees new sessions even while the tab is closed.
expect(h.es()).toBeDefined();
const descriptor = sidebar.descriptor();
expect(descriptor.id).toBe(OBSERVATION_TAB_TYPE);
expect(descriptor.single).toBe(true);
expect(descriptor.title).toBe("Browser Skill");
dispose();
expect(getSidebarMode()).toBe(false);
expect(sidebar.disposeTab).toHaveBeenCalledTimes(1);
expect(h.es()?.close).toHaveBeenCalled?.();
});
it("opens the tab right away when a session is already live at activation", async () => {
const h = makeHarness([BUSY]);
const sidebar = makeSidebar();
registerObservationSidebar(sidebar.service, h.store);
// The initial state fetch lands asynchronously; the 0→N watcher then fires.
await waitFor(() =>
expect(sidebar.openTab).toHaveBeenCalledWith({
type: OBSERVATION_TAB_TYPE,
path: OBSERVATION_TAB_PATH,
}),
);
});
it("auto-opens on the first session but never re-focuses an open tab", async () => {
const h = makeHarness([]);
const sidebar = makeSidebar();
registerObservationSidebar(sidebar.service, h.store);
expect(sidebar.openTab).not.toHaveBeenCalled();
// First session arrives: the tab lands in the sidebar.
h.push([BUSY]);
await waitFor(() => expect(sidebar.openTab).toHaveBeenCalledTimes(1));
// The sidebar now reports the tab open (persisted/restored): further
// session arrivals must not steal focus from whatever the user reads.
sidebar.setState(leafWith(OBSERVATION_TAB_TYPE));
h.push([]);
h.push([{ ...BUSY, sessionId: "s2" }]);
h.push([]);
h.push([{ ...BUSY, sessionId: "s3" }]);
await waitFor(() => expect(h.store.getSnapshot().sessions.length).toBe(1));
expect(sidebar.openTab).toHaveBeenCalledTimes(1);
});
it("does not open while the sidebar has no active session state", () => {
const h = makeHarness([BUSY]);
const sidebar = makeSidebar(undefined);
registerObservationSidebar(sidebar.service, h.store);
expect(sidebar.openTab).not.toHaveBeenCalled();
});
it("nudges an existing tab back into sight when the panel is collapsed", async () => {
const h = makeHarness([]);
// Tab already open (persisted), but the panel is collapsed: a new
// session should surface the tracking view again (focus + expand via a
// content open), like the floating card reappearing.
const sidebar = makeSidebar(leafWith(OBSERVATION_TAB_TYPE, false));
registerObservationSidebar(sidebar.service, h.store);
h.push([BUSY]);
await waitFor(() =>
expect(sidebar.openTab).toHaveBeenCalledWith({
type: OBSERVATION_TAB_TYPE,
path: OBSERVATION_TAB_PATH,
}),
);
});
it("shows the visible session count as the tab badge", () => {
const h = makeHarness([]);
const sidebar = makeSidebar();
registerObservationSidebar(sidebar.service, h.store);
const badge = sidebar.descriptor().badge;
const scope = { sessionId: "conv-1" };
expect(badge?.(undefined, scope, undefined)).toBeNull();
h.store.acquire();
h.push([BUSY, FOREIGN]);
expect(badge?.(undefined, scope, undefined)).toBe(1);
expect(badge?.(undefined, { sessionId: "other-conv" }, undefined)).toBe(1);
expect(badge?.(undefined, { sessionId: "nobody" }, undefined)).toBeNull();
h.store.release();
});
it("does not auto-open for sessions owned by other conversations", async () => {
const h = makeHarness([]);
const sidebar = makeSidebar();
registerObservationSidebar(sidebar.service, h.store);
h.push([FOREIGN]);
await waitFor(() => expect(h.store.getSnapshot().sessions.length).toBe(1));
expect(sidebar.openTab).not.toHaveBeenCalled();
// A session visible to the active conversation does open the tab.
h.push([BUSY]);
await waitFor(() => expect(sidebar.openTab).toHaveBeenCalledTimes(1));
});
});
describe("observationTabOpen", () => {
it("finds the tab in either workbench tree", () => {
expect(observationTabOpen(undefined)).toBe(false);
expect(observationTabOpen(leafWith("explorer"))).toBe(false);
expect(observationTabOpen(leafWith(OBSERVATION_TAB_TYPE))).toBe(true);
const inBottom: SidebarStateLike = {
splits: { kind: "leaf", id: "p1", active: null, tabs: [] },
bottomSplits: {
kind: "leaf",
id: "p2",
active: OBSERVATION_TAB_TYPE,
tabs: [{ id: OBSERVATION_TAB_TYPE, type: OBSERVATION_TAB_TYPE, title: "Browser" }],
},
};
expect(observationTabOpen(inBottom)).toBe(true);
});
});
describe("ObservationSidebarTab", () => {
it("renders the tracking view without card chrome (no collapse, no drag header)", async () => {
const h = makeHarness([BUSY]);
render(<ObservationSidebarTab store={h.store} scopeId="conv-1" />);
await screen.findByText(/s1 · clicking/);
expect(screen.queryByRole("button", { name: "Collapse" })).toBeNull();
expect(screen.getByTestId("obs-header").dataset.draggable).toBeUndefined();
// No resize handles — the sidebar owns the geometry.
expect(screen.queryByTestId("obs-resize-se")).toBeNull();
// Session controls ride along: interrupt + stop-with-confirm.
expect(screen.getByRole("button", { name: "Stop session s1" })).toBeTruthy();
});
it("scopes the view to the sidebar's conversation", async () => {
const h = makeHarness([BUSY, FOREIGN]);
render(<ObservationSidebarTab store={h.store} scopeId="conv-1" />);
// Focus and content belong to the owned session; the foreign one never
// reaches the stage, the status line, or the strip (which needs 2+).
await screen.findByText(/s1 · clicking/);
expect(screen.queryByText(/s9/)).toBeNull();
expect(screen.queryByTestId("obs-strip")).toBeNull();
// Scoping to the other conversation swaps what's visible.
render(<ObservationSidebarTab store={h.store} scopeId="other-conv" />);
await screen.findByText(/s9 · clicking/);
});
});
describe("carrier switching", () => {
it("hides the floating overlay while sidebar mode is active", async () => {
const h = makeHarness([]);
const { container } = render(<ObservationOverlay store={h.store} />);
h.push([BUSY]);
await screen.findByTestId("obs-card");
setSidebarMode(true);
await waitFor(() => expect(container.firstChild).toBeNull());
setSidebarMode(false);
await screen.findByTestId("obs-card");
});
it("keeps the feed alive across overlapping carriers (refcount)", async () => {
const h = makeHarness([]);
const overlay = render(<ObservationOverlay store={h.store} />);
await waitFor(() => expect(h.es()).toBeDefined());
// A second carrier mounts (the sidebar tab), then the overlay unmounts:
// the stream must survive for the remaining consumer.
const tab = render(<ObservationSidebarTab store={h.store} scopeId="conv-1" />);
overlay.unmount();
h.push([BUSY]);
await screen.findByText(/s1 · clicking/);
expect(h.es()?.close ?? vi.fn()).not.toHaveBeenCalled();
tab.unmount();
});
});
@@ -137,6 +137,9 @@ describe("client plugin registration", () => {
const sessions = { binding: () => undefined };
const ctx = {
get: (key: string) => (key === "sessions" ? sessions : undefined),
// cordis ctx.inject: the betterSidebar carrier upgrade stays dormant in
// this composition (the callback only runs once the service exists).
inject: (_deps: string[], _fn: (injected: unknown) => unknown) => {},
slots: {
inject: (_name: string, fn: () => unknown) => fn(),
register: (slot: { name: string; key?: string; id?: string }, view: unknown) => {
@@ -61,6 +61,8 @@ function fakeRunner(
opts: {
screenshotFails?: boolean;
screenshotNotFound?: boolean;
stopNotFound?: boolean;
stopFails?: boolean;
killCount?: number;
screenshotBytes?: () => Uint8Array;
} = {},
@@ -72,6 +74,21 @@ function fakeRunner(
killed,
async run(args: string[], options: BskRunOptions = {}): Promise<BskRunResult> {
calls.push({ args, options });
if (args[0] === "session" && args[1] === "stop") {
if (opts.stopNotFound) {
return {
code: 4,
stdout: JSON.stringify({ code: "session_not_found", message: "no such session" }),
stderr: "",
timedOut: false,
aborted: false,
};
}
if (opts.stopFails) {
return { code: 1, stdout: "", stderr: "boom", timedOut: false, aborted: false };
}
return { code: 0, stdout: "{}", stderr: "", timedOut: false, aborted: false };
}
if (args[0] === "screenshot") {
if (opts.screenshotNotFound) {
return {
@@ -180,6 +197,23 @@ describe("state machine", () => {
expect(events[events.length - 1].type).toBe("reset");
});
it("stamps the registry-recorded DSH owners onto new entries", () => {
const registry = new SessionRegistry(5);
own(registry, "s1");
own(registry, "s2");
registry.trackOwner("s1", ["conv-a", "root"]);
const { service } = setup({ registry });
service.addSession("s1");
service.addSession("s2");
expect(service.getState()).toEqual([
{ sessionId: "s1", action: "idle", since: 1_000_000, dshSessionIds: ["conv-a", "root"] },
{ sessionId: "s2", action: "idle", since: 1_000_000 },
]);
// The owner stamp survives later upserts (action/url/frame churn).
service.beginAction("s1", "clicking");
expect(service.getState()[0].dshSessionIds).toEqual(["conv-a", "root"]);
});
it("ignores instrumentation for unknown sessions (owned-only by construction)", () => {
const { service } = setup({});
service.beginAction("foreign", "clicking");
@@ -307,6 +341,57 @@ describe("interrupt routing", () => {
});
});
describe("stopSession", () => {
it("stops an owned session: kills in-flight tools, runs session stop, removes the entry", async () => {
const registry = new SessionRegistry(5);
own(registry, "s1");
const runner = fakeRunner();
const { service, events } = setup({ registry, runner });
service.addSession("s1");
expect(registry.isOwned("s1")).toBe(true);
await expect(service.stopSession("s1")).resolves.toBe(true);
expect(runner.killed).toEqual(["s1"]);
const stop = runner.calls.find((c) => c.args[0] === "session" && c.args[1] === "stop");
expect(stop?.args).toEqual(["session", "stop", "s1"]);
expect(registry.isOwned("s1")).toBe(false);
expect(service.getState()).toEqual([]);
expect(events[events.length - 1]).toMatchObject({ type: "remove" });
});
it("refuses foreign sessions without touching the runner", async () => {
const registry = new SessionRegistry(5);
own(registry, "s1");
const runner = fakeRunner();
const { service } = setup({ registry, runner });
await expect(service.stopSession("foreign")).resolves.toBe(false);
expect(runner.killed).toEqual([]);
expect(runner.calls).toEqual([]);
});
it("stops idempotently when the daemon already forgot the session (dead entries)", async () => {
const registry = new SessionRegistry(5);
own(registry, "s1");
const runner = fakeRunner({ stopNotFound: true });
const { service, events } = setup({ registry, runner });
service.addSession("s1");
await expect(service.stopSession("s1")).resolves.toBe(true);
expect(registry.isOwned("s1")).toBe(false);
expect(events[events.length - 1]).toMatchObject({ type: "remove" });
});
it("returns false and keeps the entry when the stop itself fails", async () => {
const registry = new SessionRegistry(5);
own(registry, "s1");
const runner = fakeRunner({ stopFails: true });
const { service } = setup({ registry, runner });
service.addSession("s1");
await expect(service.stopSession("s1")).resolves.toBe(false);
expect(registry.isOwned("s1")).toBe(true);
expect(service.getState().map((s) => s.sessionId)).toEqual(["s1"]);
});
});
describe("HTTP/SSE interface", () => {
interface RecordedRoute {
path: string;
@@ -405,11 +490,45 @@ describe("HTTP/SSE interface", () => {
await interruptRoute?.handler(postReq, interruptRes.res as never);
expect(JSON.parse(interruptRes.res.body())).toEqual({ interrupted: true });
// stop (POST {sessionId}): owned session stops and leaves the state
const stopRoute = routes.get("/bsk-observation/stop");
const stopRes = fakeRes();
await stopRoute?.handler(postReq, stopRes.res as never);
// The stop is async behind the response-less handler — poll the state.
await waitFor(() => service.getState().length === 0);
expect(JSON.parse(stopRes.res.body())).toEqual({ stopped: true });
dispose();
expect(routes.size).toBe(0);
service.dispose();
});
it("refuses a stop without a sessionId", async () => {
const registry = new SessionRegistry(5);
own(registry, "s1");
const { service } = setup({ registry });
const { routes, webServer } = routeHarness();
const ctx = { get: (key: string) => (key === "webServer" ? webServer : undefined) } as never;
const dispose = registerObservationRoutes(ctx, service);
const stopRoute = routes.get("/bsk-observation/stop");
const res = fakeRes();
const emptyReq = fakeReq({
method: "POST",
headers: { host: "127.0.0.1:3999", "content-type": "application/json" },
on: (event: string, fn: (chunk?: string) => void) => {
if (event === "data") fn("{}");
if (event === "end") fn();
},
});
await stopRoute?.handler(emptyReq, res.res as never);
expect(res.res.status).toBe(400);
expect(registry.isOwned("s1")).toBe(true);
dispose();
service.dispose();
});
it("fences off non-loopback, cross-site, and simple-request traffic", async () => {
const registry = new SessionRegistry(5);
own(registry, "s1");