fix(record): record switch_tab and keep Trace v3 states per tab

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
“haonan”
2026-08-24 10:42:45 +08:00
committed by GitHub
co-authored by Cursor
parent 38b0debba9
commit 72834980d2
26 changed files with 1000 additions and 139 deletions
@@ -11,11 +11,11 @@ vi.mock("../recording/observation-capture", async (importOriginal) => {
import { ObservationNodeIndex } from "../recording/observation-capture";
import { RecordingObservationRuntime } from "../recording/recording-runtime";
function observation() {
function observation(url = "https://example.com/") {
return {
rootFrameId: "root",
index: new ObservationNodeIndex({ rootFrameId: "root", matchNodes: [], refs: [] }),
url: "https://example.com/",
url,
title: "Example",
vomText: '@vom 1\nRootWebArea "Example"',
truncated: false,
@@ -130,4 +130,21 @@ describe("RecordingObservationRuntime", () => {
expect(dumped).not.toContain("backendNodeId");
expect(dumped).not.toContain("ObservationNodeIndex");
});
it("captures a tab transition without sharing observation cursors", async () => {
captureRecordingObservation
.mockResolvedValueOnce(observation("https://example.com/first"))
.mockResolvedValueOnce(observation("https://example.com/second"));
const recording = runtime();
await recording.captureInitial(4);
const transition = await recording.captureTabTransition(4, 5);
expect(transition).toMatchObject({
preStateId: "s1",
postStateId: "s2",
targetUrl: "https://example.com/second",
});
expect(captureRecordingObservation.mock.calls.map(([input]) => input.tabId)).toEqual([4, 5]);
});
});
@@ -3,7 +3,7 @@ import { appendRecordedPayload, observeRecordedNavigation } from "../recording/s
describe("recording-step-buffer", () => {
it("stores semantic click without summary", () => {
const buffer = { steps: [], pendingNavigation: false };
const buffer = { steps: [], navigation: { pendingNavigation: false } };
appendRecordedPayload(buffer, {
op: "click",
target: { tag: "button", role: "button", name: "发布" },
@@ -15,11 +15,11 @@ describe("recording-step-buffer", () => {
captureTarget: { tag: "button", role: "button", name: "发布" },
},
]);
expect(buffer.pendingNavigation).toBe(true);
expect(buffer.navigation.pendingNavigation).toBe(true);
});
it("keeps the hovered element description as a capture fallback", () => {
const buffer = { steps: [], pendingNavigation: false };
const buffer = { steps: [], navigation: { pendingNavigation: false } };
appendRecordedPayload(
buffer,
{
@@ -59,9 +59,11 @@ describe("recording-step-buffer", () => {
captureTarget: { tag: "button", role: "button", name: "发布" },
},
],
currentUrl: "https://example.com/a",
pendingNavigation: true,
pendingNavigationDeadline: Date.now() + 5_000,
navigation: {
currentUrl: "https://example.com/a",
pendingNavigation: true,
pendingNavigationDeadline: Date.now() + 5_000,
},
};
observeRecordedNavigation(buffer, "https://example.com/b", true);
expect(buffer.steps).toEqual([
@@ -83,9 +85,11 @@ describe("recording-step-buffer", () => {
values: ["tech"],
},
],
currentUrl: "https://example.com/list",
pendingNavigation: true,
pendingNavigationDeadline: Date.now() + 5_000,
navigation: {
currentUrl: "https://example.com/list",
pendingNavigation: true,
pendingNavigationDeadline: Date.now() + 5_000,
},
};
observeRecordedNavigation(buffer, "https://example.com/list?cat=tech", true);
expect(buffer.steps).toEqual([
@@ -101,8 +105,7 @@ describe("recording-step-buffer", () => {
it("emits navigate for uncaused URL changes", () => {
const buffer = {
steps: [],
currentUrl: "https://example.com/a",
pendingNavigation: false,
navigation: { currentUrl: "https://example.com/a", pendingNavigation: false },
};
const result = observeRecordedNavigation(buffer, "https://example.com/b", false);
expect(result).toEqual({ kind: "appended", index: 0 });
@@ -115,8 +118,10 @@ describe("recording-step-buffer", () => {
it("asks the recorder to coalesce redirect hops instead of emitting each one", () => {
const buffer = {
steps: [],
currentUrl: "https://passport.example/login",
pendingNavigation: false,
navigation: {
currentUrl: "https://passport.example/login",
pendingNavigation: false,
},
};
const hop1 = observeRecordedNavigation(
buffer,
@@ -139,14 +144,16 @@ describe("recording-step-buffer", () => {
url: "https://app.example/dashboard",
});
expect(buffer.steps).toEqual([]);
expect(buffer.currentUrl).toBe("https://app.example/dashboard");
expect(buffer.navigation.currentUrl).toBe("https://app.example/dashboard");
});
it("does not reuse redirect metadata for a later content-observed URL change", () => {
const buffer = {
steps: [],
currentUrl: "https://example.com/redirect",
pendingNavigation: false,
navigation: {
currentUrl: "https://example.com/redirect",
pendingNavigation: false,
},
};
const result = observeRecordedNavigation(buffer, "https://example.com/spa");
@@ -0,0 +1,29 @@
import { describe, expect, it } from "vitest";
import { RecordingTabCoordinator } from "../recording/tab-coordinator";
describe("RecordingTabCoordinator", () => {
it("keeps navigation state isolated per tab", () => {
const tabs = new RecordingTabCoordinator(4, "https://example.com/first");
const first = tabs.navigation(4);
first.pendingNavigation = true;
const second = tabs.navigation(5, "https://example.com/second");
expect(second).toEqual({
currentUrl: "https://example.com/second",
pendingNavigation: false,
});
expect(tabs.navigation(4)).toBe(first);
expect(tabs.navigation(4).pendingNavigation).toBe(true);
});
it("invalidates an earlier activation when a newer tab becomes active", () => {
const tabs = new RecordingTabCoordinator(4);
const second = tabs.noteActivation(5);
const third = tabs.noteActivation(6);
expect(tabs.isLatest(second)).toBe(false);
expect(tabs.isLatest(third)).toBe(true);
expect(tabs.commit(6)).toBe(4);
expect(tabs.currentTabId).toBe(6);
});
});
@@ -27,6 +27,17 @@ describe("shouldRecordPress", () => {
});
describe("reduceTraceSteps", () => {
it("does not expose internal tab transitions in trace v2", () => {
const trace = reduceTraceSteps([
{
op: "switch_tab",
preStateId: "s1",
postStateId: "s2",
},
]);
expect(trace.steps).toEqual([]);
});
it("builds steps with pages dictionary and page id references", () => {
const drafts: RecordingDraftStep[] = [
{
@@ -33,6 +33,21 @@ describe("trace reducer v3", () => {
expect(JSON.stringify(reduced.steps)).not.toContain("private-account-id");
});
it("emits tab transitions only for callers that advertised support", () => {
const drafts: RecordingDraftStep[] = [
{
op: "switch_tab",
preStateId: "s1",
postStateId: "s2",
},
];
expect(reduceTraceStepsV3(drafts).steps).toEqual([]);
expect(reduceTraceStepsV3(drafts, { includeTabSwitches: true }).steps).toEqual([
{ op: "switch_tab", id: 1, state: "s1", result: { state: "s2" } },
]);
});
it("collapses redirect hops while retaining draft-to-step identity", () => {
const drafts: RecordingDraftStep[] = [
{ op: "navigate", url: "https://example.com/start", preStateId: "s1", postStateId: "s2" },
@@ -28,7 +28,13 @@ function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
}
function isTargeted(draft: RecordingDraftStep): draft is TargetedRecordingDraft {
return draft.op !== "navigate" && draft.op !== "scroll";
return (
draft.op === "click" ||
draft.op === "hover" ||
draft.op === "fill" ||
draft.op === "press" ||
draft.op === "select"
);
}
export class RecordingObservationSession {
@@ -1,6 +1,7 @@
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
import type { StopReason, TraceV3 } from "@/transport/types";
import type { DocumentSettleScope } from "./document-settle";
import type { RegisteredObservation } from "./observation-capture";
import { RecordingObservationSession } from "./observation-session";
import { inferMissingPostStates, SettleController } from "./settle-controller";
import { RecordingStateRegistry } from "./state-registry";
@@ -10,11 +11,11 @@ import type { RecordingDraftStep, StepAnnotation } from "./types";
interface TabRecordingContext {
session: RecordingObservationSession;
settle: SettleController;
initialCapture: PendingInitialCapture | null;
pendingCapture: PendingCapture | null;
}
interface PendingInitialCapture {
promise: Promise<void>;
interface PendingCapture {
promise: Promise<RegisteredObservation>;
abort: AbortController;
}
@@ -56,32 +57,71 @@ export class RecordingObservationRuntime {
tabsApi: this.#tabsApi,
tabId,
}),
initialCapture: null,
pendingCapture: null,
};
this.#contexts.set(tabId, context);
return context;
}
async captureInitial(tabId: number): Promise<void> {
async #capture(tabId: number): Promise<RegisteredObservation> {
const context = this.#context(tabId);
if (context.session.cursor.lastSettled) return;
if (context.initialCapture) return context.initialCapture.promise;
if (context.pendingCapture) return context.pendingCapture.promise;
const abort = new AbortController();
let pending!: PendingInitialCapture;
let pending!: PendingCapture;
const promise = context.session.capture(this.#cdp, this.#tabsApi, tabId, abort.signal);
pending = {
abort,
promise: promise
.then(() => {})
.finally(() => {
if (context.initialCapture === pending) context.initialCapture = null;
}),
promise: promise.finally(() => {
if (context.pendingCapture === pending) context.pendingCapture = null;
}),
};
context.initialCapture = pending;
context.pendingCapture = pending;
return pending.promise;
}
async captureInitial(tabId: number): Promise<void> {
const context = this.#context(tabId);
if (context.session.cursor.lastSettled) return;
await this.#capture(tabId);
}
async captureTabTransition(
fromTabId: number,
toTabId: number,
): Promise<{ preStateId?: string; postStateId?: string; targetUrl?: string }> {
const from = this.#context(fromTabId);
await this.#flushContext(from);
if (!from.session.cursor.lastSettled) {
try {
await this.captureInitial(fromTabId);
} catch {
// A target-tab observation can still preserve the forward transition.
}
}
const to = this.#context(toTabId);
let target = to.session.cursor.lastSettled;
try {
target = await this.#capture(toTabId);
} catch {
// Reuse the last settled target state when a refresh races navigation.
}
const source = from.session.cursor.lastSettled;
return {
...(source ? { preStateId: source.stateId } : {}),
...(target ? { postStateId: target.stateId, targetUrl: target.url } : {}),
};
}
bindTabTransition(
draft: Extract<RecordingDraftStep, { op: "switch_tab" }>,
draftId: number,
): void {
if (draft.preStateId) this.#registry.markStep(draft.preStateId, draftId);
}
async processDraft(
tabId: number,
drafts: RecordingDraftStep[],
@@ -119,23 +159,27 @@ export class RecordingObservationRuntime {
this.#context(tabId).settle.scheduleRedirect(drafts, url);
}
async flushRedirects(): Promise<void> {
async #flushContext(context: TabRecordingContext): Promise<void> {
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
await context.settle.flushRedirects();
await context.settle.flush();
}
async flushRedirects(tabId?: number): Promise<void> {
if (tabId !== undefined) {
await this.#contexts.get(tabId)?.settle.flushRedirects();
return;
}
for (const context of this.#contexts.values()) await context.settle.flushRedirects();
}
async flush(): Promise<void> {
await Promise.allSettled(
[...this.#contexts.values()].flatMap((context) =>
context.initialCapture ? [context.initialCapture.promise] : [],
),
);
await this.flushRedirects();
for (const context of this.#contexts.values()) await context.settle.flush();
for (const context of this.#contexts.values()) await this.#flushContext(context);
}
async settleTrailing(tabId: number, drafts: RecordingDraftStep[]): Promise<void> {
const context = this.#context(tabId);
if (context.initialCapture) await Promise.allSettled([context.initialCapture.promise]);
if (context.pendingCapture) await Promise.allSettled([context.pendingCapture.promise]);
if (!context.session.cursor.lastSettled) {
try {
await this.captureInitial(tabId);
@@ -149,7 +193,7 @@ export class RecordingObservationRuntime {
cancel(): void {
for (const context of this.#contexts.values()) {
context.initialCapture?.abort.abort();
context.pendingCapture?.abort.abort();
context.settle.cancel();
}
}
@@ -161,6 +205,7 @@ export class RecordingObservationRuntime {
startUrl?: string;
stoppedBy: StopReason;
bskVersion: string;
includeTabSwitches?: boolean;
}): TraceV3 {
return buildTraceV3({
registry: this.#registry,
@@ -172,6 +217,7 @@ export class RecordingObservationRuntime {
stoppedBy: input.stoppedBy,
bskVersion: input.bskVersion,
redactValues: this.#redactValues,
includeTabSwitches: input.includeTabSwitches,
});
}
}
+17 -12
View File
@@ -4,6 +4,10 @@ import type { RecordingDraftStep, TargetMatchHint } from "./types";
export interface RecordingStepBuffer {
steps: RecordingDraftStep[];
navigation: RecordingNavigationCursor;
}
export interface RecordingNavigationCursor {
currentUrl?: string;
pendingNavigation: boolean;
pendingNavigationDeadline?: number;
@@ -87,22 +91,23 @@ export function observeRecordedNavigation(
transitionType?: string,
transitionQualifiers?: string[],
): NavigationObserveResult {
if (!url || url === buffer.currentUrl) return { kind: "noop" };
buffer.currentUrl = url;
const navigation = buffer.navigation;
if (!url || url === navigation.currentUrl) return { kind: "noop" };
navigation.currentUrl = url;
const pendingIsCurrent =
buffer.pendingNavigation &&
(buffer.pendingNavigationDeadline === undefined ||
buffer.pendingNavigationDeadline >= Date.now());
navigation.pendingNavigation &&
(navigation.pendingNavigationDeadline === undefined ||
navigation.pendingNavigationDeadline >= Date.now());
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
buffer.pendingNavigation = false;
buffer.pendingNavigationDeadline = undefined;
navigation.pendingNavigation = false;
navigation.pendingNavigationDeadline = undefined;
const annotatedIndex = annotateLastStepNavigation(buffer, url);
if (annotatedIndex >= 0) return { kind: "annotated", index: annotatedIndex };
} else {
buffer.pendingNavigation = false;
buffer.pendingNavigationDeadline = undefined;
navigation.pendingNavigation = false;
navigation.pendingNavigationDeadline = undefined;
}
if (hasRedirectQualifier(transitionQualifiers)) {
@@ -135,14 +140,14 @@ export function appendRecordedPayload(
return result.kind === "appended" ? result.index : null;
}
const step = toDraftStep(
{ ...payload, page_url: payload.page_url ?? buffer.currentUrl },
{ ...payload, page_url: payload.page_url ?? buffer.navigation.currentUrl },
targetHint,
);
if (!step) return null;
buffer.steps.push(step);
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
buffer.pendingNavigation = payload.expects_navigation === true;
buffer.pendingNavigationDeadline = buffer.pendingNavigation
buffer.navigation.pendingNavigation = payload.expects_navigation === true;
buffer.navigation.pendingNavigationDeadline = buffer.navigation.pendingNavigation
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
: undefined;
}
@@ -0,0 +1,62 @@
import type { RecordingNavigationCursor } from "./step-buffer";
export interface TabActivation {
tabId: number;
revision: number;
}
export class RecordingTabCoordinator {
readonly #navigationByTab = new Map<number, RecordingNavigationCursor>();
#activeTabId: number;
#currentTabId: number;
#activationRevision = 0;
constructor(initialTabId: number, initialUrl?: string) {
this.#activeTabId = initialTabId;
this.#currentTabId = initialTabId;
this.#navigationByTab.set(initialTabId, {
currentUrl: initialUrl,
pendingNavigation: false,
});
}
get activeTabId(): number {
return this.#activeTabId;
}
get currentTabId(): number {
return this.#currentTabId;
}
noteActivation(tabId: number): TabActivation {
this.#activeTabId = tabId;
this.#activationRevision += 1;
return { tabId, revision: this.#activationRevision };
}
isLatest(activation: TabActivation): boolean {
return (
activation.tabId === this.#activeTabId && activation.revision === this.#activationRevision
);
}
commit(tabId: number): number {
const previousTabId = this.#currentTabId;
this.#currentTabId = tabId;
return previousTabId;
}
navigation(tabId: number, fallbackUrl?: string): RecordingNavigationCursor {
const existing = this.#navigationByTab.get(tabId);
if (existing) {
if (!existing.currentUrl && fallbackUrl) existing.currentUrl = fallbackUrl;
return existing;
}
const created: RecordingNavigationCursor = {
currentUrl: fallbackUrl,
pendingNavigation: false,
};
this.#navigationByTab.set(tabId, created);
return created;
}
}
@@ -40,8 +40,12 @@ export function buildTraceV3(input: {
stoppedBy: StopReason;
bskVersion: string;
redactValues?: boolean;
includeTabSwitches?: boolean;
}): TraceV3 {
const reduced = reduceTraceStepsV3(input.drafts, { redactValues: input.redactValues });
const reduced = reduceTraceStepsV3(input.drafts, {
includeTabSwitches: input.includeTabSwitches,
redactValues: input.redactValues,
});
const entries = publishedEntries(input.registry, reduced.steps);
const publishedId = new Map(entries.map((entry, index) => [entry.id, `s${index + 1}`]));
const annotationsByState = new Map<string, StepAnnotation[]>();
@@ -162,6 +162,7 @@ function toV2Step(
effectForNavigation(step.navigatedTo, urlToId),
);
case "scroll":
case "switch_tab":
return null;
}
}
@@ -58,7 +58,16 @@ function selection(values: string[], labels?: string[]): Array<{ value: string;
}));
}
function reduceDraft(draft: RecordingDraftStep, id: number, redactValues: boolean): StepV3 | null {
interface ReduceDraftOptions {
includeTabSwitches: boolean;
redactValues: boolean;
}
function reduceDraft(
draft: RecordingDraftStep,
id: number,
options: ReduceDraftOptions,
): StepV3 | null {
if (!shouldIncludeDraft(draft)) return null;
const state = draft.preStateId ?? draft.postStateId;
const resultState = draft.postStateId ?? draft.preStateId;
@@ -68,6 +77,8 @@ function reduceDraft(draft: RecordingDraftStep, id: number, redactValues: boolea
switch (draft.op) {
case "navigate":
return { op: "navigate", ...common, to: draft.url, cause: navigationCause(draft) };
case "switch_tab":
return options.includeTabSwitches ? { op: "switch_tab", ...common } : null;
case "click":
return {
op: "click",
@@ -81,7 +92,7 @@ function reduceDraft(draft: RecordingDraftStep, id: number, redactValues: boolea
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
};
case "fill":
const fillIsRedacted = redactValues || draft.redacted === true;
const fillIsRedacted = options.redactValues || draft.redacted === true;
return {
op: "fill",
...common,
@@ -105,7 +116,7 @@ function reduceDraft(draft: RecordingDraftStep, id: number, redactValues: boolea
op: "select",
...common,
target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
...(!redactValues ? { selection: selection(draft.values, draft.labels) } : {}),
...(!options.redactValues ? { selection: selection(draft.values, draft.labels) } : {}),
};
case "scroll":
return { op: "scroll", ...common };
@@ -119,12 +130,15 @@ export interface ReducedTraceV3 {
export function reduceTraceStepsV3(
steps: RecordingDraftStep[],
options: { redactValues?: boolean } = {},
options: { includeTabSwitches?: boolean; redactValues?: boolean } = {},
): ReducedTraceV3 {
const output: StepV3[] = [];
const stepIdByDraftId = new Map<number, number>();
for (const { draft, draftIds } of collapseRedirects(steps)) {
const step = reduceDraft(draft, output.length + 1, options.redactValues ?? false);
const step = reduceDraft(draft, output.length + 1, {
includeTabSwitches: options.includeTabSwitches === true,
redactValues: options.redactValues === true,
});
if (!step) continue;
output.push(step);
for (const draftId of draftIds) stepIdByDraftId.set(draftId, step.id);
@@ -69,6 +69,7 @@ export type RecordingDraftStep =
DraftTarget &
DraftNavigationEffect)
| ({ op: "scroll" } & DraftStateLink)
| ({ op: "switch_tab" } & DraftStateLink)
| ({
op: "navigate";
url: string;
@@ -15,7 +15,12 @@ import {
const AGENT_WINDOW_ID = 100;
const TAB_ID = 4;
const START_URL = "https://example.com/";
const RECORD_START_V3 = { session_id: "abcd", url: START_URL, trace_version: 3 as const };
const RECORD_START_V3 = {
session_id: "abcd",
url: START_URL,
trace_version: 3 as const,
supports_tab_switch_steps: true,
};
const stepSequenceByProducer = new Map<string, number>();
function asTraceV3(trace: RecordedTrace): TraceV3 {
@@ -48,6 +53,7 @@ function chromeEvent<T extends (...args: never[]) => unknown>() {
function installChrome() {
const runtimeOnMessage = chromeEvent<RuntimeListener>();
const tabsOnActivated = chromeEvent<(activeInfo: chrome.tabs.TabActiveInfo) => unknown>();
const webNavigationOnCompleted =
chromeEvent<(details: chrome.webNavigation.WebNavigationFramedCallbackDetails) => unknown>();
const webNavigationOnCommitted =
@@ -57,7 +63,7 @@ function installChrome() {
vi.stubGlobal("chrome", {
runtime: { onMessage: runtimeOnMessage },
tabs: {
onActivated: chromeEvent(),
onActivated: tabsOnActivated,
onCreated: chromeEvent(),
onUpdated: chromeEvent(),
},
@@ -66,7 +72,7 @@ function installChrome() {
onCommitted: webNavigationOnCommitted,
},
});
return { runtimeOnMessage, webNavigationOnCompleted, webNavigationOnCommitted };
return { runtimeOnMessage, tabsOnActivated, webNavigationOnCompleted, webNavigationOnCommitted };
}
function fakeManager() {
@@ -118,7 +124,10 @@ type FakeCdp = CdpRunner & {
const LONG_IDLE_MS = 10_000;
/** AX-only page: DOMSnapshot calls throw so capture falls back to the AX tree. */
function makeFakeCdp(trees?: unknown[], options?: { failCaptures?: boolean }): FakeCdp {
function makeFakeCdp(
trees?: unknown[],
options?: { failCaptures?: boolean; treesByTab?: Record<number, unknown> },
): FakeCdp {
type EventListener = (source: chrome.debugger.Debuggee, method: string, params: unknown) => void;
const events: EventListener[] = [];
const script = [...(trees ?? [])];
@@ -153,6 +162,8 @@ function makeFakeCdp(trees?: unknown[], options?: { failCaptures?: boolean }): F
"Accessibility.enable": () => ({}),
"Accessibility.getFullAXTree": (_params, _tabId) => {
if (failing) throw new Error("Execution context was destroyed");
const tabTree = options?.treesByTab?.[_tabId];
if (tabTree) return tabTree;
if (script.length === 0) return axTree("Example Domain", ["Submit"]);
return script.length === 1 ? script[0] : script.shift();
},
@@ -202,6 +213,31 @@ function makeTabsApi() {
};
}
function makeMultiTabsApi(tabs: chrome.tabs.Tab[]) {
const byId = new Map(tabs.flatMap((tab) => (typeof tab.id === "number" ? [[tab.id, tab]] : [])));
return {
get: async (tabId: number) => {
const tab = byId.get(tabId);
if (!tab) throw new Error(`tab ${tabId} closed`);
return tab;
},
query: async (query: chrome.tabs.QueryInfo) =>
[...byId.values()].filter(
(tab) =>
(query.windowId === undefined || tab.windowId === query.windowId) &&
(query.active === undefined || tab.active === query.active),
),
activate(tabId: number) {
for (const tab of byId.values()) tab.active = tab.id === tabId;
},
goTo(tabId: number, url: string, title: string) {
const tab = byId.get(tabId);
if (!tab) throw new Error(`tab ${tabId} closed`);
Object.assign(tab, { url, title });
},
};
}
/** Outlast a settle on a quiet page plus the per-tab observation cooldown. */
function settleWait(): Promise<void> {
return new Promise((resolve) => setTimeout(resolve, 800));
@@ -212,16 +248,19 @@ function runtimeOnMessageEmit(
requestId: string,
step: unknown,
tabId = TAB_ID,
): void {
active = true,
): ReturnType<typeof vi.fn> {
const producerId = `test-producer-${tabId}`;
const producerKey = `${requestId}:${producerId}`;
const sequence = (stepSequenceByProducer.get(producerKey) ?? 0) + 1;
stepSequenceByProducer.set(producerKey, sequence);
const sendResponse = vi.fn();
chromeApi.runtimeOnMessage.emit(
{ type: RECORD_STEP, requestId, producerId, sequence, step },
{ tab: { id: tabId } } as chrome.runtime.MessageSender,
() => {},
{ tab: { id: tabId, active } } as chrome.runtime.MessageSender,
sendResponse,
);
return sendResponse;
}
describe("recorded user steps reach the exported trace", () => {
@@ -1173,4 +1212,335 @@ describe("recorded user steps reach the exported trace", () => {
);
expect(result).toMatchObject({ code: "invalid_params" });
});
it("records tab transitions and binds actions to each tab's observation", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const secondUrl = "https://example.com/second";
const tabsApi = makeMultiTabsApi([
{
id: TAB_ID,
windowId: AGENT_WINDOW_ID,
active: true,
status: "complete",
url: START_URL,
title: "First tab",
} as chrome.tabs.Tab,
{
id: 5,
windowId: AGENT_WINDOW_ID,
active: false,
status: "complete",
url: secondUrl,
title: "Second tab",
} as chrome.tabs.Tab,
]);
const cdp = makeFakeCdp(undefined, {
treesByTab: {
[TAB_ID]: axTree("First tab", ["First action"]),
5: axTree("Second tab", ["Second action"]),
},
});
let requestId = "";
const sendToTab = vi.fn(async (_tabId: number, message: unknown) => {
const typed = message as { type?: string; requestId?: string };
if (typed.type === RECORD_START && typed.requestId) requestId = typed.requestId;
return { ok: true };
});
const deps = { tabsApi, sendToTab, cdp };
await handleRecordStart(manager, RECORD_START_V3, deps);
attachRecordStepListener(deps);
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
await settleWait();
runtimeOnMessageEmit(
chromeApi,
requestId,
{
op: "click",
page_url: secondUrl,
target: { role: "button", name: "Second action", tag: "button" },
},
5,
);
await settleWait();
tabsApi.activate(TAB_ID);
chromeApi.tabsOnActivated.emit({ tabId: TAB_ID, windowId: AGENT_WINDOW_ID });
await settleWait();
runtimeOnMessageEmit(chromeApi, requestId, {
op: "click",
page_url: START_URL,
target: { role: "button", name: "First action", tag: "button" },
});
await settleWait();
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
const trace = asTraceV3((stopped as RecordStopResult).trace);
const stateUrl = (stateId: string) => trace.states.find((state) => state.id === stateId)?.url;
const switches = trace.steps.filter((step) => step.op === "switch_tab");
const clicks = trace.steps.filter((step) => step.op === "click");
expect(switches).toHaveLength(2);
expect(clicks).toHaveLength(2);
expect(stateUrl(switches[0]!.state)).toBe(START_URL);
expect(stateUrl(switches[0]!.result.state)).toBe(secondUrl);
expect(stateUrl(clicks[0]!.state)).toBe(secondUrl);
expect(stateUrl(switches[1]!.result.state)).toBe(START_URL);
expect(stateUrl(clicks[1]!.state)).toBe(START_URL);
});
it("keeps switch_tab internal when the v3 caller did not advertise support", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const tabsApi = makeMultiTabsApi([
{
id: TAB_ID,
windowId: AGENT_WINDOW_ID,
active: true,
status: "complete",
url: START_URL,
} as chrome.tabs.Tab,
{
id: 5,
windowId: AGENT_WINDOW_ID,
active: false,
status: "complete",
url: "https://example.com/second",
} as chrome.tabs.Tab,
]);
const deps = { tabsApi, sendToTab: vi.fn(async () => ({ ok: true })), cdp: makeFakeCdp() };
await handleRecordStart(
manager,
{ session_id: "abcd", url: START_URL, trace_version: 3 },
deps,
);
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
await settleWait();
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
const trace = asTraceV3((stopped as RecordStopResult).trace);
expect(trace.steps.some((step) => step.op === "switch_tab")).toBe(false);
});
it("keeps the latest rapid tab activation when an earlier rearm acknowledges late", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const tabsApi = makeMultiTabsApi(
[TAB_ID, 5, 6].map(
(id) =>
({
id,
windowId: AGENT_WINDOW_ID,
active: id === TAB_ID,
status: "complete",
url: id === TAB_ID ? START_URL : `https://example.com/tab-${id}`,
}) as chrome.tabs.Tab,
),
);
let releaseSecond!: () => void;
const secondGate = new Promise<void>((resolve) => {
releaseSecond = resolve;
});
let announceSecond!: () => void;
const secondStarted = new Promise<void>((resolve) => {
announceSecond = resolve;
});
const sendToTab = vi.fn(async (tabId: number, message: unknown) => {
if ((message as { type?: string }).type === RECORD_START && tabId === 5) {
announceSecond();
await secondGate;
}
return { ok: true };
});
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
await handleRecordStart(manager, RECORD_START_V3, deps);
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
await secondStarted;
tabsApi.activate(6);
chromeApi.tabsOnActivated.emit({ tabId: 6, windowId: AGENT_WINDOW_ID });
await settleWait();
releaseSecond();
await new Promise((resolve) => setTimeout(resolve, 50));
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
const trace = asTraceV3((stopped as RecordStopResult).trace);
const switches = trace.steps.filter((step) => step.op === "switch_tab");
const stateUrl = (stateId: string) => trace.states.find((state) => state.id === stateId)?.url;
expect(switches).toHaveLength(1);
expect(stateUrl(switches[0]!.result.state)).toBe("https://example.com/tab-6");
});
it("keeps pending action settles scoped to their tab during an immediate switch", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const secondUrl = "https://example.com/second";
const tabsApi = makeMultiTabsApi([
{
id: TAB_ID,
windowId: AGENT_WINDOW_ID,
active: true,
status: "complete",
url: START_URL,
} as chrome.tabs.Tab,
{
id: 5,
windowId: AGENT_WINDOW_ID,
active: false,
status: "complete",
url: secondUrl,
} as chrome.tabs.Tab,
]);
let requestId = "";
const sendToTab = vi.fn(async (_tabId: number, message: unknown) => {
const typed = message as { type?: string; requestId?: string };
if (typed.type === RECORD_START && typed.requestId) requestId = typed.requestId;
return { ok: true };
});
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
await handleRecordStart(manager, RECORD_START_V3, deps);
attachRecordStepListener(deps);
runtimeOnMessageEmit(chromeApi, requestId, {
op: "click",
page_url: START_URL,
target: { role: "button", name: "First action", tag: "button" },
});
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
runtimeOnMessageEmit(
chromeApi,
requestId,
{
op: "click",
page_url: secondUrl,
target: { role: "button", name: "Second action", tag: "button" },
},
5,
);
await settleWait();
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
const trace = asTraceV3((stopped as RecordStopResult).trace);
const stateUrl = (stateId: string) => trace.states.find((state) => state.id === stateId)?.url;
expect(trace.steps.map((step) => step.op)).toEqual(["click", "switch_tab", "click"]);
expect(stateUrl(trace.steps[0]!.result.state)).toBe(START_URL);
expect(stateUrl(trace.steps[1]!.result.state)).toBe(secondUrl);
expect(stateUrl(trace.steps[2]!.state)).toBe(secondUrl);
});
it("tracks the same navigation URL independently for each tab", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const tabsApi = makeMultiTabsApi([
{
id: TAB_ID,
windowId: AGENT_WINDOW_ID,
active: true,
status: "complete",
url: START_URL,
} as chrome.tabs.Tab,
{
id: 5,
windowId: AGENT_WINDOW_ID,
active: false,
status: "complete",
url: "https://example.com/second",
} as chrome.tabs.Tab,
]);
const deps = { tabsApi, sendToTab: vi.fn(async () => ({ ok: true })), cdp: makeFakeCdp() };
await handleRecordStart(manager, RECORD_START_V3, deps);
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
await settleWait();
tabsApi.goTo(5, START_URL, "Same URL in second tab");
chromeApi.webNavigationOnCommitted.emit({
tabId: 5,
frameId: 0,
url: START_URL,
transitionType: "typed",
transitionQualifiers: ["from_address_bar"],
} as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails);
await settleWait();
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
const trace = asTraceV3((stopped as RecordStopResult).trace);
expect(
trace.steps.filter((step) => step.op === "navigate" && step.to === START_URL),
).toHaveLength(1);
});
it("drops late input and frame navigation from an inactive tab", async () => {
const chromeApi = installChrome();
const manager = fakeManager();
const secondUrl = "https://example.com/second";
const tabsApi = makeMultiTabsApi([
{
id: TAB_ID,
windowId: AGENT_WINDOW_ID,
active: true,
status: "complete",
url: START_URL,
} as chrome.tabs.Tab,
{
id: 5,
windowId: AGENT_WINDOW_ID,
active: false,
status: "complete",
url: secondUrl,
} as chrome.tabs.Tab,
]);
let requestId = "";
const sendToTab = vi.fn(async (_tabId: number, message: unknown) => {
const typed = message as { type?: string; requestId?: string };
if (typed.type === RECORD_START && typed.requestId) requestId = typed.requestId;
return { ok: true };
});
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
await handleRecordStart(manager, RECORD_START_V3, deps);
attachRecordStepListener(deps);
tabsApi.activate(5);
chromeApi.tabsOnActivated.emit({ tabId: 5, windowId: AGENT_WINDOW_ID });
await settleWait();
const lateAck = runtimeOnMessageEmit(
chromeApi,
requestId,
{
op: "fill",
page_url: START_URL,
target: { role: "textbox", name: "Draft", tag: "input" },
value: "late",
},
TAB_ID,
false,
);
expect(lateAck).toHaveBeenCalledWith({ ok: true, sequence: 1 });
chromeApi.webNavigationOnCommitted.emit({
tabId: TAB_ID,
frameId: 0,
url: "https://example.com/late",
transitionType: "link",
transitionQualifiers: [],
} as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails);
chromeApi.webNavigationOnCommitted.emit({
tabId: 5,
frameId: 7,
url: "https://frame.example/late",
transitionType: "link",
transitionQualifiers: [],
} as unknown as chrome.webNavigation.WebNavigationTransitionCallbackDetails);
await new Promise((resolve) => setTimeout(resolve, 50));
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
const trace = asTraceV3((stopped as RecordStopResult).trace);
expect(trace.steps.map((step) => step.op)).toEqual(["switch_tab"]);
});
});
+116 -71
View File
@@ -19,7 +19,12 @@ import {
type RecordStopMessage,
} from "@/lib/record-bridge";
import { RecordingObservationRuntime } from "@/lib/recording/recording-runtime";
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording/step-buffer";
import {
appendRecordedPayload,
observeRecordedNavigation,
type RecordingStepBuffer,
} from "@/lib/recording/step-buffer";
import { RecordingTabCoordinator, type TabActivation } from "@/lib/recording/tab-coordinator";
import { buildTraceV2 } from "@/lib/recording/trace-reducer-v2";
import type { RecordingDraftStep } from "@/lib/recording/types";
import type { SessionManager } from "@/session-manager/manager";
@@ -48,7 +53,7 @@ import {
interface ActiveRecording {
requestId: string;
tabId: number;
tabs: RecordingTabCoordinator;
agentWindowId: number;
startUrl?: string;
purpose?: string;
@@ -56,14 +61,12 @@ interface ActiveRecording {
startedAt: string;
startedAtMs: number;
traceVersion: 2 | 3;
supportsTabSwitchSteps: boolean;
finishPromise: Promise<RecordedTrace>;
resolveFinish: (trace: RecordedTrace) => void;
rejectFinish: (err: Error) => void;
settled: boolean;
finishAttempt: Promise<RecordedTrace | null> | null;
currentUrl?: string;
pendingNavigation: boolean;
pendingNavigationDeadline?: number;
observation: RecordingObservationRuntime | null;
stoppedBy: StopReason;
/** Navigation callbacks tracked from event receipt through action enqueue. */
@@ -79,8 +82,13 @@ interface ActiveRecording {
lastStepSequenceByProducer: Map<string, number>;
}
function enqueueRecordingAction(recording: ActiveRecording, task: () => Promise<void>): void {
recording.actionQueue = recording.actionQueue.then(task, task).catch(() => {});
function enqueueRecordingAction(
recording: ActiveRecording,
task: () => Promise<void>,
): Promise<void> {
const queued = recording.actionQueue.then(task, task);
recording.actionQueue = queued.catch(() => {});
return queued;
}
const recordings = new Map<string, ActiveRecording>();
@@ -198,6 +206,7 @@ function buildTrace(recording: ActiveRecording): RecordedTrace {
startUrl: recording.startUrl,
stoppedBy: recording.stoppedBy,
bskVersion: BSK_TRACE_VERSION,
includeTabSwitches: recording.supportsTabSwitchSteps,
});
}
if (recording.traceVersion === 3) {
@@ -211,6 +220,38 @@ function buildTrace(recording: ActiveRecording): RecordedTrace {
});
}
function stepBufferFor(
recording: ActiveRecording,
tabId: number,
fallbackUrl?: string,
): RecordingStepBuffer {
return {
steps: recording.steps,
navigation: recording.tabs.navigation(tabId, fallbackUrl),
};
}
async function activateRecordingTab(
recording: ActiveRecording,
targetTabId: number,
): Promise<void> {
const previousTabId = recording.tabs.currentTabId;
if (targetTabId === previousTabId) return;
const transition = recording.observation
? await recording.observation.captureTabTransition(previousTabId, targetTabId)
: {};
const { targetUrl, ...stateLinks } = transition;
const draft: Extract<RecordingDraftStep, { op: "switch_tab" }> = {
op: "switch_tab",
...stateLinks,
};
recording.steps.push(draft);
recording.observation?.bindTabTransition(draft, recording.steps.length);
recording.tabs.navigation(targetTabId, targetUrl);
recording.tabs.commit(targetTabId);
}
async function processRecordedStep(
recording: ActiveRecording,
draftIndex: number,
@@ -307,7 +348,8 @@ export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): (
if (!isRecordStepMessage(message)) return false;
for (const recording of recordings.values()) {
if (recording.requestId !== message.requestId) continue;
const sourceTabId = sender.tab?.id ?? recording.tabId;
const sourceTabId = sender.tab?.id ?? recording.tabs.currentTabId;
const sourceWasActive = sender.tab?.active ?? sourceTabId === recording.tabs.activeTabId;
const producerKey = `${sourceTabId}:${message.producerId}`;
const expectedSequence = (recording.lastStepSequenceByProducer.get(producerKey) ?? 0) + 1;
if (message.sequence < expectedSequence) {
@@ -325,9 +367,19 @@ export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): (
recording.lastStepSequenceByProducer.set(producerKey, message.sequence);
enqueueRecordingAction(recording, async () => {
await recording.observation?.flushRedirects();
if (sourceTabId !== recording.tabs.currentTabId) {
if (!sourceWasActive) return;
if (sourceTabId !== recording.tabs.activeTabId)
recording.tabs.noteActivation(sourceTabId);
await activateRecordingTab(recording, sourceTabId);
}
await recording.observation?.flushRedirects(sourceTabId);
const targetHint = message.step.geometry ? { geometry: message.step.geometry } : undefined;
const draftIndex = appendRecordedPayload(recording, message.step, targetHint);
const draftIndex = appendRecordedPayload(
stepBufferFor(recording, sourceTabId, message.step.page_url),
message.step,
targetHint,
);
if (draftIndex !== null) {
await processRecordedStep(recording, draftIndex, sourceTabId);
}
@@ -359,7 +411,7 @@ export function attachRecordFinishListener(deps: RecordDeps = getDefaultDeps()):
function findRecordingByTabId(tabId: number): ActiveRecording | null {
for (const recording of recordings.values()) {
if (recording.tabId === tabId && !recording.settled) return recording;
if (recording.tabs.currentTabId === tabId && !recording.settled) return recording;
}
return null;
}
@@ -396,7 +448,7 @@ async function clearRearmTimersForRecording(
recording: ActiveRecording,
deps: RecordDeps,
): Promise<void> {
clearRearmTimer(recording.tabId);
clearRearmTimer(recording.tabs.currentTabId);
try {
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
for (const tab of tabs) {
@@ -412,12 +464,12 @@ async function stopRecordingOnAllAgentTabs(
deps: RecordDeps,
): Promise<void> {
const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
let tabIds = [recording.tabId];
let tabIds = [recording.tabs.currentTabId];
try {
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
tabIds = [
...new Set([
recording.tabId,
recording.tabs.currentTabId,
...tabs.flatMap((tab) => (typeof tab.id === "number" ? [tab.id] : [])),
]),
];
@@ -428,11 +480,11 @@ async function stopRecordingOnAllAgentTabs(
for (const tabId of tabIds) {
try {
const response = await deps.sendToTab(tabId, stopMsg);
if (tabId === recording.tabId && !isRecordStartAck(response)) {
if (tabId === recording.tabs.currentTabId && !isRecordStartAck(response)) {
throw new Error("content script did not confirm recorded steps");
}
} catch {
if (tabId === recording.tabId) {
if (tabId === recording.tabs.currentTabId) {
throw new Error("failed to flush recorded steps");
}
}
@@ -450,6 +502,7 @@ async function rearmRecording(
recording: ActiveRecording,
targetTabId: number,
deps: RecordDeps,
activation?: TabActivation,
): Promise<boolean> {
// Do NOT toggle automation-bypass here: each retry used to increment the
// content-script counter, and a single stop decrement left the ControlOverlay
@@ -463,7 +516,10 @@ async function rearmRecording(
};
try {
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
recording.tabId = targetTabId;
if (activation) {
if (!recording.tabs.isLatest(activation)) return true;
await enqueueRecordingAction(recording, () => activateRecordingTab(recording, targetTabId));
}
return true;
} catch {
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
@@ -474,7 +530,7 @@ async function rearmRecording(
return false;
}
function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
function scheduleRearmForTab(tabId: number, deps: RecordDeps, activation?: TabActivation): void {
const existing = rearmTimers.get(tabId);
if (existing) clearTimeout(existing);
rearmTimers.set(
@@ -483,7 +539,7 @@ function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
rearmTimers.delete(tabId);
void (async () => {
const current = await findRecordingForTab(tabId, deps);
if (current) await rearmRecording(current, tabId, deps);
if (current) await rearmRecording(current, tabId, deps, activation);
})();
}, RECORD_REARM_DEBOUNCE_MS),
);
@@ -502,7 +558,12 @@ export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): ()
};
const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => {
scheduleRearmForTab(activeInfo.tabId, deps);
for (const recording of recordings.values()) {
if (recording.settled || recording.agentWindowId !== activeInfo.windowId) continue;
const activation = recording.tabs.noteActivation(activeInfo.tabId);
scheduleRearmForTab(activeInfo.tabId, deps, activation);
return;
}
};
chrome.tabs.onCreated.addListener(onCreated);
@@ -522,54 +583,39 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
transitionQualifiers?: string[],
) => {
if (!url) return;
const direct = findRecordingByTabId(tabId);
const candidates = direct
? direct.acceptingNavigation
? [direct]
: []
: [...recordings.values()].filter(
(recording) => !recording.settled && recording.acceptingNavigation,
const candidates = [...recordings.values()].filter(
(recording) =>
!recording.settled && recording.acceptingNavigation && recording.tabs.activeTabId === tabId,
);
for (const recording of candidates) {
const queued = enqueueRecordingAction(recording, async () => {
if (tabId !== recording.tabs.currentTabId) {
await activateRecordingTab(recording, tabId);
}
const result = observeRecordedNavigation(
stepBufferFor(recording, tabId, url),
url,
causedByAction,
transitionType,
transitionQualifiers,
);
if (candidates.length === 0) return;
let resolveTracked!: () => void;
const tracked = new Promise<void>((resolve) => {
resolveTracked = resolve;
});
for (const candidate of candidates) candidate.navigationCallbacks.add(tracked);
void (async () => {
try {
const recording = await findRecordingForTab(tabId, deps);
if (!recording || !recording.acceptingNavigation || !candidates.includes(recording)) {
if (result.kind === "coalesce_redirect") {
recording.observation?.scheduleRedirect(tabId, recording.steps, result.url);
return;
}
enqueueRecordingAction(recording, async () => {
const result = observeRecordedNavigation(
recording,
url,
causedByAction,
transitionType,
transitionQualifiers,
);
if (result.kind === "coalesce_redirect") {
recording.observation?.scheduleRedirect(tabId, recording.steps, result.url);
return;
}
if (result.kind === "noop") return;
recording.observation?.clearRedirect(tabId);
if (result.kind === "appended") {
await processRecordedStep(recording, result.index, tabId);
} else {
recording.observation?.scheduleSettle(tabId, recording.steps, result.index);
}
});
} finally {
for (const candidate of candidates) candidate.navigationCallbacks.delete(tracked);
resolveTracked();
}
})();
if (result.kind === "noop") return;
recording.observation?.clearRedirect(tabId);
if (result.kind === "appended") {
await processRecordedStep(recording, result.index, tabId);
} else {
recording.observation?.scheduleSettle(tabId, recording.steps, result.index);
}
});
const tracked = queued.catch(() => {});
recording.navigationCallbacks.add(tracked);
void tracked.finally(() => recording.navigationCallbacks.delete(tracked));
}
};
const onMainFrameComplete = (tabId: number, url?: string) => {
void (async () => {
@@ -651,7 +697,8 @@ export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()):
sendResponse({ active: false });
return;
}
await rearmRecording(recording, tabId, deps);
const activation = sender.tab?.active ? recording.tabs.noteActivation(tabId) : undefined;
await rearmRecording(recording, tabId, deps, activation);
sendResponse({
active: true,
requestId: recording.requestId,
@@ -716,7 +763,7 @@ async function finishRecordingAttempt(
if (!(await drainRecordingToStability(recording))) {
return null;
}
await recording.observation?.settleTrailing(recording.tabId, recording.steps);
await recording.observation?.settleTrailing(recording.tabs.currentTabId, recording.steps);
recording.settled = true;
recordings.delete(sessionId);
@@ -768,7 +815,7 @@ export async function handleRecordStart(
const redactValues = params.redact_values ?? false;
recordings.set(params.session_id, {
requestId,
tabId: target.tabId,
tabs: new RecordingTabCoordinator(target.tabId, navigateUrl),
agentWindowId: ctx.agentWindowId,
startUrl: navigateUrl,
...(params.purpose ? { purpose: params.purpose } : {}),
@@ -776,14 +823,12 @@ export async function handleRecordStart(
startedAt: new Date(startedAtMs).toISOString(),
startedAtMs,
traceVersion: traceVersionOrErr,
supportsTabSwitchSteps: params.supports_tab_switch_steps === true,
finishPromise,
resolveFinish,
rejectFinish,
settled: false,
finishAttempt: null,
currentUrl: navigateUrl,
pendingNavigation: false,
pendingNavigationDeadline: undefined,
observation:
traceVersionOrErr === 3 && deps.cdp
? new RecordingObservationRuntime({
@@ -886,7 +931,7 @@ export async function handleRecordStart(
return cancelledError();
}
active.startUrl = startUrl;
active.currentUrl = startUrl;
active.tabs.navigation(target.tabId, startUrl).currentUrl = startUrl;
if (isContentScriptRestrictedUrl(startUrl)) {
await abortPending(false);
+3
View File
@@ -780,6 +780,7 @@ export interface SelectedOptionV3 {
export type StepV3 =
| ({ op: "navigate" } & StepCommonV3 & { to: string; cause: NavigationCause })
| ({ op: "switch_tab" } & StepCommonV3)
| ({ op: "click" } & StepCommonV3 & { target: TargetDescriptorV3 })
| ({ op: "hover" } & StepCommonV3 & { target: TargetDescriptorV3 })
| ({ op: "fill" } & StepCommonV3 & {
@@ -823,6 +824,8 @@ export interface RecordStartParams {
redact_values?: boolean;
/** Omitted means v2; `3` requests a state-linked v3 trace. */
trace_version?: number;
/** Client can decode the v3 `switch_tab` step variant. */
supports_tab_switch_steps?: boolean;
}
export interface RecordStartResult {
+1
View File
@@ -115,6 +115,7 @@ fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError>
max_page_tokens: args.max_page_tokens,
redact_values: Some(args.redact_values),
trace_version: Some(TRACE_VERSION_V3),
supports_tab_switch_steps: Some(true),
};
let start_result = business_rpc::call::<RecordStartParams, RecordStartResult>(
info.sock_path.clone(),
@@ -480,6 +480,36 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
@@ -29,6 +29,13 @@
"session_id": {
"type": "string"
},
"supports_tab_switch_steps": {
"description": "Client can decode the v3 `switch_tab` step variant.",
"type": [
"boolean",
"null"
]
},
"tab_id": {
"type": [
"integer",
@@ -480,6 +480,36 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
+30
View File
@@ -469,6 +469,36 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
@@ -431,6 +431,36 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
@@ -41,6 +41,36 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
+30
View File
@@ -173,6 +173,36 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"result",
"state"
],
"properties": {
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"switch_tab"
]
},
"result": {
"$ref": "#/definitions/StepResultV3"
},
"state": {
"description": "Observation id immediately before this action.",
"type": "string"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
+20
View File
@@ -39,6 +39,9 @@ pub struct RecordStartParams {
/// Desired trace export format. Omitted means v2; `3` requests v3.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub trace_version: Option<u32>,
/// Client can decode the v3 `switch_tab` step variant.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub supports_tab_switch_steps: Option<bool>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
@@ -423,6 +426,19 @@ mod tests {
assert_eq!(round, step);
}
#[test]
fn step_switch_tab_round_trips() {
let step = StepV3::SwitchTab {
common: sample_common(2, "s2", "s5"),
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["op"], "switch_tab");
assert_eq!(v["state"], "s2");
assert_eq!(v["result"]["state"], "s5");
let round: StepV3 = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn trace_v3_round_trips() {
let trace = sample_trace();
@@ -463,10 +479,12 @@ mod tests {
max_page_tokens: None,
redact_values: None,
trace_version: None,
supports_tab_switch_steps: None,
};
let legacy_value = serde_json::to_value(legacy).unwrap();
assert!(legacy_value.get("trace_version").is_none());
assert!(legacy_value.get("max_page_tokens").is_none());
assert!(legacy_value.get("supports_tab_switch_steps").is_none());
let v3 = RecordStartParams {
session_id: "session".into(),
@@ -476,11 +494,13 @@ mod tests {
max_page_tokens: Some(4_000),
redact_values: Some(true),
trace_version: Some(TRACE_VERSION_V3),
supports_tab_switch_steps: Some(true),
};
let v3_value = serde_json::to_value(v3).unwrap();
assert_eq!(v3_value["trace_version"], TRACE_VERSION_V3);
assert_eq!(v3_value["max_page_tokens"], 4_000);
assert_eq!(v3_value["redact_values"], true);
assert_eq!(v3_value["supports_tab_switch_steps"], true);
}
#[test]
@@ -126,6 +126,10 @@ pub enum StepV3 {
to: String,
cause: NavigationCause,
},
SwitchTab {
#[serde(flatten)]
common: StepCommonV3,
},
Click {
#[serde(flatten)]
common: StepCommonV3,
@@ -340,6 +344,19 @@ mod tests {
assert_eq!(round, step);
}
#[test]
fn step_switch_tab_round_trips() {
let step = StepV3::SwitchTab {
common: sample_common(2, "s2", "s5"),
};
let v = serde_json::to_value(&step).unwrap();
assert_eq!(v["op"], "switch_tab");
assert_eq!(v["state"], "s2");
assert_eq!(v["result"]["state"], "s5");
let round: StepV3 = serde_json::from_value(v).unwrap();
assert_eq!(round, step);
}
#[test]
fn trace_v3_round_trips() {
let trace = sample_trace();