fix(recording): using unified geometry

apps/extension/src/lib/recording/document-settle.ts

@@ -0,0 +1,134 @@
+import type { CdpTarget } from "@/browser-driver/frame-graph";
+import { type CdpRunner, sendToCdpTarget } from "@/tools/shared";
+
+const SETTLE_MIN_MS = 150;
+const SETTLE_QUIET_MS = 250;
+const SETTLE_MAX_MS = 2_000;
+const SETTLE_POLL_MS = 60;
+const SETTLE_WORLD_NAME = "__bsk_record_settle__";
+
+const QUIET_PROBE = `(() => {
+  const scope = window;
+  let probe = scope.__bskRecordQuiet;
+  if (!probe) {
+    probe = { changedAt: Date.now() };
+    const observer = new MutationObserver(() => {
+      probe.changedAt = Date.now();
+    });
+    observer.observe(document.documentElement, {
+      subtree: true,
+      childList: true,
+      attributes: true,
+      characterData: true,
+    });
+    scope.__bskRecordQuiet = probe;
+  }
+  return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState };
+})()`;
+
+export interface DocumentSettleScope {
+  target: CdpTarget;
+  /** Omit only for the target's root document. */
+  frameId?: string;
+}
+
+type SettleOutcome = "quiet" | "timeout" | "cancelled";
+
+interface QuietProbe {
+  idleMs: number;
+  readyState: string;
+}
+
+interface ProbeContext {
+  executionContextId?: number;
+}
+
+function sleep(ms: number, signal?: AbortSignal): Promise<void> {
+  return new Promise((resolve) => {
+    if (signal?.aborted) return resolve();
+    const timer = setTimeout(resolve, ms);
+    signal?.addEventListener(
+      "abort",
+      () => {
+        clearTimeout(timer);
+        resolve();
+      },
+      { once: true },
+    );
+  });
+}
+
+async function executionContextForScope(
+  cdp: CdpRunner,
+  scope: DocumentSettleScope,
+): Promise<number | undefined> {
+  if (!scope.frameId) return undefined;
+  const result = await sendToCdpTarget<{ executionContextId?: number }>(
+    cdp,
+    scope.target,
+    "Page.createIsolatedWorld",
+    {
+      frameId: scope.frameId,
+      worldName: SETTLE_WORLD_NAME,
+      grantUniveralAccess: false,
+    },
+  );
+  return result.executionContextId;
+}
+
+async function readQuietProbe(
+  cdp: CdpRunner,
+  scope: DocumentSettleScope,
+  context: ProbeContext,
+): Promise<QuietProbe | null> {
+  try {
+    if (scope.frameId && context.executionContextId === undefined) {
+      context.executionContextId = await executionContextForScope(cdp, scope);
+      if (context.executionContextId === undefined) return null;
+    }
+    const reply = await sendToCdpTarget<{ result?: { value?: unknown } }>(
+      cdp,
+      scope.target,
+      "Runtime.evaluate",
+      {
+        expression: QUIET_PROBE,
+        returnByValue: true,
+        ...(context.executionContextId !== undefined
+          ? { contextId: context.executionContextId }
+          : {}),
+      },
+    );
+    const value = reply.result?.value;
+    if (!value || typeof value !== "object") return null;
+    const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown };
+    if (typeof idleMs !== "number" || typeof readyState !== "string") return null;
+    return { idleMs, readyState };
+  } catch {
+    context.executionContextId = undefined;
+    return null;
+  }
+}
+
+export async function waitForDocumentSettled(
+  cdp: CdpRunner,
+  scope: DocumentSettleScope,
+  options: { signal?: AbortSignal } = {},
+): Promise<SettleOutcome> {
+  const startedAt = Date.now();
+  const floor = startedAt + SETTLE_MIN_MS;
+  const deadline = startedAt + SETTLE_MAX_MS;
+  const context: ProbeContext = {};
+
+  for (;;) {
+    if (options.signal?.aborted) return "cancelled";
+    await sleep(SETTLE_POLL_MS, options.signal);
+    if (options.signal?.aborted) return "cancelled";
+
+    const probe = await readQuietProbe(cdp, scope, context);
+    const now = Date.now();
+    if (now < floor) continue;
+    if (now >= deadline) return "timeout";
+    if (!probe || probe.readyState === "loading") continue;
+    if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet";
+  }
+}

apps/extension/src/lib/recording/draft-policy.ts

@@ -0,0 +1,34 @@
+import type { RecordingDraftStep } from "./types";
+
+const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
+const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
+
+export function shouldRecordPress(
+  key: string,
+  modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
+): boolean {
+  if (MODIFIER_ONLY_KEYS.has(key)) return false;
+  const mods = modifiers ?? [];
+  const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
+  if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
+  if (key === "Enter" || key === "Escape") return true;
+  if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
+  return false;
+}
+
+export function shouldIncludeDraft(step: RecordingDraftStep): boolean {
+  return step.op !== "press" || shouldRecordPress(step.key, step.modifiers);
+}
+
+export function resolveDraftStartUrl(
+  drafts: RecordingDraftStep[],
+  explicitStartUrl?: string,
+  fallbackUrl?: string,
+): string {
+  if (explicitStartUrl) return explicitStartUrl;
+  const navigation = drafts.find(
+    (step): step is Extract<RecordingDraftStep, { op: "navigate" }> => step.op === "navigate",
+  );
+  if (navigation) return navigation.url;
+  return drafts.find((step) => step.pageUrl)?.pageUrl ?? fallbackUrl ?? "about:blank";
+}

apps/extension/src/lib/recording/observation-capture.ts

@@ -0,0 +1,115 @@
+import type { RenderedRef } from "@browser-skill/vom";
+import { captureVomObservation } from "@/tools/capture-vom-observation";
+import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
+import type { CapturedNode } from "@/tools/vom/capture";
+
+export interface IndexedObservationNode {
+  frameId: string;
+  node: CapturedNode;
+  ref?: RenderedRef;
+}
+
+export interface CapturedRecordingObservation {
+  rootFrameId: string;
+  index: ObservationNodeIndex;
+  url: string;
+  title?: string;
+  vomText: string;
+  truncated: boolean;
+}
+
+export interface RegisteredObservation {
+  stateId: string;
+  rootFrameId: string;
+  index: ObservationNodeIndex;
+  url: string;
+}
+
+function nodeKey(frameId: string, backendNodeId: number): string {
+  return `${frameId}:${backendNodeId}`;
+}
+
+function frameTagKey(frameId: string, tag: string): string {
+  return `${frameId}:${tag.toLowerCase()}`;
+}
+
+export class ObservationNodeIndex {
+  readonly #nodesByFrameTag = new Map<string, IndexedObservationNode[]>();
+  readonly #refById = new Map<string, RenderedRef>();
+  readonly #refsByFrame = new Map<string, RenderedRef[]>();
+
+  constructor(input: {
+    rootFrameId: string;
+    frameDocuments: Array<{ frameId: string; domNodes: CapturedNode[] }>;
+    refs: RenderedRef[];
+  }) {
+    const refByNode = new Map<string, RenderedRef>();
+    for (const ref of input.refs) {
+      const frameId = ref.frameId ?? input.rootFrameId;
+      refByNode.set(nodeKey(frameId, ref.backendNodeId), ref);
+      this.#refById.set(ref.ref, ref);
+      const frameRefs = this.#refsByFrame.get(frameId) ?? [];
+      frameRefs.push(ref);
+      this.#refsByFrame.set(frameId, frameRefs);
+    }
+    for (const document of input.frameDocuments) {
+      for (const node of document.domNodes) {
+        const frameId = node.frameId ?? document.frameId;
+        const entry = { frameId, node, ref: refByNode.get(nodeKey(frameId, node.backendNodeId)) };
+        const key = frameTagKey(frameId, node.tag);
+        const bucket = this.#nodesByFrameTag.get(key) ?? [];
+        bucket.push(entry);
+        this.#nodesByFrameTag.set(key, bucket);
+      }
+    }
+  }
+
+  candidates(frameId: string, tag: string): readonly IndexedObservationNode[] {
+    return this.#nodesByFrameTag.get(frameTagKey(frameId, tag)) ?? [];
+  }
+
+  ref(refId: string): RenderedRef | undefined {
+    return this.#refById.get(refId);
+  }
+
+  refs(frameId: string): readonly RenderedRef[] {
+    return this.#refsByFrame.get(frameId) ?? [];
+  }
+}
+
+async function readTabMeta(
+  tabsApi: ChromeTabsApi,
+  tabId: number,
+): Promise<{ url: string; title?: string }> {
+  try {
+    const tab = await tabsApi.get(tabId);
+    return { url: tab.url ?? "about:blank", title: tab.title };
+  } catch {
+    return { url: "about:blank" };
+  }
+}
+
+export async function captureRecordingObservation(input: {
+  cdp: CdpRunner;
+  tabsApi: ChromeTabsApi;
+  tabId: number;
+  maxTokens: number;
+  redactValues: boolean;
+  signal?: AbortSignal;
+}): Promise<CapturedRecordingObservation> {
+  const { url, title } = await readTabMeta(input.tabsApi, input.tabId);
+  const captured = await captureVomObservation(input.cdp, input.tabId, url, {
+    maxTokens: input.maxTokens,
+    redactValues: input.redactValues,
+    conditionalSurfaceProbe: false,
+    signal: input.signal,
+  });
+  return {
+    rootFrameId: captured.rootFrameId,
+    index: new ObservationNodeIndex(captured),
+    url,
+    title,
+    vomText: captured.text,
+    truncated: captured.truncated,
+  };
+}

apps/extension/src/lib/recording/observation-session.ts

@@ -0,0 +1,122 @@
+import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
+import { captureRecordingObservation, type RegisteredObservation } from "./observation-capture";
+import { RecordingStateRegistry } from "./state-registry";
+import { matchObservationTarget, unmatchedTarget } from "./target-matcher";
+import type { RecordingDraftStep, StepAnnotation, TargetedRecordingDraft } from "./types";
+
+const DEFAULT_MAX_PAGE_TOKENS = 3_000;
+const MIN_CAPTURE_INTERVAL_MS = 200;
+
+export interface TabObservationCursor {
+  lastSettled: RegisteredObservation | null;
+  lastCaptureAt: number;
+}
+
+function abortableDelay(ms: number, signal?: AbortSignal): Promise<void> {
+  return new Promise((resolve) => {
+    if (signal?.aborted) return resolve();
+    const timer = setTimeout(resolve, ms);
+    signal?.addEventListener(
+      "abort",
+      () => {
+        clearTimeout(timer);
+        resolve();
+      },
+      { once: true },
+    );
+  });
+}
+
+function isTargeted(draft: RecordingDraftStep): draft is TargetedRecordingDraft {
+  return draft.op !== "navigate" && draft.op !== "scroll";
+}
+
+export class RecordingObservationSession {
+  readonly registry: RecordingStateRegistry;
+  readonly cursor: TabObservationCursor;
+  readonly annotations: StepAnnotation[];
+  readonly #maxTokens: number;
+  readonly #redactValues: boolean;
+
+  constructor(
+    options: {
+      registry?: RecordingStateRegistry;
+      cursor?: TabObservationCursor;
+      annotations?: StepAnnotation[];
+      maxTokens?: number;
+      redactValues?: boolean;
+    } = {},
+  ) {
+    this.registry = options.registry ?? new RecordingStateRegistry();
+    this.cursor = options.cursor ?? { lastSettled: null, lastCaptureAt: 0 };
+    this.annotations = options.annotations ?? [];
+    this.#maxTokens = options.maxTokens ?? DEFAULT_MAX_PAGE_TOKENS;
+    this.#redactValues = options.redactValues ?? false;
+  }
+
+  async capture(
+    cdp: CdpRunner,
+    tabsApi: ChromeTabsApi,
+    tabId: number,
+    signal?: AbortSignal,
+  ): Promise<RegisteredObservation> {
+    const waitMs = Math.max(0, MIN_CAPTURE_INTERVAL_MS - (Date.now() - this.cursor.lastCaptureAt));
+    if (waitMs > 0) await abortableDelay(waitMs, signal);
+    if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
+    const captured = await captureRecordingObservation({
+      cdp,
+      tabsApi,
+      tabId,
+      maxTokens: this.#maxTokens,
+      redactValues: this.#redactValues,
+      signal,
+    });
+    const state = this.registry.register({
+      url: captured.url,
+      title: captured.title,
+      rawVomText: captured.vomText,
+      truncated: captured.truncated,
+    });
+    const observation: RegisteredObservation = {
+      stateId: state.id,
+      rootFrameId: captured.rootFrameId,
+      index: captured.index,
+      url: captured.url,
+    };
+    this.cursor.lastSettled = observation;
+    this.cursor.lastCaptureAt = Date.now();
+    return observation;
+  }
+
+  bindDraft(draft: RecordingDraftStep, draftId: number, previousActionPending = false): void {
+    const observation = this.cursor.lastSettled;
+    if (isTargeted(draft)) {
+      draft.matchedTarget = observation
+        ? matchObservationTarget({
+            observation,
+            hint: draft.targetHint,
+            fallback: draft.captureTarget,
+          })
+        : unmatchedTarget(draft.captureTarget);
+    }
+    if (!observation) return;
+
+    const unmatched = isTargeted(draft) && draft.matchedTarget?.unmatched === true;
+    if (previousActionPending && unmatched) return;
+    draft.preStateId = observation.stateId;
+    this.registry.markStep(observation.stateId, draftId);
+
+    if (!isTargeted(draft) || !draft.matchedTarget?.ref) return;
+    const ref = observation.index.ref(draft.matchedTarget.ref);
+    if (!ref) return;
+    this.annotations.push({
+      draftId,
+      op: draft.op,
+      line: ref.line,
+      stateId: observation.stateId,
+      ...(draft.op === "fill" && !this.#redactValues
+        ? { detail: JSON.stringify(draft.value) }
+        : {}),
+    });
+  }
+}

apps/extension/src/lib/recording/settle-controller.ts

@@ -0,0 +1,232 @@
+import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
+import { type DocumentSettleScope, waitForDocumentSettled } from "./document-settle";
+import { RecordingObservationSession } from "./observation-session";
+import type { RecordingDraftStep } from "./types";
+
+interface PendingSettle {
+  abort: AbortController;
+  scope: DocumentSettleScope;
+}
+
+interface PendingRedirect {
+  url: string;
+  abort: AbortController;
+}
+
+const CAPTURE_RETRY_DELAY_MS = 250;
+
+function delay(ms: number, signal?: AbortSignal): Promise<void> {
+  return new Promise((resolve) => {
+    if (signal?.aborted) return resolve();
+    const timer = setTimeout(resolve, ms);
+    signal?.addEventListener(
+      "abort",
+      () => {
+        clearTimeout(timer);
+        resolve();
+      },
+      { once: true },
+    );
+  });
+}
+
+function isAbortError(error: unknown): boolean {
+  return (
+    typeof error === "object" &&
+    error !== null &&
+    (error as { name?: string }).name === "AbortError"
+  );
+}
+
+export function inferMissingPostStates(drafts: RecordingDraftStep[]): void {
+  for (let index = 0; index < drafts.length - 1; index += 1) {
+    const draft = drafts[index];
+    const next = drafts[index + 1];
+    if (draft && next && !draft.postStateId && next.preStateId) {
+      draft.postStateId = next.preStateId;
+    }
+  }
+}
+
+export class SettleController {
+  readonly #session: RecordingObservationSession;
+  readonly #cdp: CdpRunner;
+  readonly #tabsApi: ChromeTabsApi;
+  readonly #tabId: number;
+  readonly #rootScope: DocumentSettleScope;
+  readonly #pending = new Map<number, PendingSettle>();
+  #queue = Promise.resolve();
+  #pendingRedirect: PendingRedirect | null = null;
+  #redirectQueue = Promise.resolve();
+
+  constructor(input: {
+    session: RecordingObservationSession;
+    cdp: CdpRunner;
+    tabsApi: ChromeTabsApi;
+    tabId: number;
+  }) {
+    this.#session = input.session;
+    this.#cdp = input.cdp;
+    this.#tabsApi = input.tabsApi;
+    this.#tabId = input.tabId;
+    this.#rootScope = { target: { tabId: input.tabId } };
+  }
+
+  get hasPending(): boolean {
+    return this.#pending.size > 0 || this.#pendingRedirect !== null;
+  }
+
+  async #captureWithRetry(signal?: AbortSignal) {
+    try {
+      return await this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
+    } catch (error) {
+      if (signal?.aborted) throw error;
+      await delay(CAPTURE_RETRY_DELAY_MS, signal);
+      return this.#session.capture(this.#cdp, this.#tabsApi, this.#tabId, signal);
+    }
+  }
+
+  schedule(
+    drafts: RecordingDraftStep[],
+    draftIndex: number,
+    scope: DocumentSettleScope = this.#rootScope,
+  ): void {
+    if (scope.target.tabId !== this.#tabId) {
+      throw new Error(
+        `settle scope tab ${scope.target.tabId} does not belong to tab ${this.#tabId}`,
+      );
+    }
+    const landing = drafts[draftIndex]?.preStateId;
+    for (const [index, pending] of this.#pending) {
+      if (index >= draftIndex) continue;
+      pending.abort.abort();
+      this.#pending.delete(index);
+      if (landing && drafts[index] && !drafts[index].postStateId) {
+        drafts[index].postStateId = landing;
+      }
+    }
+
+    this.#pending.get(draftIndex)?.abort.abort();
+    const pending = { abort: new AbortController(), scope };
+    this.#pending.set(draftIndex, pending);
+    const task = async () => {
+      if (pending.abort.signal.aborted) return;
+      try {
+        const outcome = await waitForDocumentSettled(this.#cdp, pending.scope, {
+          signal: pending.abort.signal,
+        });
+        if (outcome === "cancelled") return;
+        const observation = await this.#captureWithRetry(pending.abort.signal);
+        if (!pending.abort.signal.aborted && drafts[draftIndex]) {
+          drafts[draftIndex].postStateId = observation.stateId;
+        }
+      } catch (error) {
+        if (!isAbortError(error)) {
+          console.warn(
+            `[bsk record] post-action observation failed for step ${draftIndex + 1}`,
+            error,
+          );
+        }
+      } finally {
+        if (this.#pending.get(draftIndex) === pending) this.#pending.delete(draftIndex);
+      }
+    };
+    this.#queue = this.#queue.then(task, task).catch(() => {});
+  }
+
+  cancel(): void {
+    for (const pending of this.#pending.values()) pending.abort.abort();
+    this.#pending.clear();
+    this.clearRedirect();
+  }
+
+  async flush(): Promise<void> {
+    for (let round = 0; round < 10; round += 1) {
+      const current = this.#queue;
+      await current;
+      if (current === this.#queue) return;
+    }
+    console.warn("[bsk record] settle queue kept growing; using completed observations");
+  }
+
+  async settleTrailing(drafts: RecordingDraftStep[]): Promise<void> {
+    const trailing: RecordingDraftStep[] = [];
+    for (let index = drafts.length - 1; index >= 0; index -= 1) {
+      const draft = drafts[index];
+      if (!draft || draft.postStateId) break;
+      trailing.push(draft);
+    }
+    if (trailing.length === 0) return;
+    try {
+      const observation = await this.#captureWithRetry();
+      for (const draft of trailing) draft.postStateId = observation.stateId;
+    } catch (error) {
+      console.warn("[bsk record] final observation at stop failed", error);
+    }
+  }
+
+  clearRedirect(): void {
+    this.#pendingRedirect?.abort.abort();
+    this.#pendingRedirect = null;
+  }
+
+  scheduleRedirect(drafts: RecordingDraftStep[], url: string): void {
+    this.#pendingRedirect?.abort.abort();
+    const pending: PendingRedirect = { url, abort: new AbortController() };
+    this.#pendingRedirect = pending;
+    const task = () => this.#settleRedirect(drafts, pending);
+    this.#redirectQueue = this.#redirectQueue.then(task, task).catch(() => {});
+  }
+
+  async #settleRedirect(drafts: RecordingDraftStep[], pending: PendingRedirect): Promise<void> {
+    const outcome = await waitForDocumentSettled(this.#cdp, this.#rootScope, {
+      signal: pending.abort.signal,
+    });
+    if (outcome === "cancelled" || this.#pendingRedirect !== pending) return;
+
+    let finalUrl = pending.url;
+    try {
+      finalUrl = (await this.#tabsApi.get(this.#tabId)).url || finalUrl;
+    } catch {
+      // The navigation event URL remains the best available destination.
+    }
+    if (this.#pendingRedirect !== pending) return;
+    this.#pendingRedirect = null;
+    if (
+      !finalUrl ||
+      finalUrl === "about:blank" ||
+      this.#session.cursor.lastSettled?.url === finalUrl
+    ) {
+      return;
+    }
+
+    const last = drafts[drafts.length - 1];
+    if (last?.op === "navigate" && last.url === finalUrl) {
+      if (!last.postStateId) this.schedule(drafts, drafts.length - 1);
+      await this.flush();
+      return;
+    }
+
+    const draft: RecordingDraftStep = {
+      op: "navigate",
+      url: finalUrl,
+      pageUrl: finalUrl,
+      cause: "browser",
+      preStateId: this.#session.cursor.lastSettled?.stateId,
+    };
+    drafts.push(draft);
+    const draftIndex = drafts.length - 1;
+    if (draft.preStateId) this.#session.registry.markStep(draft.preStateId, draftIndex + 1);
+    this.schedule(drafts, draftIndex);
+    await this.flush();
+  }
+
+  async flushRedirects(): Promise<void> {
+    for (let round = 0; round < 10; round += 1) {
+      const current = this.#redirectQueue;
+      await current;
+      if (current === this.#redirectQueue) return;
+    }
+    console.warn("[bsk record] redirect queue kept growing; using the final completed landing");
+  }
+}

apps/extension/src/lib/recording/state-registry.ts

@@ -0,0 +1,56 @@
+export interface RecordedStateEntry {
+  id: string;
+  url: string;
+  title?: string;
+  rawVomText: string;
+  truncated: boolean;
+  stepsHere: number[];
+}
+
+function stateIdentity(url: string, body: string): string {
+  return `${url}\0${body}`;
+}
+
+export class RecordingStateRegistry {
+  readonly #entriesById = new Map<string, RecordedStateEntry>();
+  readonly #idByIdentity = new Map<string, string>();
+  #nextId = 1;
+
+  register(input: {
+    url: string;
+    title?: string;
+    rawVomText: string;
+    truncated?: boolean;
+  }): RecordedStateEntry {
+    const identity = stateIdentity(input.url, input.rawVomText);
+    const existingId = this.#idByIdentity.get(identity);
+    if (existingId) {
+      const existing = this.#entriesById.get(existingId)!;
+      if (!existing.title && input.title) existing.title = input.title;
+      if (input.truncated) existing.truncated = true;
+      return existing;
+    }
+
+    const entry: RecordedStateEntry = {
+      id: `s${this.#nextId}`,
+      url: input.url,
+      ...(input.title ? { title: input.title } : {}),
+      rawVomText: input.rawVomText,
+      truncated: input.truncated ?? false,
+      stepsHere: [],
+    };
+    this.#nextId += 1;
+    this.#entriesById.set(entry.id, entry);
+    this.#idByIdentity.set(identity, entry.id);
+    return entry;
+  }
+
+  values(): RecordedStateEntry[] {
+    return [...this.#entriesById.values()];
+  }
+
+  markStep(stateId: string, draftId: number): void {
+    const entry = this.#entriesById.get(stateId);
+    if (entry && !entry.stepsHere.includes(draftId)) entry.stepsHere.push(draftId);
+  }
+}

apps/extension/src/lib/recording/step-buffer.ts

@@ -0,0 +1,136 @@
+import type { RecordStepPayload } from "../record-bridge";
+import type { RecordingDraftStep } from "./types";
+
+export interface RecordingStepBuffer {
+  steps: RecordingDraftStep[];
+  currentUrl?: string;
+  pendingNavigation: boolean;
+  pendingNavigationDeadline?: number;
+}
+
+const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
+
+function toDraftStep(payload: RecordStepPayload): RecordingDraftStep | null {
+  const pageUrl = payload.page_url;
+  switch (payload.op) {
+    case "click":
+      return payload.target
+        ? {
+            op: "click",
+            captureTarget: payload.target,
+            ...(pageUrl ? { pageUrl } : {}),
+          }
+        : null;
+    case "hover":
+      return payload.target
+        ? {
+            op: "hover",
+            captureTarget: payload.target,
+            ...(pageUrl ? { pageUrl } : {}),
+          }
+        : null;
+    case "fill":
+      return payload.target
+        ? {
+            op: "fill",
+            captureTarget: payload.target,
+            value: payload.value ?? "",
+            ...(payload.redacted ? { redacted: true } : {}),
+            ...(pageUrl ? { pageUrl } : {}),
+          }
+        : null;
+    case "press":
+      return payload.key
+        ? {
+            op: "press",
+            key: payload.key,
+            ...(payload.target ? { captureTarget: payload.target } : {}),
+            ...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
+            ...(pageUrl ? { pageUrl } : {}),
+          }
+        : null;
+    case "select":
+      return payload.target && payload.values
+        ? {
+            op: "select",
+            captureTarget: payload.target,
+            values: payload.values,
+            ...(payload.labels?.length ? { labels: payload.labels } : {}),
+            ...(pageUrl ? { pageUrl } : {}),
+          }
+        : null;
+    case "navigate":
+      return null;
+  }
+}
+
+function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): boolean {
+  for (let i = buffer.steps.length - 1; i >= 0; i -= 1) {
+    const step = buffer.steps[i];
+    if (!step) continue;
+    if (step.op === "click" || step.op === "press" || step.op === "select") {
+      buffer.steps[i] = { ...step, navigatedTo: url };
+      return true;
+    }
+    break;
+  }
+  return false;
+}
+
+export function observeRecordedNavigation(
+  buffer: RecordingStepBuffer,
+  url: string,
+  causedByAction?: boolean,
+): void {
+  if (!url || url === buffer.currentUrl) return;
+  buffer.currentUrl = url;
+  const pendingIsCurrent =
+    buffer.pendingNavigation &&
+    (buffer.pendingNavigationDeadline === undefined ||
+      buffer.pendingNavigationDeadline >= Date.now());
+
+  if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
+    buffer.pendingNavigation = false;
+    buffer.pendingNavigationDeadline = undefined;
+    if (!annotateLastStepNavigation(buffer, url)) {
+      buffer.steps.push({
+        op: "navigate",
+        url,
+        pageUrl: url,
+      });
+    }
+    return;
+  }
+
+  buffer.pendingNavigation = false;
+  buffer.pendingNavigationDeadline = undefined;
+  buffer.steps.push({
+    op: "navigate",
+    url,
+    pageUrl: url,
+  });
+}
+
+export function appendRecordedPayload(
+  buffer: RecordingStepBuffer,
+  payload: RecordStepPayload,
+): void {
+  if (payload.op === "navigate") {
+    if (payload.url) {
+      observeRecordedNavigation(buffer, payload.url, payload.navigation_caused_by_action);
+    }
+    return;
+  }
+  const step = toDraftStep({
+    ...payload,
+    page_url: payload.page_url ?? buffer.currentUrl,
+  });
+  if (!step) return;
+  buffer.steps.push(step);
+  if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
+    buffer.pendingNavigation = payload.expects_navigation === true;
+    buffer.pendingNavigationDeadline = buffer.pendingNavigation
+      ? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
+      : undefined;
+  }
+}

apps/extension/src/lib/recording/target-matcher.ts

@@ -0,0 +1,76 @@
+import type { RenderedRef } from "@browser-skill/vom";
+import type { TargetDescriptorV3 } from "@/transport/types";
+import type { CaptureTargetDescriptor } from "../describe-target";
+import type { IndexedObservationNode, RegisteredObservation } from "./observation-capture";
+import type { TargetGeometry, TargetMatchHint } from "./types";
+
+const MATCH_TOLERANCE_PX = 2;
+
+function close(a: number, b: number): boolean {
+  return Math.abs(a - b) <= MATCH_TOLERANCE_PX;
+}
+
+function rectMatches(a: TargetGeometry["rect"], b: TargetGeometry["rect"]): boolean {
+  return close(a.x, b.x) && close(a.y, b.y) && close(a.w, b.w) && close(a.h, b.h);
+}
+
+function candidateMatches(candidate: IndexedObservationNode, geometry: TargetGeometry): boolean {
+  return candidate.node.rect !== null && rectMatches(geometry.rect, candidate.node.rect);
+}
+
+export function unmatchedTarget(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 {
+  return {
+    ...(fallback?.role ? { role: fallback.role } : {}),
+    ...(fallback?.name ? { name: fallback.name } : {}),
+    unmatched: true,
+  };
+}
+
+function normalized(value: string | undefined): string {
+  return value?.replace(/\s+/g, " ").trim().toLowerCase() ?? "";
+}
+
+function matchesSemantics(ref: RenderedRef, fallback?: CaptureTargetDescriptor): boolean {
+  const role = normalized(fallback?.role);
+  const name = normalized(fallback?.name);
+  if (!role && !name) return false;
+  if (role && normalized(ref.role) !== role) return false;
+  if (name && normalized(ref.name) !== name) return false;
+  return true;
+}
+
+function descriptor(ref: RenderedRef): TargetDescriptorV3 {
+  return {
+    ref: ref.ref,
+    ...(ref.role ? { role: ref.role } : {}),
+    ...(ref.name ? { name: ref.name } : {}),
+    ...(ref.ctx ? { ctx: ref.ctx } : {}),
+  };
+}
+
+export function matchObservationTarget(input: {
+  observation: RegisteredObservation;
+  hint?: TargetMatchHint;
+  fallback?: CaptureTargetDescriptor;
+}): TargetDescriptorV3 {
+  const frameId = input.hint?.frameId ?? input.observation.rootFrameId;
+  const geometry = input.hint?.geometry;
+  if (geometry) {
+    const matches = input.observation.index
+      .candidates(frameId, geometry.tag)
+      .filter((candidate) => candidate.ref && candidateMatches(candidate, geometry));
+    if (matches.length === 1) return descriptor(matches[0]!.ref!);
+    const semanticMatches = matches.filter(
+      (candidate) => candidate.ref && matchesSemantics(candidate.ref, input.fallback),
+    );
+    if (semanticMatches.length === 1) return descriptor(semanticMatches[0]!.ref!);
+    return unmatchedTarget(input.fallback);
+  }
+
+  const semanticMatches = input.observation.index
+    .refs(frameId)
+    .filter((ref) => matchesSemantics(ref, input.fallback));
+  return semanticMatches.length === 1
+    ? descriptor(semanticMatches[0]!)
+    : unmatchedTarget(input.fallback);
+}

apps/extension/src/lib/recording/trace-builder-v3.ts

@@ -0,0 +1,87 @@
+import {
+  type StepV3,
+  type StopReason,
+  TRACE_VERSION_V3,
+  type TraceStateV3,
+  type TraceV3,
+  VOM_FORMAT_VERSION,
+} from "@/transport/types";
+import { resolveDraftStartUrl } from "./draft-policy";
+import type { RecordedStateEntry, RecordingStateRegistry } from "./state-registry";
+import { reduceTraceStepsV3 } from "./trace-reducer-v3";
+import { formatTraceStateBody } from "./trace-state-body";
+import type { RecordingDraftStep, StepAnnotation } from "./types";
+
+function publishedEntries(registry: RecordingStateRegistry, steps: StepV3[]): RecordedStateEntry[] {
+  const entries = registry.values();
+  if (steps.length === 0) return entries.slice(0, 1);
+  const referenced = new Set(steps.flatMap((step) => [step.state, step.result.state]));
+  return entries.filter((entry) => referenced.has(entry.id));
+}
+
+function remapDraftIds(draftIds: number[], stepIdByDraftId: Map<number, number>): number[] {
+  return [
+    ...new Set(
+      draftIds.flatMap((id) => {
+        const stepId = stepIdByDraftId.get(id);
+        return stepId === undefined ? [] : [stepId];
+      }),
+    ),
+  ].sort((a, b) => a - b);
+}
+
+export function buildTraceV3(input: {
+  registry: RecordingStateRegistry;
+  drafts: RecordingDraftStep[];
+  annotations?: StepAnnotation[];
+  startedAt: string;
+  purpose?: string;
+  startUrl?: string;
+  stoppedBy: StopReason;
+  bskVersion: string;
+}): TraceV3 {
+  const reduced = reduceTraceStepsV3(input.drafts);
+  const entries = publishedEntries(input.registry, reduced.steps);
+  const publishedId = new Map(entries.map((entry, index) => [entry.id, `s${index + 1}`]));
+  const annotationsByState = new Map<string, StepAnnotation[]>();
+  for (const annotation of input.annotations ?? []) {
+    const bucket = annotationsByState.get(annotation.stateId) ?? [];
+    bucket.push(annotation);
+    annotationsByState.set(annotation.stateId, bucket);
+  }
+  const steps = reduced.steps.map((step) => ({
+    ...step,
+    state: publishedId.get(step.state) ?? step.state,
+    result: { state: publishedId.get(step.result.state) ?? step.result.state },
+  }));
+  const states: TraceStateV3[] = entries.map((entry) => {
+    const id = publishedId.get(entry.id) ?? entry.id;
+    return {
+      id,
+      url: entry.url,
+      ...(entry.title ? { title: entry.title } : {}),
+      body: formatTraceStateBody({
+        stateId: id,
+        url: entry.url,
+        title: entry.title,
+        stepIds: remapDraftIds(entry.stepsHere, reduced.stepIdByDraftId),
+        vomText: entry.rawVomText,
+        annotations: annotationsByState.get(entry.id) ?? [],
+        stepIdByDraftId: reduced.stepIdByDraftId,
+      }),
+      ...(entry.truncated ? { truncated: true } : {}),
+    };
+  });
+
+  return {
+    version: TRACE_VERSION_V3,
+    ...(input.purpose ? { purpose: input.purpose } : {}),
+    recorded_at: new Date().toISOString(),
+    started_at: input.startedAt,
+    stopped_by: input.stoppedBy,
+    entry: { start_url: resolveDraftStartUrl(input.drafts, input.startUrl, states[0]?.url) },
+    recorder: { bsk: input.bskVersion, vom: VOM_FORMAT_VERSION },
+    states,
+    steps,
+  };
+}

apps/extension/src/lib/recording/trace-reducer-v2.ts

@@ -0,0 +1,219 @@
+import type { PageRefV2, SelectedOptionV2, StepV2, TraceV2 } from "@/transport/types";
+import { resolveDraftStartUrl, shouldIncludeDraft } from "./draft-policy";
+import type { RecordingDraftStep } from "./types";
+
+/** Collapse consecutive navigations to the last hop. */
+function collapseNavigations(steps: RecordingDraftStep[]): RecordingDraftStep[] {
+  const out: RecordingDraftStep[] = [];
+  for (const step of steps) {
+    const prev = out[out.length - 1];
+    if (step.op === "navigate" && prev?.op === "navigate") {
+      out[out.length - 1] = step;
+      continue;
+    }
+    out.push(step);
+  }
+  return out;
+}
+
+function collectUrls(steps: RecordingDraftStep[], startUrl?: string): string[] {
+  const urls: string[] = [];
+  if (startUrl) urls.push(startUrl);
+  for (const step of steps) {
+    if (step.op === "navigate") {
+      urls.push(step.url);
+      continue;
+    }
+    if (step.pageUrl) urls.push(step.pageUrl);
+    if ("navigatedTo" in step && step.navigatedTo) urls.push(step.navigatedTo);
+  }
+  const seen = new Set<string>();
+  const unique: string[] = [];
+  for (const url of urls) {
+    if (!url || seen.has(url)) continue;
+    seen.add(url);
+    unique.push(url);
+  }
+  return unique;
+}
+
+function buildPageRegistry(
+  steps: RecordingDraftStep[],
+  startUrl?: string,
+): { pages: PageRefV2[]; urlToId: Map<string, string> } {
+  const urls = collectUrls(steps, startUrl);
+  const urlToId = new Map<string, string>();
+  const pages = urls.map((url, index) => {
+    const id = `p${index + 1}`;
+    urlToId.set(url, id);
+    return { id, url };
+  });
+  return { pages, urlToId };
+}
+
+function pageIdFor(
+  url: string | undefined,
+  urlToId: Map<string, string>,
+  fallbackUrl?: string,
+): string {
+  if (url && urlToId.has(url)) return urlToId.get(url)!;
+  if (fallbackUrl && urlToId.has(fallbackUrl)) return urlToId.get(fallbackUrl)!;
+  return urlToId.values().next().value ?? "p1";
+}
+
+function pageUrlForDraft(step: RecordingDraftStep, fallbackUrl?: string): string | undefined {
+  if (step.op === "navigate") return step.pageUrl ?? step.url;
+  if (step.pageUrl) return step.pageUrl;
+  return fallbackUrl;
+}
+
+function effectForNavigation(
+  navigatedTo: string | undefined,
+  urlToId: Map<string, string>,
+): StepV2["effect"] {
+  if (!navigatedTo) return undefined;
+  const pageId = urlToId.get(navigatedTo);
+  if (!pageId) return undefined;
+  return { navigated_to: pageId };
+}
+
+function withEffect(step: StepV2, effect: StepV2["effect"]): StepV2 {
+  if (!effect) return step;
+  return { ...step, effect };
+}
+
+function toSelection(values: string[], labels?: string[]): SelectedOptionV2[] {
+  return values.map((value, index) => ({
+    value,
+    ...(labels?.[index] ? { label: labels[index] } : {}),
+  }));
+}
+
+function toV2Step(
+  step: RecordingDraftStep,
+  id: number,
+  urlToId: Map<string, string>,
+  fallbackUrl?: string,
+): StepV2 | null {
+  if (!shouldIncludeDraft(step)) return null;
+
+  const pageUrl = pageUrlForDraft(step, fallbackUrl);
+  const page = pageIdFor(pageUrl, urlToId, fallbackUrl);
+
+  switch (step.op) {
+    case "navigate":
+      return {
+        op: "navigate",
+        id,
+        page: pageIdFor(step.url, urlToId, fallbackUrl),
+        to: step.url,
+      };
+    case "click":
+      if (!step.captureTarget) return null;
+      return withEffect(
+        {
+          op: "click",
+          id,
+          page,
+          target: step.captureTarget,
+        },
+        effectForNavigation(step.navigatedTo, urlToId),
+      );
+    case "hover":
+      if (!step.captureTarget) return null;
+      return {
+        op: "hover",
+        id,
+        page,
+        target: step.captureTarget,
+      };
+    case "fill":
+      if (!step.captureTarget) return null;
+      return {
+        op: "fill",
+        id,
+        page,
+        target: step.captureTarget,
+        value: step.value,
+        ...(step.redacted ? { redacted: true } : {}),
+      };
+    case "press":
+      return withEffect(
+        {
+          op: "press",
+          id,
+          page,
+          key: step.key,
+          ...(step.captureTarget ? { target: step.captureTarget } : {}),
+          ...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
+        },
+        effectForNavigation(step.navigatedTo, urlToId),
+      );
+    case "select":
+      if (!step.captureTarget) return null;
+      return withEffect(
+        {
+          op: "select",
+          id,
+          page,
+          target: step.captureTarget,
+          selection: toSelection(step.values, step.labels),
+        },
+        effectForNavigation(step.navigatedTo, urlToId),
+      );
+    case "scroll":
+      return null;
+  }
+}
+
+interface ReducedTrace {
+  pages: PageRefV2[];
+  steps: StepV2[];
+}
+
+/**
+ * Compile capture drafts into record-only trace v2 steps.
+ * Variable inputs are NOT classified here — executing agents infer that at run time.
+ */
+function reduceTraceSteps(steps: RecordingDraftStep[], startUrl?: string): ReducedTrace {
+  const collapsed = collapseNavigations(steps);
+  const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
+  const out: StepV2[] = [];
+  let id = 1;
+  let lastUrl = startUrl;
+  for (const draft of collapsed) {
+    if (draft.op === "navigate") lastUrl = draft.url;
+    else if ("navigatedTo" in draft && draft.navigatedTo) lastUrl = draft.navigatedTo;
+    else if (draft.pageUrl) lastUrl = draft.pageUrl;
+    const step = toV2Step(draft, id, urlToId, lastUrl);
+    if (!step) continue;
+    out.push(step);
+    id += 1;
+  }
+  return { pages, steps: out };
+}
+
+function resolveTraceStartUrl(
+  drafts: RecordingDraftStep[],
+  startUrl?: string,
+  pages?: PageRefV2[],
+): string {
+  return resolveDraftStartUrl(drafts, startUrl, pages?.[0]?.url);
+}
+
+export function buildTraceV2(input: {
+  steps: RecordingDraftStep[];
+  startedAt: string;
+  startUrl?: string;
+  purpose?: string;
+}): TraceV2 {
+  const { pages, steps } = reduceTraceSteps(input.steps, input.startUrl);
+  return {
+    recorded_at: new Date().toISOString(),
+    started_at: input.startedAt,
+    ...(input.purpose ? { purpose: input.purpose } : {}),
+    entry: { start_url: resolveTraceStartUrl(input.steps, input.startUrl, pages) },
+    pages,
+    steps,
+  };
+}

apps/extension/src/lib/recording/trace-reducer-v3.ts

@@ -0,0 +1,129 @@
+import type { NavigationCause, StepV3 } from "@/transport/types";
+import { shouldIncludeDraft } from "./draft-policy";
+import { unmatchedTarget } from "./target-matcher";
+import type { RecordingDraftStep } from "./types";
+
+interface CollapsedDraft {
+  draft: RecordingDraftStep;
+  draftIds: number[];
+}
+
+const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
+const TRANSITION_CAUSES: Record<string, NavigationCause> = {
+  typed: "user_typed",
+  generated: "user_typed",
+  keyword: "user_typed",
+  keyword_generated: "user_typed",
+  link: "link",
+  form_submit: "form_submit",
+  reload: "reload",
+  auto_bookmark: "browser",
+  start_page: "browser",
+};
+
+function isRedirect(step: Extract<RecordingDraftStep, { op: "navigate" }>): boolean {
+  return (step.transitionQualifiers ?? []).some((qualifier) => REDIRECT_QUALIFIERS.has(qualifier));
+}
+
+function collapseRedirects(steps: RecordingDraftStep[]): CollapsedDraft[] {
+  const output: CollapsedDraft[] = [];
+  steps.forEach((step, index) => {
+    const previous = output[output.length - 1];
+    if (step.op === "navigate" && previous?.draft.op === "navigate" && isRedirect(step)) {
+      previous.draft = {
+        ...previous.draft,
+        url: step.url,
+        postStateId: step.postStateId ?? previous.draft.postStateId,
+      };
+      previous.draftIds.push(index + 1);
+      return;
+    }
+    output.push({ draft: { ...step }, draftIds: [index + 1] });
+  });
+  return output;
+}
+
+function navigationCause(step: Extract<RecordingDraftStep, { op: "navigate" }>): NavigationCause {
+  if (step.cause) return step.cause;
+  const qualifiers = step.transitionQualifiers ?? [];
+  if (qualifiers.includes("forward_back")) return "history";
+  if (qualifiers.includes("from_address_bar")) return "user_typed";
+  return TRANSITION_CAUSES[step.transitionType ?? ""] ?? "browser";
+}
+
+function selection(values: string[], labels?: string[]): Array<{ value: string; label?: string }> {
+  return values.map((value, index) => ({
+    value,
+    ...(labels?.[index] ? { label: labels[index] } : {}),
+  }));
+}
+
+function reduceDraft(draft: RecordingDraftStep, id: number): StepV3 | null {
+  if (!shouldIncludeDraft(draft)) return null;
+  const state = draft.preStateId ?? draft.postStateId;
+  const resultState = draft.postStateId ?? draft.preStateId;
+  if (!state || !resultState) return null;
+  const common = { id, state, result: { state: resultState } };
+
+  switch (draft.op) {
+    case "navigate":
+      return { op: "navigate", ...common, to: draft.url, cause: navigationCause(draft) };
+    case "click":
+      return {
+        op: "click",
+        ...common,
+        target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
+      };
+    case "hover":
+      return {
+        op: "hover",
+        ...common,
+        target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
+      };
+    case "fill":
+      return {
+        op: "fill",
+        ...common,
+        target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
+        value: draft.value,
+        commit: draft.commit ?? "blur",
+        ...(draft.redacted ? { redacted: true } : {}),
+      };
+    case "press":
+      return {
+        op: "press",
+        ...common,
+        key: draft.key,
+        ...(draft.captureTarget || draft.matchedTarget
+          ? { target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget) }
+          : {}),
+        ...(draft.modifiers?.length ? { modifiers: draft.modifiers } : {}),
+      };
+    case "select":
+      return {
+        op: "select",
+        ...common,
+        target: draft.matchedTarget ?? unmatchedTarget(draft.captureTarget),
+        selection: selection(draft.values, draft.labels),
+      };
+    case "scroll":
+      return { op: "scroll", ...common };
+  }
+}
+
+export interface ReducedTraceV3 {
+  steps: StepV3[];
+  stepIdByDraftId: Map<number, number>;
+}
+
+export function reduceTraceStepsV3(steps: RecordingDraftStep[]): ReducedTraceV3 {
+  const output: StepV3[] = [];
+  const stepIdByDraftId = new Map<number, number>();
+  for (const { draft, draftIds } of collapseRedirects(steps)) {
+    const step = reduceDraft(draft, output.length + 1);
+    if (!step) continue;
+    output.push(step);
+    for (const draftId of draftIds) stepIdByDraftId.set(draftId, step.id);
+  }
+  return { steps: output, stepIdByDraftId };
+}

apps/extension/src/lib/recording/trace-state-body.ts

@@ -0,0 +1,39 @@
+import type { StepAnnotation } from "./types";
+
+function annotationText(annotation: StepAnnotation, stepId: number): string {
+  return ` ⟵ step ${stepId}: ${annotation.op}${annotation.detail ? `: ${annotation.detail}` : ""}`;
+}
+
+export function formatTraceStateBody(input: {
+  stateId: string;
+  url: string;
+  title?: string;
+  stepIds: number[];
+  vomText: string;
+  annotations: StepAnnotation[];
+  stepIdByDraftId: Map<number, number>;
+}): string {
+  const lines = ["# bsk-observation 1", `state: ${JSON.stringify(input.stateId)}`];
+  lines.push(`url: ${JSON.stringify(input.url)}`);
+  if (input.title) lines.push(`title: ${JSON.stringify(input.title)}`);
+  if (input.stepIds.length > 0) lines.push(`steps_here: [${input.stepIds.join(", ")}]`);
+  lines.push("---");
+
+  const byLine = new Map<number, Array<{ annotation: StepAnnotation; stepId: number }>>();
+  for (const annotation of input.annotations) {
+    const stepId = input.stepIdByDraftId.get(annotation.draftId);
+    if (stepId === undefined) continue;
+    const bucket = byLine.get(annotation.line) ?? [];
+    bucket.push({ annotation, stepId });
+    byLine.set(annotation.line, bucket);
+  }
+
+  input.vomText.split("\n").forEach((bodyLine, lineIndex) => {
+    let line = bodyLine;
+    for (const item of byLine.get(lineIndex) ?? []) {
+      line += annotationText(item.annotation, item.stepId);
+    }
+    lines.push(line);
+  });
+  return `${lines.join("\n")}\n`;
+}

apps/extension/src/lib/recording/types.ts

@@ -0,0 +1,83 @@
+import type { CaptureTargetDescriptor } from "@/lib/describe-target";
+import type {
+  FillCommit,
+  KeyModifier,
+  NavigationCause,
+  StepV3,
+  TargetDescriptorV3,
+} from "@/transport/types";
+
+export interface TargetGeometry {
+  /** Top-level viewport-relative CSS pixels, as defined by the geometry module. */
+  rect: { x: number; y: number; w: number; h: number };
+  tag: string;
+}
+
+export interface TargetMatchHint {
+  geometry?: TargetGeometry;
+  /** Missing means the current top frame, never an unrestricted frame search. */
+  frameId?: string;
+}
+
+export interface StepAnnotation {
+  draftId: number;
+  op: StepV3["op"];
+  line: number;
+  stateId: string;
+  detail?: string;
+}
+
+interface DraftStateLink {
+  pageUrl?: string;
+  preStateId?: string;
+  postStateId?: string;
+}
+
+interface DraftTarget {
+  captureTarget?: CaptureTargetDescriptor;
+  targetHint?: TargetMatchHint;
+  matchedTarget?: TargetDescriptorV3;
+}
+
+interface DraftNavigationEffect {
+  navigatedTo?: string;
+}
+
+export type RecordingDraftStep =
+  | ({ op: "click" } & DraftStateLink & DraftTarget & DraftNavigationEffect)
+  | ({ op: "hover" } & DraftStateLink & DraftTarget)
+  | ({
+      op: "fill";
+      value: string;
+      commit?: FillCommit;
+      redacted?: boolean;
+    } & DraftStateLink &
+      DraftTarget &
+      DraftNavigationEffect)
+  | ({
+      op: "press";
+      key: string;
+      modifiers?: KeyModifier[];
+    } & DraftStateLink &
+      DraftTarget &
+      DraftNavigationEffect)
+  | ({
+      op: "select";
+      values: string[];
+      labels?: string[];
+    } & DraftStateLink &
+      DraftTarget &
+      DraftNavigationEffect)
+  | ({ op: "scroll" } & DraftStateLink)
+  | ({
+      op: "navigate";
+      url: string;
+      cause?: NavigationCause;
+      transitionType?: string;
+      transitionQualifiers?: string[];
+    } & DraftStateLink);
+
+export type TargetedRecordingDraft = Extract<
+  RecordingDraftStep,
+  { op: "click" | "hover" | "fill" | "press" | "select" }
+>;