diff --git a/apps/extension/src/content/__tests__/record-capture.test.ts b/apps/extension/src/content/__tests__/record-capture.test.ts index 96df004..30108d8 100644 --- a/apps/extension/src/content/__tests__/record-capture.test.ts +++ b/apps/extension/src/content/__tests__/record-capture.test.ts @@ -1,6 +1,6 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; -import { RECORD_START, RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge"; -import { handleRecordContentMessage, startRecordCapture } from "../record-capture"; +import type { RecordStepPayload } from "@/lib/record-bridge"; +import { startRecordCapture } from "../record-capture"; vi.stubGlobal("chrome", { runtime: { @@ -48,118 +48,6 @@ function click(el: Element): void { el.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); } -describe("handleRecordContentMessage stop/cancel", () => { - it("ignores STOP when no recording is active", () => { - const dispose = vi.fn(); - const onStop = vi.fn(); - const setActiveRequestId = vi.fn(); - const setCapture = vi.fn(); - const sendResponse = vi.fn(); - - const needsAsync = handleRecordContentMessage( - { type: RECORD_STOP, requestId: "rec-stale" }, - { - activeRequestId: null, - capture: { dispose }, - setActiveRequestId, - setCapture, - onStart: vi.fn(), - onStop, - }, - sendResponse, - ); - - expect(needsAsync).toBe(false); - expect(dispose).not.toHaveBeenCalled(); - expect(onStop).not.toHaveBeenCalled(); - expect(setActiveRequestId).not.toHaveBeenCalled(); - expect(setCapture).not.toHaveBeenCalled(); - expect(sendResponse).not.toHaveBeenCalled(); - }); - - it("ignores STOP for a mismatched requestId", () => { - const dispose = vi.fn(); - const onStop = vi.fn(); - - const needsAsync = handleRecordContentMessage( - { type: RECORD_STOP, requestId: "rec-other" }, - { - activeRequestId: "rec-1", - capture: { dispose }, - setActiveRequestId: vi.fn(), - setCapture: vi.fn(), - onStart: vi.fn(), - onStop, - }, - vi.fn(), - ); - - expect(needsAsync).toBe(false); - expect(dispose).not.toHaveBeenCalled(); - expect(onStop).not.toHaveBeenCalled(); - }); - - it("keeps a failed STOP retryable and redelivers its recorded step", async () => { - document.body.innerHTML = ``; - const sendMessage = vi.mocked(chrome.runtime.sendMessage); - sendMessage.mockReset(); - sendMessage.mockRejectedValueOnce(new Error("service worker unavailable")); - sendMessage.mockRejectedValueOnce(new Error("service worker still unavailable")); - sendMessage.mockResolvedValueOnce({ ok: true, sequence: 1 }); - - let activeRequestId: string | null = null; - let capture: ReturnType | null = null; - const onStop = vi.fn(); - const dispatch = ( - message: - | { type: typeof RECORD_START; requestId: string; startedAtMs?: number } - | { type: typeof RECORD_STOP; requestId: string }, - sendResponse?: (response: unknown) => void, - ) => - handleRecordContentMessage( - message, - { - activeRequestId, - capture, - setActiveRequestId: (id) => { - activeRequestId = id; - }, - setCapture: (next) => { - capture = next; - }, - onStart: vi.fn(), - onStop, - }, - sendResponse, - ); - - dispatch({ type: RECORD_START, requestId: "rec-retry" }); - const input = document.querySelector("input")!; - input.dispatchEvent(new FocusEvent("focusin", { bubbles: true })); - input.value = "dirty final value"; - input.dispatchEvent(new Event("input", { bubbles: true })); - - const firstResponse = vi.fn(); - expect(dispatch({ type: RECORD_STOP, requestId: "rec-retry" }, firstResponse)).toBe(true); - await vi.waitFor(() => - expect(firstResponse).toHaveBeenCalledWith({ - ok: false, - error: "failed to deliver one or more recorded steps", - }), - ); - expect(activeRequestId).toBe("rec-retry"); - expect(onStop).not.toHaveBeenCalled(); - - const retryResponse = vi.fn(); - expect(dispatch({ type: RECORD_STOP, requestId: "rec-retry" }, retryResponse)).toBe(true); - await vi.waitFor(() => expect(retryResponse).toHaveBeenCalledWith({ ok: true })); - - expect(sendMessage).toHaveBeenCalledTimes(3); - expect(activeRequestId).toBeNull(); - expect(onStop).toHaveBeenCalledTimes(1); - }); -}); - describe("record-capture semantic", () => { let steps: RecordStepPayload[]; @@ -236,6 +124,19 @@ describe("record-capture semantic", () => { ]); }); + it("does not emit page navigation from a child Document capture", () => { + const originalUrl = location.href; + const capture = startRecordCapture("rec-child", (step) => steps.push(step), { + captureNavigation: false, + }); + + history.pushState({}, "", "#inside-frame"); + expect(steps).toEqual([]); + + capture.dispose(); + history.replaceState({}, "", originalUrl); + }); + it("does not record clicks on anonymous layout divs", () => { document.body.innerHTML = `
page chrome
diff --git a/apps/extension/src/content/__tests__/record-frame-agent.test.ts b/apps/extension/src/content/__tests__/record-frame-agent.test.ts new file mode 100644 index 0000000..d81479f --- /dev/null +++ b/apps/extension/src/content/__tests__/record-frame-agent.test.ts @@ -0,0 +1,102 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { RECORD_DOCUMENT_ATTRIBUTE } from "@/lib/recording/document-marker"; +import type { RecordFramePortMessage } from "@/lib/recording/frame-bridge"; +import { RECORD_FRAME_START } from "@/lib/recording/frame-bridge"; +import { RecordFrameAgent } from "../recording/frame-agent"; + +class PortListeners unknown> { + readonly values = new Set(); + addListener = (listener: T) => this.values.add(listener); + removeListener = (listener: T) => this.values.delete(listener); +} + +function portHarness() { + const onMessage = new PortListeners<(message: unknown) => void>(); + const onDisconnect = new PortListeners<() => void>(); + const outbound: RecordFramePortMessage[] = []; + const port = { + name: "bsk-record-frame", + onMessage, + onDisconnect, + postMessage(message: RecordFramePortMessage) { + outbound.push(message); + if (message.type === "ready") { + queueMicrotask(() => { + for (const listener of onMessage.values) { + listener({ + type: "ready_ack", + requestId: message.requestId, + producerId: message.producerId, + }); + } + }); + } + }, + disconnect: vi.fn(), + } as unknown as chrome.runtime.Port; + return { + port, + outbound, + receive(message: RecordFramePortMessage) { + for (const listener of onMessage.values) listener(message); + }, + }; +} + +describe("RecordFrameAgent", () => { + afterEach(() => { + document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE); + document.body.replaceChildren(); + }); + + it("keeps a failed stop retryable and flushes the final dirty fill", async () => { + const harness = portHarness(); + const sendMessage = vi + .fn<(message: { sequence: number }) => Promise>() + .mockRejectedValueOnce(new Error("offline")) + .mockRejectedValueOnce(new Error("still offline")) + .mockImplementation(async (message) => ({ ok: true, sequence: message.sequence })); + vi.stubGlobal("chrome", { + runtime: { + connect: vi.fn(() => harness.port), + sendMessage, + }, + }); + document.body.innerHTML = ``; + const agent = new RecordFrameAgent(); + + await expect( + agent.start({ type: RECORD_FRAME_START, requestId: "rec-1", startedAtMs: 10 }), + ).resolves.toEqual({ ok: true }); + expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true); + + const input = document.querySelector("input")!; + input.dispatchEvent(new FocusEvent("focusin", { bubbles: true })); + input.value = "final value"; + input.dispatchEvent(new Event("input", { bubbles: true })); + + harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-1" }); + await vi.waitFor(() => + expect(harness.outbound).toContainEqual({ + type: "stopped", + requestId: "rec-1", + commandId: "stop-1", + ok: false, + error: "failed to deliver one or more recorded steps", + }), + ); + expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(true); + + harness.receive({ type: "stop", requestId: "rec-1", commandId: "stop-2" }); + await vi.waitFor(() => + expect(harness.outbound).toContainEqual({ + type: "stopped", + requestId: "rec-1", + commandId: "stop-2", + ok: true, + }), + ); + expect(sendMessage).toHaveBeenCalledTimes(3); + expect(document.documentElement.hasAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe(false); + }); +}); diff --git a/apps/extension/src/content/record-capture.ts b/apps/extension/src/content/record-capture.ts index 29e596a..22f7001 100644 --- a/apps/extension/src/content/record-capture.ts +++ b/apps/extension/src/content/record-capture.ts @@ -12,21 +12,7 @@ import { hasDirectHoverInteractiveSignal, hasStrongHoverExpansionSignal, } from "@/lib/hover-trigger-policy"; -import { - isRecordCancelMessage, - isRecordStartMessage, - isRecordStopMessage, - RECORD_CANCEL, - RECORD_START, - RECORD_STEP, - RECORD_STOP, - type RecordCancelMessage, - type RecordStartAck, - type RecordStartMessage, - type RecordStepPayload, - type RecordStopAck, - type RecordStopMessage, -} from "@/lib/record-bridge"; +import type { RecordStepPayload } from "@/lib/record-bridge"; import { shouldRecordPress } from "@/lib/recording/draft-policy"; import { closestHoverSurfaceCandidate, @@ -36,18 +22,6 @@ import { isHoverSurfaceCandidateElement, isLikelyHoverSurfaceOwner, } from "./record-hover-surface"; -import { RecordStepDelivery } from "./record-step-delivery"; - -const stepDeliveries = new Map(); -const pendingStopFlushes = new Map>(); - -function deliveryFor(requestId: string): RecordStepDelivery { - const existing = stepDeliveries.get(requestId); - if (existing) return existing; - const created = new RecordStepDelivery(requestId); - stepDeliveries.set(requestId, created); - return created; -} export interface RecordCaptureController { dispose(): void; @@ -385,6 +359,7 @@ function scheduleInputCompletionCommit( export function startRecordCapture( _requestId: string, sendStep: (step: RecordStepPayload) => void, + options: { captureNavigation?: boolean } = {}, ): RecordCaptureController { const emitStep = (step: RecordStepPayload) => { sendStep({ page_url: location.href, ...step }); @@ -921,26 +896,33 @@ export function startRecordCapture( document.addEventListener("change", onChange, true); document.addEventListener("keydown", onKeyDown, true); - const urlObserver = new MutationObserver(() => emitNavigateIfChanged()); - urlObserver.observe(document, { subtree: true, childList: true }); + const captureNavigation = options.captureNavigation ?? true; + const urlObserver = captureNavigation + ? new MutationObserver(() => emitNavigateIfChanged()) + : null; + urlObserver?.observe(document, { subtree: true, childList: true }); const onUrlEvent = () => emitNavigateIfChanged(); // Wrap: passing commitFillSession directly would forward the DOM event as // the `commit` argument and stamp it onto the recorded fill step. const onPageHide = () => commitFillSession(); - window.addEventListener("hashchange", onUrlEvent); - window.addEventListener("popstate", onUrlEvent); + if (captureNavigation) { + window.addEventListener("hashchange", onUrlEvent); + window.addEventListener("popstate", onUrlEvent); + } window.addEventListener("pagehide", onPageHide); const originalPushState = history.pushState; const originalReplaceState = history.replaceState; - history.pushState = function (...args: Parameters) { - originalPushState.apply(this, args); - emitNavigateIfChanged(navigationActionPending ? true : undefined); - }; - history.replaceState = function (...args: Parameters) { - originalReplaceState.apply(this, args); - emitNavigateIfChanged(navigationActionPending ? true : undefined); - }; + if (captureNavigation) { + history.pushState = function (...args: Parameters) { + originalPushState.apply(this, args); + emitNavigateIfChanged(navigationActionPending ? true : undefined); + }; + history.replaceState = function (...args: Parameters) { + originalReplaceState.apply(this, args); + emitNavigateIfChanged(navigationActionPending ? true : undefined); + }; + } return { dispose() { @@ -954,99 +936,16 @@ export function startRecordCapture( document.removeEventListener("compositionend", onCompositionEnd, true); document.removeEventListener("change", onChange, true); document.removeEventListener("keydown", onKeyDown, true); - urlObserver.disconnect(); - window.removeEventListener("hashchange", onUrlEvent); - window.removeEventListener("popstate", onUrlEvent); + urlObserver?.disconnect(); + if (captureNavigation) { + window.removeEventListener("hashchange", onUrlEvent); + window.removeEventListener("popstate", onUrlEvent); + } window.removeEventListener("pagehide", onPageHide); - history.pushState = originalPushState; - history.replaceState = originalReplaceState; + if (captureNavigation) { + history.pushState = originalPushState; + history.replaceState = originalReplaceState; + } }, }; } - -export type RecordContentMessage = RecordStartMessage | RecordStopMessage | RecordCancelMessage; - -export function isRecordContentMessage(msg: unknown): msg is RecordContentMessage { - return isRecordStartMessage(msg) || isRecordStopMessage(msg) || isRecordCancelMessage(msg); -} - -export function handleRecordContentMessage( - message: RecordContentMessage, - state: { - activeRequestId: string | null; - capture: RecordCaptureController | null; - setActiveRequestId(id: string | null): void; - setCapture(capture: RecordCaptureController | null): void; - onStart(requestId: string, startedAtMs?: number): void; - onStop(): void; - }, - sendResponse?: (response: RecordStartAck | RecordStopAck) => void, -): boolean { - if (isRecordStartMessage(message)) { - const delivery = deliveryFor(message.requestId); - state.capture?.dispose(); - state.setCapture( - startRecordCapture(message.requestId, (step) => { - delivery.enqueue(step); - }), - ); - state.setActiveRequestId(message.requestId); - state.onStart(message.requestId, message.startedAtMs); - sendResponse?.({ ok: true }); - return sendResponse !== undefined; - } - - if (isRecordStopMessage(message) || isRecordCancelMessage(message)) { - // Require an active recording that matches this requestId — otherwise a - // stray STOP/CANCEL (e.g. after teardown) would still run finishStop and - // clear overlay state even though nothing was capturing. - if (!state.activeRequestId || state.activeRequestId !== message.requestId) { - return false; - } - state.capture?.dispose(); - state.setCapture(null); - const finishStop = () => { - state.onStop(); - state.setActiveRequestId(null); - }; - if (isRecordStopMessage(message) && sendResponse) { - const existingFlush = pendingStopFlushes.get(message.requestId); - if (existingFlush) { - void existingFlush.then(sendResponse); - return true; - } - - const flush = deliveryFor(message.requestId) - .flush() - .then((succeeded): RecordStopAck => { - if (succeeded) { - stepDeliveries.delete(message.requestId); - finishStop(); - } - return succeeded - ? { ok: true } - : { - ok: false, - error: "failed to deliver one or more recorded steps", - }; - }); - pendingStopFlushes.set(message.requestId, flush); - void flush.then(sendResponse).finally(() => { - if (pendingStopFlushes.get(message.requestId) === flush) { - pendingStopFlushes.delete(message.requestId); - } - }); - return true; - } - if (isRecordCancelMessage(message)) { - stepDeliveries.delete(message.requestId); - pendingStopFlushes.delete(message.requestId); - } - finishStop(); - return false; - } - - return false; -} - -export { RECORD_CANCEL, RECORD_START, RECORD_STEP, RECORD_STOP }; diff --git a/apps/extension/src/content/recording/frame-agent.ts b/apps/extension/src/content/recording/frame-agent.ts new file mode 100644 index 0000000..68e7b41 --- /dev/null +++ b/apps/extension/src/content/recording/frame-agent.ts @@ -0,0 +1,209 @@ +import { + markRecordingDocument, + type RecordingDocumentMarker, + waitForDocumentElement, +} from "@/lib/recording/document-marker"; +import { + isRecordFrameStartMessage, + RECORD_FRAME_PORT, + RECORD_FRAME_QUERY, + RECORD_FRAME_START, + type RecordFramePortMessage, + type RecordFrameQueryResponse, + type RecordFrameStartMessage, +} from "@/lib/recording/frame-bridge"; +import { type RecordCaptureController, startRecordCapture } from "../record-capture"; +import { RecordStepDelivery } from "../record-step-delivery"; + +interface ActiveFrameRecording { + requestId: string; + producerId: string; + port: chrome.runtime.Port; + delivery: RecordStepDelivery; + capture: RecordCaptureController; + marker: RecordingDocumentMarker; +} + +const RECORD_FRAME_REGISTER_TIMEOUT_MS = 5_000; + +export class RecordFrameAgent { + #active: ActiveFrameRecording | null = null; + #startPromise: Promise<{ ok: true } | { ok: false; error: string }> | null = null; + #disposed = false; + + async start( + message: RecordFrameStartMessage, + ): Promise<{ ok: true } | { ok: false; error: string }> { + if (this.#active?.requestId === message.requestId) return { ok: true }; + if (this.#disposed) return { ok: false, error: "recording document is disposed" }; + if (this.#startPromise) { + await this.#startPromise; + return this.start(message); + } + if (this.#active) this.cancel(this.#active.requestId); + const starting = this.#start(message); + this.#startPromise = starting; + try { + return await starting; + } finally { + if (this.#startPromise === starting) this.#startPromise = null; + } + } + + cancel(requestId: string): void { + const active = this.#active; + if (!active || active.requestId !== requestId) return; + active.capture.dispose(); + active.marker.restore(); + active.port.disconnect(); + this.#active = null; + } + + dispose(): void { + this.#disposed = true; + if (this.#active) this.cancel(this.#active.requestId); + } + + async #start( + message: RecordFrameStartMessage, + ): Promise<{ ok: true } | { ok: false; error: string }> { + const producerId = crypto.randomUUID(); + const root = await waitForDocumentElement(); + if (this.#disposed) return { ok: false, error: "recording document is disposed" }; + const marker = markRecordingDocument(producerId, root); + const port = chrome.runtime.connect({ name: RECORD_FRAME_PORT }); + try { + await registerPort(port, message.requestId, producerId); + } catch { + marker.restore(); + port.disconnect(); + return { ok: false, error: "failed to register recording document" }; + } + if (this.#disposed) { + marker.restore(); + port.disconnect(); + return { ok: false, error: "recording document is disposed" }; + } + + const delivery = new RecordStepDelivery(message.requestId, undefined, producerId); + const capture = startRecordCapture( + message.requestId, + (step) => { + marker.ensure(); + delivery.enqueue(step); + }, + { captureNavigation: window.top === window }, + ); + const active: ActiveFrameRecording = { + requestId: message.requestId, + producerId, + port, + delivery, + capture, + marker, + }; + this.#active = active; + port.onMessage.addListener((raw: unknown) => { + const command = raw as Partial; + if (command.type === "stop" && command.requestId === active.requestId) { + void this.#stop(active, command.commandId); + } else if (command.type === "cancel" && command.requestId === active.requestId) { + this.cancel(active.requestId); + } + }); + port.onDisconnect.addListener(() => { + if (this.#active !== active) return; + active.capture.dispose(); + active.marker.restore(); + this.#active = null; + }); + return { ok: true }; + } + + async #stop(active: ActiveFrameRecording, commandId: string | undefined): Promise { + if (!commandId || this.#active !== active) return; + active.capture.dispose(); + const succeeded = await active.delivery.flush(); + try { + active.port.postMessage({ + type: "stopped", + requestId: active.requestId, + commandId, + ok: succeeded, + ...(succeeded ? {} : { error: "failed to deliver one or more recorded steps" }), + } satisfies RecordFramePortMessage); + } catch { + return; + } + if (!succeeded || this.#active !== active) return; + active.marker.restore(); + this.#active = null; + active.port.disconnect(); + } +} + +function registerPort( + port: chrome.runtime.Port, + requestId: string, + producerId: string, +): Promise { + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + cleanup(); + reject(new Error("recording document registration timed out")); + }, RECORD_FRAME_REGISTER_TIMEOUT_MS); + const cleanup = () => { + clearTimeout(timer); + port.onMessage.removeListener(onMessage); + port.onDisconnect.removeListener(onDisconnect); + }; + const onMessage = (raw: unknown) => { + const message = raw as Partial; + if ( + message.type !== "ready_ack" || + message.requestId !== requestId || + message.producerId !== producerId + ) { + return; + } + cleanup(); + resolve(); + }; + const onDisconnect = () => { + cleanup(); + reject(new Error("recording document port disconnected")); + }; + port.onMessage.addListener(onMessage); + port.onDisconnect.addListener(onDisconnect); + port.postMessage({ type: "ready", requestId, producerId } satisfies RecordFramePortMessage); + }); +} + +export function attachRecordFrameAgent(): () => void { + const agent = new RecordFrameAgent(); + const onMessage = ( + message: unknown, + _sender: chrome.runtime.MessageSender, + sendResponse: (response: { ok: true } | { ok: false; error: string }) => void, + ) => { + if (!isRecordFrameStartMessage(message)) return false; + void agent.start(message).then(sendResponse); + return true; + }; + chrome.runtime.onMessage.addListener(onMessage); + void chrome.runtime + .sendMessage({ type: RECORD_FRAME_QUERY }) + .then((response: RecordFrameQueryResponse | undefined) => { + if (!response?.active || !response.requestId || response.startedAtMs === undefined) return; + return agent.start({ + type: RECORD_FRAME_START, + requestId: response.requestId, + startedAtMs: response.startedAtMs, + }); + }) + .catch(() => {}); + return () => { + chrome.runtime.onMessage.removeListener(onMessage); + agent.dispose(); + }; +} diff --git a/apps/extension/src/entrypoints/background.ts b/apps/extension/src/entrypoints/background.ts index 4154b4b..8e819d7 100644 --- a/apps/extension/src/entrypoints/background.ts +++ b/apps/extension/src/entrypoints/background.ts @@ -21,6 +21,7 @@ import { type OverlayMode, } from "@/lib/overlay-bridge"; import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge"; +import { recordFrameCoordinator } from "@/lib/recording/frame-coordinator"; import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag"; import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup"; import { attachSessionEventHandler } from "@/session-manager/event-handler"; @@ -173,6 +174,7 @@ export default defineBackground(() => { const recordDeps = { tabsApi: chrome.tabs, cdp, + frameCoordinator: recordFrameCoordinator, sendToTab: (tabId: number, msg: Parameters[1]) => chrome.tabs.sendMessage(tabId, msg), bypassOverlay: async (tabId: number, enabled: boolean) => { @@ -192,6 +194,7 @@ export default defineBackground(() => { attachRecordStepListener(recordDeps); attachRecordFinishListener(recordDeps); attachRecordQueryListener(recordDeps); + recordFrameCoordinator.attach(); if (typeof chrome.notifications?.onClicked?.addListener === "function") { attachBorrowNotificationClickHandler({ onClicked: chrome.notifications.onClicked, diff --git a/apps/extension/src/entrypoints/content.ts b/apps/extension/src/entrypoints/content.ts index 7552a09..827eca6 100644 --- a/apps/extension/src/entrypoints/content.ts +++ b/apps/extension/src/entrypoints/content.ts @@ -11,11 +11,6 @@ import { createHelpRequestData } from "@/content/help-request"; import overlayCss from "@/content/overlay.css?inline"; import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller"; import { RecordOverlay } from "@/content/RecordOverlay"; -import { - handleRecordContentMessage, - isRecordContentMessage, - type RecordCaptureController, -} from "@/content/record-capture"; import { type CaptureSuppressAck, type CaptureSuppressMessage, @@ -45,11 +40,17 @@ import { } from "@/lib/overlay-bridge"; import { sendInterrupt } from "@/lib/overlay-interrupt-client"; import { + isRecordCancelMessage, + isRecordStartMessage, + isRecordStopMessage, RECORD_FINISH, RECORD_QUERY, + type RecordCancelMessage, type RecordQueryResponse, type RecordStartAck, + type RecordStartMessage, type RecordStopAck, + type RecordStopMessage, } from "@/lib/record-bridge"; import type { BorrowCancelMessage, @@ -69,7 +70,6 @@ export default defineContentScript({ if (window.top !== window) return; const overlays = new OverlayController(); - let recordCapture: RecordCaptureController | null = null; let activeRecordRequestId: string | null = null; let reactRoot: ReactDOM.Root | null = null; let overlayHost: HTMLElement | null = null; @@ -228,8 +228,6 @@ export default defineContentScript({ if (previousHelp) { void sendHelpFinish(previousHelp.id, "cancelled"); } - recordCapture?.dispose(); - recordCapture = null; activeRecordRequestId = null; renderAll(); } @@ -259,6 +257,9 @@ export default defineContentScript({ | HelpRequestMessage | HelpCancelMessage | CaptureSuppressMessage + | RecordStartMessage + | RecordStopMessage + | RecordCancelMessage | OverlayAgentOverlayResetMessage | OverlayAgentStateMessage | OverlayAutomationBypassMessage, @@ -269,41 +270,32 @@ export default defineContentScript({ return captureSuppress.handleMessage(message, sendResponse); } - if (isRecordContentMessage(message)) { - const needsAsync = handleRecordContentMessage( - message, - { - activeRequestId: activeRecordRequestId, - capture: recordCapture, - setActiveRequestId: (id) => { - activeRecordRequestId = id; - }, - setCapture: (capture) => { - recordCapture = capture; - }, - onStart: (requestId, startedAtMs) => { - overlays.setAgentRecordRequest({ - id: requestId, - ...(typeof startedAtMs === "number" ? { startedAtMs } : {}), - onFinish: () => { - void chrome.runtime.sendMessage({ - type: RECORD_FINISH, - requestId, - }); - }, - }); - renderAll(); - }, - onStop: () => { - overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined); - renderAll(); - }, + if (isRecordStartMessage(message)) { + activeRecordRequestId = message.requestId; + overlays.setAgentRecordRequest({ + id: message.requestId, + ...(typeof message.startedAtMs === "number" ? { startedAtMs: message.startedAtMs } : {}), + onFinish: () => { + void chrome.runtime.sendMessage({ + type: RECORD_FINISH, + requestId: message.requestId, + }); }, - sendResponse as unknown as - | ((response: RecordStartAck | RecordStopAck) => void) - | undefined, - ); - return needsAsync; + }); + renderAll(); + (sendResponse as unknown as (response: RecordStartAck) => void)({ ok: true }); + return false; + } + + if (isRecordStopMessage(message) || isRecordCancelMessage(message)) { + if (activeRecordRequestId !== message.requestId) return false; + overlays.clearAgentRecordRequest(message.requestId); + activeRecordRequestId = null; + renderAll(); + if (isRecordStopMessage(message)) { + (sendResponse as unknown as (response: RecordStopAck) => void)({ ok: true }); + } + return false; } if ( @@ -526,9 +518,6 @@ export default defineContentScript({ chrome.runtime.onMessage.removeListener(onMessage); chrome.storage.onChanged.removeListener(onStorageChange); window.removeEventListener("pageshow", onPageShow); - // Restore history hooks / remove capture listeners before the CS unloads. - recordCapture?.dispose(); - recordCapture = null; activeRecordRequestId = null; }); }, diff --git a/apps/extension/src/entrypoints/record-frame.content.ts b/apps/extension/src/entrypoints/record-frame.content.ts new file mode 100644 index 0000000..6b5bdd0 --- /dev/null +++ b/apps/extension/src/entrypoints/record-frame.content.ts @@ -0,0 +1,14 @@ +import { attachRecordFrameAgent } from "@/content/recording/frame-agent"; + +export default defineContentScript({ + matches: [""], + runAt: "document_start", + allFrames: true, + matchAboutBlank: true, + matchOriginAsFallback: true, + + main(ctx) { + const dispose = attachRecordFrameAgent(); + ctx.onInvalidated(dispose); + }, +}); diff --git a/apps/extension/src/lib/__tests__/record-frame-coordinator.test.ts b/apps/extension/src/lib/__tests__/record-frame-coordinator.test.ts new file mode 100644 index 0000000..e5e7449 --- /dev/null +++ b/apps/extension/src/lib/__tests__/record-frame-coordinator.test.ts @@ -0,0 +1,121 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { RECORD_FRAME_PORT } from "../recording/frame-bridge"; +import { RecordFrameCoordinator } from "../recording/frame-coordinator"; + +class ListenerSet unknown> { + readonly listeners = new Set(); + addListener = (listener: T) => this.listeners.add(listener); + removeListener = (listener: T) => this.listeners.delete(listener); +} + +function fakePort(sender: chrome.runtime.MessageSender) { + const messages: unknown[] = []; + const onMessage = new ListenerSet<(message: unknown) => void>(); + const onDisconnect = new ListenerSet<() => void>(); + return { + port: { + name: RECORD_FRAME_PORT, + sender, + onMessage, + onDisconnect, + postMessage: vi.fn((message: unknown) => messages.push(message)), + disconnect: vi.fn(), + } as unknown as chrome.runtime.Port, + messages, + receive(message: unknown) { + for (const listener of onMessage.listeners) listener(message); + }, + }; +} + +describe("RecordFrameCoordinator", () => { + const onConnect = new ListenerSet<(port: chrome.runtime.Port) => void>(); + const onMessage = new ListenerSet< + ( + message: unknown, + sender: chrome.runtime.MessageSender, + sendResponse: (response: unknown) => void, + ) => boolean + >(); + + beforeEach(() => { + onConnect.listeners.clear(); + onMessage.listeners.clear(); + vi.stubGlobal("chrome", { + runtime: { onConnect, onMessage }, + }); + }); + + it("binds a producer to its sender Document and keeps final steps valid while stopping", async () => { + const sendToDocument = vi.fn(async () => ({ ok: true })); + const coordinator = new RecordFrameCoordinator({ + getAllFrames: async () => [ + { frameId: 0, documentId: "top-document" }, + { frameId: 7, documentId: "child-document" }, + ], + sendToDocument, + }); + coordinator.attach(); + coordinator.begin("rec-1", 10); + await expect(coordinator.armTab("rec-1", 3)).resolves.toBe(true); + + const sender = { + tab: { id: 3 }, + frameId: 7, + documentId: "child-document", + } as chrome.runtime.MessageSender; + const frame = fakePort(sender); + for (const listener of onConnect.listeners) listener(frame.port); + frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" }); + + expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toEqual({ + tabId: 3, + documentId: "child-document", + browserFrameId: 7, + producerId: "producer-1", + }); + + const stopping = coordinator.stop("rec-1"); + expect(coordinator.sourceFor("rec-1", "producer-1", sender)).not.toBeNull(); + const stop = frame.messages.find( + (message): message is { type: "stop"; commandId: string } => + typeof message === "object" && + message !== null && + (message as { type?: string }).type === "stop", + ); + expect(stop).toBeDefined(); + frame.receive({ + type: "stopped", + requestId: "rec-1", + commandId: stop!.commandId, + ok: true, + }); + await expect(stopping).resolves.toBe(true); + expect(coordinator.sourceFor("rec-1", "producer-1", sender)).toBeNull(); + }); + + it("rejects a producer used from another Document", async () => { + const coordinator = new RecordFrameCoordinator({ + getAllFrames: async () => [{ frameId: 0, documentId: "top-document" }], + sendToDocument: async () => ({ ok: true }), + }); + coordinator.attach(); + coordinator.begin("rec-1", 10); + await coordinator.armTab("rec-1", 3); + const sender = { + tab: { id: 3 }, + frameId: 0, + documentId: "top-document", + } as chrome.runtime.MessageSender; + const frame = fakePort(sender); + for (const listener of onConnect.listeners) listener(frame.port); + frame.receive({ type: "ready", requestId: "rec-1", producerId: "producer-1" }); + + expect( + coordinator.sourceFor("rec-1", "producer-1", { + ...sender, + documentId: "other-document", + }), + ).toBeNull(); + }); +}); diff --git a/apps/extension/src/lib/__tests__/recording-document-marker.test.ts b/apps/extension/src/lib/__tests__/recording-document-marker.test.ts new file mode 100644 index 0000000..cc91fb9 --- /dev/null +++ b/apps/extension/src/lib/__tests__/recording-document-marker.test.ts @@ -0,0 +1,58 @@ +import { describe, expect, it } from "vitest"; +import { RECORD_DOCUMENT_ATTRIBUTE } from "@/shared/recording-document-identity"; +import { markRecordingDocument } from "../recording/document-marker"; +import { ObservationNodeIndex } from "../recording/observation-capture"; + +describe("recording document marker", () => { + it("restores the Document attribute exactly", () => { + document.documentElement.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, "page-value"); + const marker = markRecordingDocument("producer-1"); + + expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe("producer-1"); + marker.restore(); + expect(document.documentElement.getAttribute(RECORD_DOCUMENT_ATTRIBUTE)).toBe("page-value"); + document.documentElement.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE); + }); + + it("indexes the CDP scope from the safe frame identity", () => { + const index = new ObservationNodeIndex({ + rootFrameId: "root", + frames: [ + { + frameId: "child", + target: { tabId: 3, sessionId: "oopif-session" }, + recordingDocumentId: "producer-1", + }, + ], + matchNodes: [], + refs: [], + }); + + expect(index.documentScope("producer-1")).toEqual({ + frameId: "child", + target: { tabId: 3, sessionId: "oopif-session" }, + }); + }); + + it("fails closed when a recording identity maps to multiple Documents", () => { + const index = new ObservationNodeIndex({ + rootFrameId: "root", + frames: [ + { + frameId: "left", + target: { tabId: 3, sessionId: "left-session" }, + recordingDocumentId: "duplicate", + }, + { + frameId: "right", + target: { tabId: 3, sessionId: "right-session" }, + recordingDocumentId: "duplicate", + }, + ], + matchNodes: [], + refs: [], + }); + + expect(index.documentScope("duplicate")).toBeUndefined(); + }); +}); diff --git a/apps/extension/src/lib/__tests__/recording-runtime.test.ts b/apps/extension/src/lib/__tests__/recording-runtime.test.ts index c541f3a..6dfe524 100644 --- a/apps/extension/src/lib/__tests__/recording-runtime.test.ts +++ b/apps/extension/src/lib/__tests__/recording-runtime.test.ts @@ -10,6 +10,7 @@ vi.mock("../recording/observation-capture", async (importOriginal) => { import { ObservationNodeIndex } from "../recording/observation-capture"; import { RecordingObservationRuntime } from "../recording/recording-runtime"; +import type { RecordingDraftStep } from "../recording/types"; function observation(url = "https://example.com/") { return { @@ -149,4 +150,53 @@ describe("RecordingObservationRuntime", () => { }); expect(captureRecordingObservation.mock.calls.map(([input]) => input.tabId)).toEqual([4, 5]); }); + + it("binds an iframe draft to its marked CDP Document scope", async () => { + captureRecordingObservation.mockResolvedValueOnce({ + ...observation(), + index: new ObservationNodeIndex({ + rootFrameId: "root", + frames: [ + { + frameId: "child", + target: { tabId: 7, sessionId: "oopif-session" }, + recordingDocumentId: "producer-1", + }, + ], + matchNodes: [ + { + backendNodeId: 42, + frameId: "child", + tag: "button", + rect: { x: 410, y: 20, w: 100, h: 30 }, + localRect: { x: 10, y: 20, w: 100, h: 30 }, + }, + ], + refs: [{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }], + }), + }); + const recording = runtime(); + const drafts: RecordingDraftStep[] = [ + { + op: "click" as const, + captureTarget: { tag: "button", role: "button", name: "Save" }, + targetHint: { + geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" }, + }, + }, + ]; + + await recording.captureInitial(7); + await recording.processDraft(7, drafts, 0, "producer-1"); + recording.cancel(); + + const draft = drafts[0]; + expect(draft?.op).toBe("click"); + if (!draft || draft.op !== "click") throw new Error("expected click draft"); + expect(draft.targetHint).toMatchObject({ + frameId: "child", + geometrySpace: "local", + }); + expect(draft.matchedTarget?.ref).toBe("e1"); + }); }); diff --git a/apps/extension/src/lib/__tests__/target-matcher.test.ts b/apps/extension/src/lib/__tests__/target-matcher.test.ts index d3efae2..2225351 100644 --- a/apps/extension/src/lib/__tests__/target-matcher.test.ts +++ b/apps/extension/src/lib/__tests__/target-matcher.test.ts @@ -69,6 +69,23 @@ describe("matchObservationTarget", () => { expect(target.ref).toBe("e2"); }); + it("matches iframe capture geometry in the child viewport coordinate space", () => { + const child = node(42, "child", 410); + child.localRect = { x: 10, y: 20, w: 100, h: 30 }; + const target = matchObservationTarget({ + observation: observation( + [child], + [{ ref: "e1", backendNodeId: 42, frameId: "child", line: 1 }], + ), + hint: { + frameId: "child", + geometrySpace: "local", + geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" }, + }, + }); + expect(target.ref).toBe("e1"); + }); + it("restricts a missing frame hint to the root frame", () => { const target = matchObservationTarget({ observation: observation( @@ -81,6 +98,21 @@ describe("matchObservationTarget", () => { expect(target).toEqual({ name: "发布", unmatched: true }); }); + it("fails closed when the source Document has no VOM frame mapping", () => { + const target = matchObservationTarget({ + observation: observation( + [node(42, "root")], + [{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 1 }], + ), + hint: { + frameId: null, + geometry: { rect: { x: 10, y: 20, w: 100, h: 30 }, tag: "button" }, + }, + fallback: { tag: "button", role: "button", name: "发布" }, + }); + expect(target).toEqual({ role: "button", name: "发布", unmatched: true }); + }); + it("returns unmatched for ambiguous geometry", () => { const target = matchObservationTarget({ observation: observation( diff --git a/apps/extension/src/lib/recording/document-marker.ts b/apps/extension/src/lib/recording/document-marker.ts new file mode 100644 index 0000000..b1b3226 --- /dev/null +++ b/apps/extension/src/lib/recording/document-marker.ts @@ -0,0 +1,37 @@ +import { RECORD_DOCUMENT_ATTRIBUTE } from "@/shared/recording-document-identity"; + +export { RECORD_DOCUMENT_ATTRIBUTE } from "@/shared/recording-document-identity"; + +export interface RecordingDocumentMarker { + restore(): void; + ensure(): void; +} + +export function waitForDocumentElement(): Promise { + if (document.documentElement) return Promise.resolve(document.documentElement); + return new Promise((resolve) => { + const observer = new MutationObserver(() => { + if (!document.documentElement) return; + observer.disconnect(); + resolve(document.documentElement); + }); + observer.observe(document, { childList: true }); + }); +} + +export function markRecordingDocument( + producerId: string, + root: HTMLElement = document.documentElement, +): RecordingDocumentMarker { + const previous = root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE); + const ensure = () => root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, producerId); + ensure(); + return { + ensure, + restore() { + if (root.getAttribute(RECORD_DOCUMENT_ATTRIBUTE) !== producerId) return; + if (previous === null) root.removeAttribute(RECORD_DOCUMENT_ATTRIBUTE); + else root.setAttribute(RECORD_DOCUMENT_ATTRIBUTE, previous); + }, + }; +} diff --git a/apps/extension/src/lib/recording/frame-bridge.ts b/apps/extension/src/lib/recording/frame-bridge.ts new file mode 100644 index 0000000..48af039 --- /dev/null +++ b/apps/extension/src/lib/recording/frame-bridge.ts @@ -0,0 +1,65 @@ +export const RECORD_FRAME_PORT = "bsk-record-frame"; +export const RECORD_FRAME_QUERY = "bsk-record-frame-query"; +export const RECORD_FRAME_START = "bsk-record-frame-start"; + +export interface RecordFrameQueryMessage { + type: typeof RECORD_FRAME_QUERY; +} + +export interface RecordFrameQueryResponse { + active: boolean; + requestId?: string; + startedAtMs?: number; +} + +export interface RecordFrameStartMessage { + type: typeof RECORD_FRAME_START; + requestId: string; + startedAtMs: number; +} + +export type RecordFramePortMessage = + | { + type: "ready"; + requestId: string; + producerId: string; + } + | { + type: "ready_ack"; + requestId: string; + producerId: string; + } + | { + type: "stop"; + requestId: string; + commandId: string; + } + | { + type: "cancel"; + requestId: string; + } + | { + type: "stopped"; + requestId: string; + commandId: string; + ok: boolean; + error?: string; + }; + +export function isRecordFrameQueryMessage(value: unknown): value is RecordFrameQueryMessage { + return ( + typeof value === "object" && + value !== null && + (value as { type?: unknown }).type === RECORD_FRAME_QUERY + ); +} + +export function isRecordFrameStartMessage(value: unknown): value is RecordFrameStartMessage { + if (typeof value !== "object" || value === null) return false; + const message = value as Partial; + return ( + message.type === RECORD_FRAME_START && + typeof message.requestId === "string" && + typeof message.startedAtMs === "number" + ); +} diff --git a/apps/extension/src/lib/recording/frame-coordinator.ts b/apps/extension/src/lib/recording/frame-coordinator.ts new file mode 100644 index 0000000..8bdf188 --- /dev/null +++ b/apps/extension/src/lib/recording/frame-coordinator.ts @@ -0,0 +1,330 @@ +import { + isRecordFrameQueryMessage, + RECORD_FRAME_PORT, + RECORD_FRAME_START, + type RecordFramePortMessage, + type RecordFrameQueryResponse, + type RecordFrameStartMessage, +} from "./frame-bridge"; + +export interface RecordingCaptureScope { + tabId: number; + documentId: string; + browserFrameId: number; + producerId: string; +} + +interface ArmedRecording { + requestId: string; + startedAtMs: number; + tabIds: Set; + agents: Map; + finishing: boolean; +} + +interface FrameAgent extends RecordingCaptureScope { + requestId: string; + port: chrome.runtime.Port; + stopWaiters: Map void>; +} + +interface BrowserFrame { + frameId: number; + documentId?: string; +} + +const RECORD_FRAME_STOP_TIMEOUT_MS = 5_000; + +export interface RecordFrameCoordinatorDeps { + getAllFrames(tabId: number): Promise; + sendToDocument( + tabId: number, + message: RecordFrameStartMessage, + target: { documentId?: string; frameId?: number }, + ): Promise; +} + +function documentKey(tabId: number, documentId: string): string { + return `${tabId}:${documentId}`; +} + +function senderAddress( + sender: chrome.runtime.MessageSender, +): { tabId: number; documentId: string; browserFrameId: number } | null { + const tabId = sender.tab?.id; + const documentId = sender.documentId; + const browserFrameId = sender.frameId; + if ( + typeof tabId !== "number" || + typeof documentId !== "string" || + typeof browserFrameId !== "number" + ) { + return null; + } + return { tabId, documentId, browserFrameId }; +} + +function defaultDeps(): RecordFrameCoordinatorDeps { + return { + async getAllFrames(tabId) { + return (await chrome.webNavigation.getAllFrames({ tabId })) ?? []; + }, + sendToDocument(tabId, message, target) { + return chrome.tabs.sendMessage(tabId, message, target); + }, + }; +} + +export class RecordFrameCoordinator { + readonly #deps: RecordFrameCoordinatorDeps; + readonly #recordings = new Map(); + #attached = false; + + constructor(deps: RecordFrameCoordinatorDeps = defaultDeps()) { + this.#deps = deps; + } + + attach(): () => void { + if (this.#attached) return () => {}; + this.#attached = true; + chrome.runtime.onConnect.addListener(this.#onConnect); + chrome.runtime.onMessage.addListener(this.#onMessage); + return () => { + if (!this.#attached) return; + this.#attached = false; + chrome.runtime.onConnect.removeListener(this.#onConnect); + chrome.runtime.onMessage.removeListener(this.#onMessage); + for (const recording of this.#recordings.values()) this.#cancelAgents(recording); + this.#recordings.clear(); + }; + } + + begin(requestId: string, startedAtMs: number): void { + const previous = this.#recordings.get(requestId); + if (previous) this.#cancelAgents(previous); + this.#recordings.set(requestId, { + requestId, + startedAtMs, + tabIds: new Set(), + agents: new Map(), + finishing: false, + }); + } + + async armTab(requestId: string, tabId: number): Promise { + const recording = this.#recordings.get(requestId); + if (!recording || recording.finishing) return false; + recording.tabIds.add(tabId); + + let frames: BrowserFrame[]; + try { + frames = await this.#deps.getAllFrames(tabId); + } catch { + frames = [{ frameId: 0 }]; + } + if (!frames.some((frame) => frame.frameId === 0)) frames.unshift({ frameId: 0 }); + + const message: RecordFrameStartMessage = { + type: RECORD_FRAME_START, + requestId, + startedAtMs: recording.startedAtMs, + }; + const results = await Promise.all( + frames.map(async (frame) => { + try { + const response = await this.#deps.sendToDocument(tabId, message, { + ...(frame.documentId ? { documentId: frame.documentId } : { frameId: frame.frameId }), + }); + return { frameId: frame.frameId, started: isStarted(response) }; + } catch { + return { frameId: frame.frameId, started: false }; + } + }), + ); + return results.some((result) => result.frameId === 0 && result.started); + } + + sourceFor( + requestId: string, + producerId: string, + sender: chrome.runtime.MessageSender, + ): RecordingCaptureScope | null { + const address = senderAddress(sender); + const recording = this.#recordings.get(requestId); + if (!address || !recording) return null; + const agent = recording.agents.get(documentKey(address.tabId, address.documentId)); + if (!agent || agent.producerId !== producerId) return null; + return { + tabId: agent.tabId, + documentId: agent.documentId, + browserFrameId: agent.browserFrameId, + producerId: agent.producerId, + }; + } + + async stop(requestId: string): Promise { + const recording = this.#recordings.get(requestId); + if (!recording) return true; + recording.finishing = true; + const commandId = crypto.randomUUID(); + const results = await Promise.all( + [...recording.agents.values()].map((agent) => this.#stopAgent(agent, requestId, commandId)), + ); + const succeeded = results.every(Boolean); + if (succeeded) this.#recordings.delete(requestId); + else recording.finishing = false; + return succeeded; + } + + cancel(requestId: string): void { + const recording = this.#recordings.get(requestId); + if (!recording) return; + this.#cancelAgents(recording); + this.#recordings.delete(requestId); + } + + readonly #onMessage = ( + message: unknown, + sender: chrome.runtime.MessageSender, + sendResponse: (response: RecordFrameQueryResponse) => void, + ): boolean => { + if (!isRecordFrameQueryMessage(message)) return false; + const tabId = sender.tab?.id; + const recording = + typeof tabId === "number" + ? [...this.#recordings.values()].find( + (candidate) => !candidate.finishing && candidate.tabIds.has(tabId), + ) + : undefined; + sendResponse( + recording + ? { + active: true, + requestId: recording.requestId, + startedAtMs: recording.startedAtMs, + } + : { active: false }, + ); + return false; + }; + + readonly #onConnect = (port: chrome.runtime.Port): void => { + if (port.name !== RECORD_FRAME_PORT) return; + const address = senderAddress(port.sender ?? {}); + if (!address) { + port.disconnect(); + return; + } + let registered: FrameAgent | null = null; + port.onMessage.addListener((raw: unknown) => { + const message = raw as Partial; + if (message.type === "ready") { + if (typeof message.requestId !== "string" || typeof message.producerId !== "string") { + port.disconnect(); + return; + } + const requestId = message.requestId; + const producerId = message.producerId; + if (registered) { + if (registered.requestId === requestId && registered.producerId === producerId) { + port.postMessage({ + type: "ready_ack", + requestId, + producerId, + } satisfies RecordFramePortMessage); + } else { + port.disconnect(); + } + return; + } + const recording = this.#recordings.get(requestId); + if ( + !recording || + recording.finishing || + !recording.tabIds.has(address.tabId) || + producerId.length === 0 + ) { + port.disconnect(); + return; + } + const agent: FrameAgent = { + ...address, + requestId, + producerId, + port, + stopWaiters: new Map(), + }; + registered = agent; + const key = documentKey(address.tabId, address.documentId); + const previous = recording.agents.get(key); + if (previous && previous !== agent) previous.port.disconnect(); + recording.agents.set(key, agent); + port.postMessage({ + type: "ready_ack", + requestId, + producerId, + } satisfies RecordFramePortMessage); + return; + } + if ( + message.type === "stopped" && + registered && + message.requestId === registered.requestId && + typeof message.commandId === "string" + ) { + registered.stopWaiters.get(message.commandId)?.(message.ok === true); + registered.stopWaiters.delete(message.commandId); + } + }); + port.onDisconnect.addListener(() => { + if (!registered) return; + for (const resolve of registered.stopWaiters.values()) resolve(true); + const recording = this.#recordings.get(registered.requestId); + const key = documentKey(registered.tabId, registered.documentId); + if (recording?.agents.get(key) === registered) recording.agents.delete(key); + }); + }; + + #stopAgent(agent: FrameAgent, requestId: string, commandId: string): Promise { + return new Promise((resolve) => { + const timer = setTimeout(() => { + agent.stopWaiters.delete(commandId); + resolve(false); + }, RECORD_FRAME_STOP_TIMEOUT_MS); + agent.stopWaiters.set(commandId, (ok) => { + clearTimeout(timer); + resolve(ok); + }); + try { + agent.port.postMessage({ + type: "stop", + requestId, + commandId, + } satisfies RecordFramePortMessage); + } catch { + clearTimeout(timer); + agent.stopWaiters.delete(commandId); + resolve(true); + } + }); + } + + #cancelAgents(recording: ArmedRecording): void { + for (const agent of recording.agents.values()) { + try { + agent.port.postMessage({ + type: "cancel", + requestId: recording.requestId, + } satisfies RecordFramePortMessage); + } catch { + // A destroyed Document has no remaining capture state to cancel. + } + } + } +} + +function isStarted(value: unknown): boolean { + return typeof value === "object" && value !== null && (value as { ok?: unknown }).ok === true; +} + +export const recordFrameCoordinator = new RecordFrameCoordinator(); diff --git a/apps/extension/src/lib/recording/observation-capture.ts b/apps/extension/src/lib/recording/observation-capture.ts index 9c1f459..efafad4 100644 --- a/apps/extension/src/lib/recording/observation-capture.ts +++ b/apps/extension/src/lib/recording/observation-capture.ts @@ -28,6 +28,11 @@ export interface RegisteredObservation { url: string; } +export interface RecordingDocumentScope { + frameId: string; + target: CdpTarget; +} + function nodeKey(frameId: string, backendNodeId: number): string { return `${frameId}:${backendNodeId}`; } @@ -40,8 +45,12 @@ export class ObservationNodeIndex { readonly #nodesByFrameTag = new Map(); readonly #refById = new Map(); readonly #refsByFrame = new Map(); + readonly #scopeByProducer = new Map(); - constructor(input: Pick) { + constructor( + input: Pick & + Partial>, + ) { const refByNode = new Map(); for (const ref of input.refs) { const frameId = ref.frameId ?? input.rootFrameId; @@ -51,6 +60,16 @@ export class ObservationNodeIndex { frameRefs.push(ref); this.#refsByFrame.set(frameId, frameRefs); } + for (const frame of input.frames ?? []) { + const documentId = frame.recordingDocumentId; + if (!documentId) continue; + this.#scopeByProducer.set( + documentId, + this.#scopeByProducer.has(documentId) + ? null + : { frameId: frame.frameId, target: frame.target }, + ); + } for (const geometry of input.matchNodes) { const { frameId } = geometry; const entry = { @@ -76,6 +95,10 @@ export class ObservationNodeIndex { refs(frameId: string): readonly RenderedRef[] { return this.#refsByFrame.get(frameId) ?? []; } + + documentScope(producerId: string): RecordingDocumentScope | undefined { + return this.#scopeByProducer.get(producerId) ?? undefined; + } } async function readTabMeta( diff --git a/apps/extension/src/lib/recording/recording-runtime.ts b/apps/extension/src/lib/recording/recording-runtime.ts index e27577f..d9a396e 100644 --- a/apps/extension/src/lib/recording/recording-runtime.ts +++ b/apps/extension/src/lib/recording/recording-runtime.ts @@ -126,7 +126,7 @@ export class RecordingObservationRuntime { tabId: number, drafts: RecordingDraftStep[], draftIndex: number, - scope?: DocumentSettleScope, + producerId?: string, ): Promise { const draft = drafts[draftIndex]; if (!draft) return; @@ -138,6 +138,23 @@ export class RecordingObservationRuntime { // Post-action settle can still provide a usable state. } } + const scope = producerId + ? context.session.cursor.lastSettled?.index.documentScope(producerId) + : undefined; + if ( + producerId && + (draft.op === "click" || + draft.op === "hover" || + draft.op === "fill" || + draft.op === "press" || + draft.op === "select") + ) { + draft.targetHint = { + ...(draft.targetHint ?? {}), + frameId: scope?.frameId ?? null, + geometrySpace: "local", + }; + } context.session.bindDraft(draft, draftIndex + 1, context.settle.hasPending); context.settle.schedule(drafts, draftIndex, scope); } diff --git a/apps/extension/src/lib/recording/target-matcher.ts b/apps/extension/src/lib/recording/target-matcher.ts index 5cf21aa..9fca298 100644 --- a/apps/extension/src/lib/recording/target-matcher.ts +++ b/apps/extension/src/lib/recording/target-matcher.ts @@ -14,8 +14,13 @@ function rectMatches(a: TargetGeometry["rect"], b: TargetGeometry["rect"]): bool return close(a.x, b.x) && close(a.y, b.y) && close(a.w, b.w) && close(a.h, b.h); } -function candidateMatches(candidate: IndexedObservationNode, geometry: TargetGeometry): boolean { - return candidate.geometry.rect !== null && rectMatches(geometry.rect, candidate.geometry.rect); +function candidateMatches( + candidate: IndexedObservationNode, + geometry: TargetGeometry, + geometrySpace: "top" | "local", +): boolean { + const rect = geometrySpace === "local" ? candidate.geometry.localRect : candidate.geometry.rect; + return rect != null && rectMatches(geometry.rect, rect); } export function unmatchedTarget(fallback?: CaptureTargetDescriptor): TargetDescriptorV3 { @@ -53,12 +58,17 @@ export function matchObservationTarget(input: { hint?: TargetMatchHint; fallback?: CaptureTargetDescriptor; }): TargetDescriptorV3 { + if (input.hint?.frameId === null) return unmatchedTarget(input.fallback); const frameId = input.hint?.frameId ?? input.observation.rootFrameId; const geometry = input.hint?.geometry; if (geometry) { const matches = input.observation.index .candidates(frameId, geometry.tag) - .filter((candidate) => candidate.ref && candidateMatches(candidate, geometry)); + .filter( + (candidate) => + candidate.ref && + candidateMatches(candidate, geometry, input.hint?.geometrySpace ?? "top"), + ); if (matches.length === 1) return descriptor(matches[0]!.ref!); const semanticMatches = matches.filter( (candidate) => candidate.ref && matchesSemantics(candidate.ref, input.fallback), diff --git a/apps/extension/src/lib/recording/types.ts b/apps/extension/src/lib/recording/types.ts index 9767ad6..4ccffe2 100644 --- a/apps/extension/src/lib/recording/types.ts +++ b/apps/extension/src/lib/recording/types.ts @@ -15,8 +15,9 @@ export interface TargetGeometry { export interface TargetMatchHint { geometry?: TargetGeometry; - /** Missing means the current top frame, never an unrestricted frame search. */ - frameId?: string; + /** Missing means top frame; null means the source Document could not be resolved. */ + frameId?: string | null; + geometrySpace?: "top" | "local"; } export interface StepAnnotation { diff --git a/apps/extension/src/shared/recording-document-identity.ts b/apps/extension/src/shared/recording-document-identity.ts new file mode 100644 index 0000000..a044a5d --- /dev/null +++ b/apps/extension/src/shared/recording-document-identity.ts @@ -0,0 +1,9 @@ +export const RECORD_DOCUMENT_ATTRIBUTE = "data-bsk-record-document"; + +/** Read BrowserSkill's opaque per-Document recording identity from captured attributes. */ +export function readRecordingDocumentIdentity( + attrs: Readonly>, +): string | undefined { + const identity = attrs[RECORD_DOCUMENT_ATTRIBUTE]; + return identity ? identity : undefined; +} diff --git a/apps/extension/src/tools/record.ts b/apps/extension/src/tools/record.ts index 46f984b..c99f4b7 100644 --- a/apps/extension/src/tools/record.ts +++ b/apps/extension/src/tools/record.ts @@ -18,6 +18,11 @@ import { type RecordStepAck, type RecordStopMessage, } from "@/lib/record-bridge"; +import { + type RecordFrameCoordinator, + type RecordingCaptureScope, + recordFrameCoordinator, +} from "@/lib/recording/frame-coordinator"; import { RecordingObservationRuntime } from "@/lib/recording/recording-runtime"; import { appendRecordedPayload, @@ -262,10 +267,11 @@ async function processRecordedStep( recording: ActiveRecording, draftIndex: number, tabId: number, + producerId?: string, ): Promise { if (!recording.observation) return; try { - await recording.observation.processDraft(tabId, recording.steps, draftIndex); + await recording.observation.processDraft(tabId, recording.steps, draftIndex, producerId); } catch (err) { console.warn(`[bsk record] observation failed for step ${draftIndex + 1}`, err); } @@ -278,6 +284,10 @@ export interface RecordDeps { msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage, ): Promise; bypassOverlay?: (tabId: number, enabled: boolean) => Promise; + frameCoordinator?: Pick< + RecordFrameCoordinator, + "begin" | "armTab" | "sourceFor" | "stop" | "cancel" + >; cdp?: CdpRunner; signal?: AbortSignal; } @@ -288,6 +298,7 @@ function getDefaultDeps(): RecordDeps { defaultDeps = { tabsApi: chromeTabsApi, sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg), + frameCoordinator: recordFrameCoordinator, }; } return defaultDeps; @@ -354,9 +365,15 @@ export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): ( if (!isRecordStepMessage(message)) return false; for (const recording of recordings.values()) { if (recording.requestId !== message.requestId) continue; - const sourceTabId = sender.tab?.id ?? recording.tabs.currentTabId; + const source: RecordingCaptureScope | null | undefined = deps.frameCoordinator + ? deps.frameCoordinator.sourceFor(message.requestId, message.producerId, sender) + : undefined; + if (deps.frameCoordinator && !source) return false; + const sourceTabId = source?.tabId ?? sender.tab?.id ?? recording.tabs.currentTabId; const sourceWasActive = sender.tab?.active ?? sourceTabId === recording.tabs.activeTabId; - const producerKey = `${sourceTabId}:${message.producerId}`; + const producerKey = source + ? `${source.tabId}:${source.documentId}:${source.producerId}` + : `${sourceTabId}:${message.producerId}`; const expectedSequence = (recording.lastStepSequenceByProducer.get(producerKey) ?? 0) + 1; if (message.sequence < expectedSequence) { sendResponse({ ok: true, sequence: message.sequence }); @@ -387,7 +404,7 @@ export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): ( targetHint, ); if (draftIndex !== null) { - await processRecordedStep(recording, draftIndex, sourceTabId); + await processRecordedStep(recording, draftIndex, sourceTabId, source?.producerId); } }); sendResponse({ ok: true, sequence: message.sequence }); @@ -469,6 +486,9 @@ async function stopRecordingOnAllAgentTabs( recording: ActiveRecording, deps: RecordDeps, ): Promise { + if (deps.frameCoordinator && !(await deps.frameCoordinator.stop(recording.requestId))) { + throw new Error("failed to flush one or more recording documents"); + } const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId }; let tabIds = [recording.tabs.currentTabId]; try { @@ -523,6 +543,10 @@ async function rearmRecording( startedAtMs: recording.startedAtMs, }; try { + if (deps.frameCoordinator) { + const frameStarted = await deps.frameCoordinator.armTab(recording.requestId, targetTabId); + if (!frameStarted) throw new Error("recording document did not start"); + } await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab, isFinishing); if (isFinishing()) return false; if (activation) { @@ -860,12 +884,14 @@ export async function handleRecordStart( actionQueue: Promise.resolve(), lastStepSequenceByProducer: new Map(), }); + deps.frameCoordinator?.begin(requestId, startedAtMs); // Observe navigations for the whole recording lifetime; attach before // optional navigate so the destination load can rearm capture. ensureBrowserObservationListeners(deps); const abortPending = async (notifyContent: boolean) => { recordings.get(params.session_id)?.observation?.cancel(); + deps.frameCoordinator?.cancel(requestId); recordings.delete(params.session_id); releaseBrowserObservationListenersIfIdle(); if (notifyContent) { @@ -982,6 +1008,10 @@ export async function handleRecordStart( const startMsg: RecordStartMessage = { type: RECORD_START, requestId, startedAtMs }; try { + if (deps.frameCoordinator) { + const frameStarted = await deps.frameCoordinator.armTab(requestId, target.tabId); + if (!frameStarted) throw new Error("top recording document did not start"); + } await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab); } catch { await abortPending(true); @@ -1101,6 +1131,7 @@ export function clearRecordingForSession(sessionId: string): void { } void clearRearmTimersForRecording(recording, getDefaultDeps()); if (!recording.settled) { + getDefaultDeps().frameCoordinator?.cancel(recording.requestId); recording.observation?.cancel(); recording.settled = true; recording.rejectFinish(new Error("recording cleared")); diff --git a/apps/extension/src/tools/vom/__tests__/record-safe-observation.test.ts b/apps/extension/src/tools/vom/__tests__/record-safe-observation.test.ts new file mode 100644 index 0000000..2889d9e --- /dev/null +++ b/apps/extension/src/tools/vom/__tests__/record-safe-observation.test.ts @@ -0,0 +1,48 @@ +import { describe, expect, it } from "vitest"; +import { RECORD_DOCUMENT_ATTRIBUTE } from "@/shared/recording-document-identity"; +import { projectRecordSafeObservation } from "../record-safe-observation"; + +describe("projectRecordSafeObservation", () => { + it("projects the opaque recording identity without exposing captured attributes", () => { + const result = projectRecordSafeObservation({ + rootFrameId: "child", + frameDocuments: [ + { + frameId: "child", + target: { tabId: 4, sessionId: "oopif-session" }, + contextScopeId: "scope-1", + axNodes: [], + domNodes: [ + { + backendNodeId: 1, + parentBackendNodeId: null, + tag: "html", + attrs: { + [RECORD_DOCUMENT_ATTRIBUTE]: "producer-1", + value: "user@example.com", + }, + rect: null, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + }, + ], + }, + ], + rendered: { text: '@vom 1\nRootWebArea "Example"', refs: [], truncated: false }, + }); + + expect(result.frames).toEqual([ + { + frameId: "child", + target: { tabId: 4, sessionId: "oopif-session" }, + recordingDocumentId: "producer-1", + }, + ]); + const dumped = JSON.stringify(result); + expect(dumped).not.toContain("user@example.com"); + expect(dumped).not.toContain("attrs"); + expect(dumped).not.toContain("domNodes"); + expect(dumped).not.toContain("axNodes"); + }); +}); diff --git a/apps/extension/src/tools/vom/record-safe-observation.ts b/apps/extension/src/tools/vom/record-safe-observation.ts index 3b5befd..b4fd093 100644 --- a/apps/extension/src/tools/vom/record-safe-observation.ts +++ b/apps/extension/src/tools/vom/record-safe-observation.ts @@ -1,5 +1,6 @@ import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom"; import type { CdpTarget } from "@/browser-driver/frame-graph"; +import { readRecordingDocumentIdentity } from "@/shared/recording-document-identity"; import type { CapturedSurfaceProbe } from "./capture"; import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture"; @@ -9,6 +10,8 @@ export interface CaptureVomFrame { target: CdpTarget; parentFrameId?: string; ownerBackendNodeId?: number; + /** BrowserSkill-generated identity for the recording agent in this Document. */ + recordingDocumentId?: string; } /** Allowlisted geometry used to match a recorded action to a rendered ref. */ @@ -37,14 +40,20 @@ export interface CaptureVomObservationResult { } function projectFrames(documents: CapturedFrameDocument[]): CaptureVomFrame[] { - return documents.map((document) => ({ - frameId: document.frameId, - target: document.target, - ...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}), - ...(document.ownerBackendNodeId !== undefined - ? { ownerBackendNodeId: document.ownerBackendNodeId } - : {}), - })); + return documents.map((document) => { + const recordingDocumentId = document.domNodes + .map((node) => readRecordingDocumentIdentity(node.attrs)) + .find((identity) => identity !== undefined); + return { + frameId: document.frameId, + target: document.target, + ...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}), + ...(document.ownerBackendNodeId !== undefined + ? { ownerBackendNodeId: document.ownerBackendNodeId } + : {}), + ...(recordingDocumentId ? { recordingDocumentId } : {}), + }; + }); } function projectMatchNodes(documents: CapturedFrameDocument[]): CaptureVomMatchNode[] {