From f7e08f461b5b0673933d6dbfb3d0101809fa733f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E2=80=9Chaonan=E2=80=9D?= Date: Mon, 17 Aug 2026 14:41:02 +0800 Subject: [PATCH] feat(record): capture interactions in embedded frames Track recording per document so injectable child frames are captured while partial frame failures remain visible in exported traces. Co-authored-by: Cursor --- .gitignore | 4 + apps/extension/src/content/RecordOverlay.tsx | 101 +++-- .../content/__tests__/RecordOverlay.test.tsx | 26 ++ .../content/__tests__/record-capture.test.ts | 76 ++++ apps/extension/src/content/record-capture.ts | 56 ++- apps/extension/src/entrypoints/background.ts | 21 ++ apps/extension/src/entrypoints/content.ts | 25 +- .../src/entrypoints/record-frame.content.ts | 77 ++++ .../src/lib/__tests__/match-target.test.ts | 56 +++ .../lib/__tests__/record-observation.test.ts | 2 + apps/extension/src/lib/match-target.ts | 3 +- apps/extension/src/lib/record-bridge.ts | 34 +- .../src/lib/record-document-token.ts | 24 ++ apps/extension/src/lib/record-frame-arming.ts | 250 +++++++++++++ apps/extension/src/lib/record-observation.ts | 12 +- .../__tests__/capture-vom-observation.test.ts | 86 +++++ .../src/tools/__tests__/record-steps.test.ts | 84 ++++- .../src/tools/capture-vom-observation.ts | 23 ++ apps/extension/src/tools/record.ts | 354 ++++++++++++++---- apps/extension/src/transport/types.ts | 24 ++ crates/bsk-cli/skill/SKILL.md | 1 + crates/bsk-cli/src/cli/record.rs | 12 +- crates/bsk-cli/tests/record_stop_retry.rs | 1 + crates/bsk-protocol/schema/trace.json | 81 ++++ crates/bsk-protocol/src/tools/record.rs | 78 ++++ .../i18n/src/locales/en-US/extension.json | 3 +- .../i18n/src/locales/zh-CN/extension.json | 3 +- skill/SKILL.md | 1 + 28 files changed, 1374 insertions(+), 144 deletions(-) create mode 100644 apps/extension/src/entrypoints/record-frame.content.ts create mode 100644 apps/extension/src/lib/record-document-token.ts create mode 100644 apps/extension/src/lib/record-frame-arming.ts create mode 100644 apps/extension/src/tools/__tests__/capture-vom-observation.test.ts diff --git a/.gitignore b/.gitignore index bdc6661..7dccda2 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,7 @@ dist/ # Environment .env.local .env.*.local +.pnpm-store/ + +# Local recording output +/trace/ \ No newline at end of file diff --git a/apps/extension/src/content/RecordOverlay.tsx b/apps/extension/src/content/RecordOverlay.tsx index 6cd2578..c033aeb 100644 --- a/apps/extension/src/content/RecordOverlay.tsx +++ b/apps/extension/src/content/RecordOverlay.tsx @@ -1,4 +1,5 @@ import { useTranslation } from "@browser-skill/i18n/react"; +import type { FrameCaptureFailure, FrameCaptureStatus } from "@/transport/types"; import { type PointerEvent as ReactPointerEvent, useCallback, @@ -21,6 +22,8 @@ export interface RecordRequestData { type Props = { request: RecordRequestData | null; + captureStatus?: FrameCaptureStatus; + captureFailures?: FrameCaptureFailure[]; }; const VIEWPORT_MARGIN = 16; @@ -52,7 +55,11 @@ function clampDragPos( }; } -export function RecordOverlay({ request }: Props) { +export function RecordOverlay({ + request, + captureStatus = "complete", + captureFailures = [], +}: Props) { const { t } = useTranslation("extension"); const [show, setShow] = useState(false); const [elapsedSeconds, setElapsedSeconds] = useState(0); @@ -154,6 +161,7 @@ export function RecordOverlay({ request }: Props) { const introVisible = elapsedSeconds * 1_000 < INTRO_DURATION_MS; const positioned = dragPos !== null; + const partialCapture = captureStatus === "partial" || captureFailures.length > 0; const defaultTransform = show ? "translateX(-50%) translateY(0)" : "translateX(-50%) translateY(8px)"; @@ -208,8 +216,9 @@ export function RecordOverlay({ request }: Props) { data-slot="record-overlay-status" style={{ display: "flex", - alignItems: "center", - gap: 8, + flexDirection: "column", + alignItems: "flex-start", + gap: 2, minWidth: 0, flex: 1, borderRadius: 9999, @@ -218,41 +227,57 @@ export function RecordOverlay({ request }: Props) { paddingRight: 4, }} > - - - {introVisible ? t("recordOverlay.recording") : formatElapsed(elapsedSeconds)} - +
+ + + {introVisible ? t("recordOverlay.recording") : formatElapsed(elapsedSeconds)} + +
+ {partialCapture ? ( + + {t("recordOverlay.partialCapture")} + + ) : null} + + `; + }); + + it("stamps a document token on start and clears it on dispose", () => { + const capture = startRecordCapture("rec-child-token", (step) => steps.push(step), { + frameMode: "child", + }); + const token = document.documentElement.getAttribute("data-bsk-record-document"); + expect(token).toBeTruthy(); + click(document.querySelector("button")!); + expect(steps[0]?.documentToken).toBe(token); + expect(steps[0]?.frame_url).toBeTruthy(); + capture.dispose(); + expect(document.documentElement.hasAttribute("data-bsk-record-document")).toBe(false); + }); + + it("records click, fill, select, and press in child frame mode", () => { + const capture = startRecordCapture("rec-child-ops", (step) => steps.push(step), { + frameMode: "child", + }); + const input = document.querySelector("input")!; + const select = document.querySelector("select")!; + + input.dispatchEvent(new FocusEvent("focusin", { bubbles: true })); + input.value = "child value"; + input.dispatchEvent(new Event("input", { bubbles: true })); + input.dispatchEvent(new FocusEvent("focusout", { bubbles: true })); + select.value = "off"; + select.dispatchEvent(new Event("change", { bubbles: true })); + input.dispatchEvent(new FocusEvent("focusin", { bubbles: true })); + input.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", bubbles: true })); + click(document.querySelector("button")!); + capture.dispose(); + + expect(steps.map((s) => s.op)).toEqual( + expect.arrayContaining(["fill", "select", "press", "click"]), + ); + expect(steps.every((s) => s.documentToken)).toBe(true); + }); + + it("commits the last fill when dispose runs in child frame mode", () => { + const capture = startRecordCapture("rec-child-fill-stop", (step) => steps.push(step), { + frameMode: "child", + }); + const input = document.querySelector("input")!; + input.dispatchEvent(new FocusEvent("focusin", { bubbles: true })); + input.value = "pending child fill"; + input.dispatchEvent(new Event("input", { bubbles: true })); + capture.dispose(); + + expect(steps.some((s) => s.op === "fill" && s.value === "pending child fill")).toBe(true); + }); + + it("does not emit navigate steps for child frame navigations", () => { + const capture = startRecordCapture("rec-child-nav", (step) => steps.push(step), { + frameMode: "child", + }); + history.pushState({}, "", "/child-frame-path"); + capture.dispose(); + expect(steps.filter((s) => s.op === "navigate")).toHaveLength(0); + }); +}); diff --git a/apps/extension/src/content/record-capture.ts b/apps/extension/src/content/record-capture.ts index 9f48501..80a8fc1 100644 --- a/apps/extension/src/content/record-capture.ts +++ b/apps/extension/src/content/record-capture.ts @@ -21,12 +21,18 @@ import { RECORD_STEP, RECORD_STOP, type RecordCancelMessage, + type RecordFrameMode, type RecordStartAck, type RecordStartMessage, type RecordStepPayload, type RecordStopAck, type RecordStopMessage, } from "@/lib/record-bridge"; +import { + clearRecordDocumentToken, + createRecordDocumentToken, + stampRecordDocumentToken, +} from "@/lib/record-document-token"; import { shouldRecordPress } from "@/lib/trace-reducer"; import { closestHoverSurfaceCandidate, @@ -402,12 +408,37 @@ function scheduleInputCompletionCommit( setTimeout(syncAndCommit, 0); } +export interface RecordCaptureOptions { + frameMode?: RecordFrameMode; +} + export function startRecordCapture( _requestId: string, sendStep: (step: RecordStepPayload) => void, + options: RecordCaptureOptions = {}, ): RecordCaptureController { + const frameMode = options.frameMode ?? "top"; + const emitNavigate = frameMode === "top"; + const documentToken = createRecordDocumentToken(); + stampRecordDocumentToken(documentToken); + const capturedAtMs = () => Date.now(); + + const resolveTopPageUrl = (): string => { + try { + return window.top?.location.href ?? location.href; + } catch { + return location.href; + } + }; + const emitStep = (step: RecordStepPayload) => { - sendStep({ page_url: location.href, ...step }); + sendStep({ + page_url: resolveTopPageUrl(), + frame_url: location.href, + documentToken, + capturedAtMs: capturedAtMs(), + ...step, + }); }; const hoverSurfaceStateMap = (states: HoverSurfaceState[]): Map => new Map(states.map((state) => [state.element, state.signature])); @@ -478,6 +509,7 @@ export function startRecordCapture( }; const emitNavigateIfChanged = (causedByAction?: boolean) => { + if (!emitNavigate) return; if (location.href === lastUrl) return; commitFillSession(); lastUrl = location.href; @@ -965,6 +997,7 @@ export function startRecordCapture( return { dispose() { commitFillSession(); + clearRecordDocumentToken(); document.removeEventListener("click", onClick, true); document.removeEventListener("mouseover", onMouseOver, true); document.removeEventListener("focusin", onFocusIn, true); @@ -997,8 +1030,8 @@ export function handleRecordContentMessage( capture: RecordCaptureController | null; setActiveRequestId(id: string | null): void; setCapture(capture: RecordCaptureController | null): void; - onStart(requestId: string, startedAtMs?: number): void; - onStop(): void; + onStart?(requestId: string, startedAtMs?: number): void; + onStop?(): void; }, sendResponse?: (response: RecordStartAck | RecordStopAck) => void, ): boolean { @@ -1009,13 +1042,20 @@ export function handleRecordContentMessage( failedStepPayloads.delete(message.requestId); } state.capture?.dispose(); + const frameMode = message.frameMode ?? "top"; state.setCapture( - startRecordCapture(message.requestId, (step) => { - sendRecordStep(message.requestId, step); - }), + startRecordCapture( + message.requestId, + (step) => { + sendRecordStep(message.requestId, step); + }, + { frameMode }, + ), ); state.setActiveRequestId(message.requestId); - state.onStart(message.requestId, message.startedAtMs); + if (message.showOverlay !== false && frameMode === "top") { + state.onStart?.(message.requestId, message.startedAtMs); + } sendResponse?.({ ok: true }); return sendResponse !== undefined; } @@ -1030,7 +1070,7 @@ export function handleRecordContentMessage( state.capture?.dispose(); state.setCapture(null); const finishStop = () => { - state.onStop(); + state.onStop?.(); state.setActiveRequestId(null); }; if (isRecordStopMessage(message) && sendResponse) { diff --git a/apps/extension/src/entrypoints/background.ts b/apps/extension/src/entrypoints/background.ts index 4154b4b..f5bf8b1 100644 --- a/apps/extension/src/entrypoints/background.ts +++ b/apps/extension/src/entrypoints/background.ts @@ -33,6 +33,9 @@ import { defaultBorrowChromeWindows, requestBorrowConfirmation, } from "@/tools/borrow-confirmation"; +import { + type RecordCaptureStatusMessage, +} from "@/lib/record-bridge"; import { ToolDispatcher } from "@/tools/dispatcher"; import { attachRecordFinishListener, @@ -175,6 +178,24 @@ export default defineBackground(() => { cdp, sendToTab: (tabId: number, msg: Parameters[1]) => chrome.tabs.sendMessage(tabId, msg), + sendToDocument: ( + tabId: number, + documentId: string, + msg: Parameters[1], + ) => chrome.tabs.sendMessage(tabId, msg, { documentId }), + getAllFrames: async (tabId: number) => { + if (!chrome.webNavigation?.getAllFrames) return []; + const frames = await chrome.webNavigation.getAllFrames({ tabId }); + return (frames ?? []).map((frame) => ({ + frameId: frame.frameId, + documentId: frame.documentId, + url: frame.url, + parentFrameId: frame.parentFrameId, + })); + }, + broadcastCaptureStatus: async (tabId: number, msg: RecordCaptureStatusMessage) => { + await chrome.tabs.sendMessage(tabId, msg, { frameId: 0 }); + }, bypassOverlay: async (tabId: number, enabled: boolean) => { try { await chrome.tabs.sendMessage(tabId, { diff --git a/apps/extension/src/entrypoints/content.ts b/apps/extension/src/entrypoints/content.ts index 52536ad..eab9d1a 100644 --- a/apps/extension/src/entrypoints/content.ts +++ b/apps/extension/src/entrypoints/content.ts @@ -44,12 +44,15 @@ import { } from "@/lib/overlay-bridge"; import { sendInterrupt } from "@/lib/overlay-interrupt-client"; import { + isRecordCaptureStatusMessage, RECORD_FINISH, RECORD_QUERY, + type RecordCaptureStatusMessage, type RecordQueryResponse, type RecordStartAck, type RecordStopAck, } from "@/lib/record-bridge"; +import type { FrameCaptureFailure, FrameCaptureStatus } from "@/transport/types"; import type { BorrowCancelMessage, BorrowRequestMessage, @@ -70,6 +73,8 @@ export default defineContentScript({ const overlays = new OverlayController(); let recordCapture: RecordCaptureController | null = null; let activeRecordRequestId: string | null = null; + let recordCaptureStatus: FrameCaptureStatus = "complete"; + let recordCaptureFailures: FrameCaptureFailure[] = []; let reactRoot: ReactDOM.Root | null = null; let overlayHost: HTMLElement | null = null; let overlayContainer: HTMLElement | null = null; @@ -188,7 +193,11 @@ export default defineContentScript({ requests: overlayState.borrowRequests, }), React.createElement(HelpRequestOverlay, { request: overlayState.activeHelp }), - React.createElement(RecordOverlay, { request: overlayState.activeRecord }), + React.createElement(RecordOverlay, { + request: overlayState.activeRecord, + captureStatus: recordCaptureStatus, + captureFailures: recordCaptureFailures, + }), React.createElement(ControlOverlay, { visible: controlOverlayVisible, interrupting: overlayState.interrupting, @@ -260,7 +269,8 @@ export default defineContentScript({ | CaptureSuppressMessage | OverlayAgentOverlayResetMessage | OverlayAgentStateMessage - | OverlayAutomationBypassMessage, + | OverlayAutomationBypassMessage + | RecordCaptureStatusMessage, _sender: chrome.runtime.MessageSender, sendResponse: (response: BorrowResponseMessage | HelpAckMessage | CaptureSuppressAck) => void, ) => { @@ -268,6 +278,13 @@ export default defineContentScript({ return captureSuppress.handleMessage(message, sendResponse); } + if (isRecordCaptureStatusMessage(message)) { + recordCaptureStatus = message.status; + recordCaptureFailures = message.failures ?? []; + renderAll(); + return false; + } + if (isRecordContentMessage(message)) { const needsAsync = handleRecordContentMessage( message, @@ -295,6 +312,8 @@ export default defineContentScript({ }, onStop: () => { overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined); + recordCaptureStatus = "complete"; + recordCaptureFailures = []; renderAll(); }, }, @@ -449,6 +468,8 @@ export default defineContentScript({ ) { const requestId = recordQuery.requestId; const startedAtMs = recordQuery.startedAtMs; + recordCaptureStatus = recordQuery.captureStatus ?? "complete"; + recordCaptureFailures = recordQuery.captureFailures ?? []; overlays.setAgentRecordRequest({ id: requestId, ...(typeof startedAtMs === "number" ? { startedAtMs } : {}), diff --git a/apps/extension/src/entrypoints/record-frame.content.ts b/apps/extension/src/entrypoints/record-frame.content.ts new file mode 100644 index 0000000..61bddeb --- /dev/null +++ b/apps/extension/src/entrypoints/record-frame.content.ts @@ -0,0 +1,77 @@ +import { + handleRecordContentMessage, + isRecordContentMessage, + type RecordCaptureController, +} from "@/content/record-capture"; +import { + RECORD_QUERY, + type RecordQueryResponse, +} from "@/lib/record-bridge"; + +/** Capture-only content script for child frames (no React overlay). */ +export default defineContentScript({ + matches: [""], + runAt: "document_end", + allFrames: true, + matchAboutBlank: true, + matchOriginAsFallback: true, + + main(ctx) { + if (window.top === window) return; + + let recordCapture: RecordCaptureController | null = null; + let activeRecordRequestId: string | null = null; + + const onMessage = ( + message: unknown, + _sender: chrome.runtime.MessageSender, + sendResponse: (response: unknown) => void, + ) => { + if (isRecordContentMessage(message)) { + const needsAsync = handleRecordContentMessage( + message, + { + activeRequestId: activeRecordRequestId, + capture: recordCapture, + setActiveRequestId: (id) => { + activeRecordRequestId = id; + }, + setCapture: (capture) => { + recordCapture = capture; + }, + }, + sendResponse as never, + ); + return needsAsync; + } + return false; + }; + + async function queryActiveRecord(): Promise { + try { + const recordQuery = (await chrome.runtime.sendMessage({ + type: RECORD_QUERY, + })) as RecordQueryResponse | undefined; + if ( + recordQuery?.active && + typeof recordQuery.requestId === "string" && + activeRecordRequestId === null + ) { + // Background rearm sends RECORD_START; nothing to mount locally. + } + } catch (err) { + console.debug("[bsk record-frame] record query failed", err); + } + } + + chrome.runtime.onMessage.addListener(onMessage); + void queryActiveRecord(); + + ctx.onInvalidated(() => { + chrome.runtime.onMessage.removeListener(onMessage); + recordCapture?.dispose(); + recordCapture = null; + activeRecordRequestId = null; + }); + }, +}); diff --git a/apps/extension/src/lib/__tests__/match-target.test.ts b/apps/extension/src/lib/__tests__/match-target.test.ts index f4edddc..8592d2e 100644 --- a/apps/extension/src/lib/__tests__/match-target.test.ts +++ b/apps/extension/src/lib/__tests__/match-target.test.ts @@ -113,4 +113,60 @@ describe("matchTarget", () => { }); expect(target.ref).toBe("e1"); }); + + it("isolates geometry matching to the owner iframe backend node", () => { + const sharedRect = { x: 10, y: 20, w: 100, h: 30 }; + const iframeRefs: RenderedRef[] = [ + { ref: "e1", backendNodeId: 42, role: "button", name: "Top", line: 1 }, + { ref: "e2", backendNodeId: 99, role: "button", name: "Iframe", line: 2 }, + ]; + const target = matchTarget({ + geometry: { + rect: sharedRect, + scrollX: 0, + scrollY: 0, + position: "static", + tag: "button", + ownerFrameBackendNodeId: 13, + }, + captured: [ + node(42, { rect: sharedRect, documentRect: sharedRect }), + node(99, { + rect: sharedRect, + documentRect: sharedRect, + ownerFrameBackendNodeId: 13, + }), + ], + refs: iframeRefs, + fallback: { tag: "button", name: "Iframe" }, + }); + expect(target).toEqual({ ref: "e2", role: "button", name: "Iframe" }); + }); + + it("matches iframe static elements using frame scroll and document coordinates", () => { + const iframeRefs: RenderedRef[] = [ + { ref: "e2", backendNodeId: 99, role: "button", name: "Iframe", line: 2 }, + ]; + const target = matchTarget({ + geometry: { + rect: { x: 10, y: 50, w: 100, h: 30 }, + scrollX: 0, + scrollY: 120, + position: "static", + tag: "button", + ownerFrameBackendNodeId: 13, + }, + captured: [ + node(99, { + ownerFrameBackendNodeId: 13, + documentRect: { x: 10, y: 170, w: 100, h: 30 }, + localRect: { x: 10, y: 50, w: 100, h: 30 }, + rect: { x: 10, y: 50, w: 100, h: 30 }, + }), + ], + refs: iframeRefs, + fallback: { tag: "button", name: "Iframe" }, + }); + expect(target).toEqual({ ref: "e2", role: "button", name: "Iframe" }); + }); }); diff --git a/apps/extension/src/lib/__tests__/record-observation.test.ts b/apps/extension/src/lib/__tests__/record-observation.test.ts index c1f9b3d..d795b59 100644 --- a/apps/extension/src/lib/__tests__/record-observation.test.ts +++ b/apps/extension/src/lib/__tests__/record-observation.test.ts @@ -33,6 +33,8 @@ function finalizedFillBody(value: string, redactValues: boolean): string { obs.lastSettled = { stateId, captured: [capturedInput()], + matchNodes: [capturedInput()], + documentTokenOwners: new Map(), refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }], url: URL, vomText: rawVomText, diff --git a/apps/extension/src/lib/match-target.ts b/apps/extension/src/lib/match-target.ts index 9c27662..4cca21e 100644 --- a/apps/extension/src/lib/match-target.ts +++ b/apps/extension/src/lib/match-target.ts @@ -40,9 +40,8 @@ function matchingRects( target: { x: number; y: number; w: number; h: number }; node: { x: number; y: number; w: number; h: number } | null; } { - const topFrame = (geometry.ownerFrameBackendNodeId ?? null) === null; const viewportPosition = geometry.position === "fixed" || geometry.position === "sticky"; - if (!topFrame || viewportPosition || !node.documentRect) { + if (viewportPosition || !node.documentRect) { return { target: geometry.rect, node: nodeViewportRect(node) }; } return { diff --git a/apps/extension/src/lib/record-bridge.ts b/apps/extension/src/lib/record-bridge.ts index 3528a3a..7e5067d 100644 --- a/apps/extension/src/lib/record-bridge.ts +++ b/apps/extension/src/lib/record-bridge.ts @@ -4,6 +4,7 @@ */ import type { CaptureTargetDescriptor } from "./describe-target"; +import type { FrameCaptureFailure, FrameCaptureStatus } from "@/transport/types"; export const RECORD_START = "bsk-record-start"; export const RECORD_STEP = "bsk-record-step"; @@ -11,6 +12,7 @@ export const RECORD_STOP = "bsk-record-stop"; export const RECORD_CANCEL = "bsk-record-cancel"; export const RECORD_FINISH = "bsk-record-finish"; export const RECORD_QUERY = "bsk-record-query"; +export const RECORD_CAPTURE_STATUS = "bsk-record-capture-status"; export interface RecordStartAck { ok: true; @@ -27,8 +29,12 @@ export interface RecordQueryResponse { requestId?: string; /** Epoch ms when the recording began; see `RecordStartMessage.startedAtMs`. */ startedAtMs?: number; + captureStatus?: FrameCaptureStatus; + captureFailures?: FrameCaptureFailure[]; } +export type RecordFrameMode = "top" | "child"; + export interface RecordStartMessage { type: typeof RECORD_START; requestId: string; @@ -38,6 +44,10 @@ export interface RecordStartMessage { * content-script remounts instead of restarting per page. */ startedAtMs?: number; + /** When true, the top-frame overlay should mount. Child frames omit this. */ + showOverlay?: boolean; + /** Whether this document runs top-frame or child-frame capture semantics. */ + frameMode?: RecordFrameMode; } export interface RecordStepPayload { @@ -51,8 +61,14 @@ export interface RecordStepPayload { url?: string; redacted?: boolean; commit?: "enter" | "suggestion" | "blur"; - /** Page URL when the step was captured. */ + /** Top-level tab URL when the step was captured. */ page_url?: string; + /** URL of the frame document where the interaction occurred. */ + frame_url?: string; + /** Ephemeral token stamped on the active document root; not persisted in trace steps. */ + documentToken?: string; + /** Epoch ms when the step was captured in the content script. */ + capturedAtMs?: number; /** Document geometry at capture time for background geometric matching. */ geometry?: { rect: { x: number; y: number; w: number; h: number }; @@ -92,6 +108,12 @@ export interface RecordFinishMessage { requestId: string; } +export interface RecordCaptureStatusMessage { + type: typeof RECORD_CAPTURE_STATUS; + status: FrameCaptureStatus; + failures?: FrameCaptureFailure[]; +} + export function isRecordStartMessage(msg: unknown): msg is RecordStartMessage { if (typeof msg !== "object" || msg === null) return false; const m = msg as Record; @@ -122,6 +144,16 @@ export function isRecordQueryMessage(msg: unknown): msg is RecordQueryMessage { return m.type === RECORD_QUERY; } +export function isRecordCaptureStatusMessage(msg: unknown): msg is RecordCaptureStatusMessage { + if (typeof msg !== "object" || msg === null) return false; + const m = msg as Record; + return ( + m.type === RECORD_CAPTURE_STATUS && + (m.status === "complete" || m.status === "partial") && + (m.failures === undefined || Array.isArray(m.failures)) + ); +} + export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage { if (typeof msg !== "object" || msg === null) return false; const m = msg as Record; diff --git a/apps/extension/src/lib/record-document-token.ts b/apps/extension/src/lib/record-document-token.ts new file mode 100644 index 0000000..64cc67f --- /dev/null +++ b/apps/extension/src/lib/record-document-token.ts @@ -0,0 +1,24 @@ +/** Attribute written on `` while a document is actively recording. */ +export const RECORD_DOCUMENT_TOKEN_ATTR = "data-bsk-record-document"; + +export function createRecordDocumentToken(): string { + if (typeof crypto !== "undefined" && typeof crypto.randomUUID === "function") { + return crypto.randomUUID(); + } + return `doc-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`; +} + +export function stampRecordDocumentToken(token: string): void { + document.documentElement.setAttribute(RECORD_DOCUMENT_TOKEN_ATTR, token); +} + +export function clearRecordDocumentToken(): void { + document.documentElement.removeAttribute(RECORD_DOCUMENT_TOKEN_ATTR); +} + +export function readRecordDocumentTokenFromAttrs( + attrs: Record | undefined, +): string | undefined { + const token = attrs?.[RECORD_DOCUMENT_TOKEN_ATTR]; + return token?.trim() ? token : undefined; +} diff --git a/apps/extension/src/lib/record-frame-arming.ts b/apps/extension/src/lib/record-frame-arming.ts new file mode 100644 index 0000000..cc12c1e --- /dev/null +++ b/apps/extension/src/lib/record-frame-arming.ts @@ -0,0 +1,250 @@ +import { + RECORD_CANCEL, + RECORD_START, + RECORD_STOP, + type RecordCancelMessage, + type RecordStartAck, + type RecordStartMessage, + type RecordStopAck, + type RecordStopMessage, +} from "@/lib/record-bridge"; +import type { FrameCaptureFailure, FrameCaptureFailureReason } from "@/transport/types"; + +export interface WebNavigationFrameInfo { + frameId: number; + documentId?: string; + url?: string; + parentFrameId: number; +} + +export interface ArmedDocument { + tabId: number; + frameId: number; + documentId: string; + url?: string; + parentFrameId: number; +} + +export interface RecordFrameSendDeps { + sendToDocument( + tabId: number, + documentId: string, + msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage, + ): Promise; + getAllFrames(tabId: number): Promise; +} + +const RECORD_START_RETRIES = 3; +const RECORD_START_RETRY_DELAY_MS = 500; + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +export function isFrameUrlRestricted(url: string | undefined): boolean { + if (!url) return false; + const lower = url.toLowerCase(); + return ( + lower.startsWith("chrome://") || + lower.startsWith("chrome-extension://") || + lower.startsWith("edge://") || + lower.startsWith("devtools://") || + lower.startsWith("devtools:") || + lower.startsWith("https://chrome.google.com/webstore") + ); +} + +function isRecordStartAck(response: unknown): response is RecordStartAck { + return ( + typeof response === "object" && + response !== null && + "ok" in response && + (response as RecordStartAck).ok === true + ); +} + +function isRecordStopAckOk(response: unknown): boolean { + return ( + typeof response === "object" && + response !== null && + "ok" in response && + (response as RecordStopAck).ok === true + ); +} + +export function makeFrameCaptureFailure( + reason: FrameCaptureFailureReason, + input: { frameId: number; documentId?: string; url?: string; detail?: string }, +): FrameCaptureFailure { + return { + reason, + frame_id: input.frameId, + ...(input.documentId ? { document_id: input.documentId } : {}), + ...(input.url ? { url: input.url } : {}), + ...(input.detail ? { detail: input.detail } : {}), + }; +} + +export async function listInjectableFrames( + tabId: number, + deps: RecordFrameSendDeps, +): Promise<{ injectable: WebNavigationFrameInfo[]; failures: FrameCaptureFailure[] }> { + const failures: FrameCaptureFailure[] = []; + let frames: WebNavigationFrameInfo[] = []; + try { + frames = await deps.getAllFrames(tabId); + } catch (err) { + failures.push( + makeFrameCaptureFailure("arm_failed", { + frameId: 0, + detail: err instanceof Error ? err.message : "webNavigation.getAllFrames failed", + }), + ); + return { injectable: [], failures }; + } + + const injectable = frames.filter((frame) => { + if (!frame.documentId) { + failures.push( + makeFrameCaptureFailure("not_injectable", { + frameId: frame.frameId, + url: frame.url, + detail: "missing documentId", + }), + ); + return false; + } + if (isFrameUrlRestricted(frame.url)) { + failures.push( + makeFrameCaptureFailure("not_injectable", { + frameId: frame.frameId, + documentId: frame.documentId, + url: frame.url, + }), + ); + return false; + } + return true; + }); + return { injectable, failures }; +} + +export async function sendRecordStartWithAckToDocument( + tabId: number, + documentId: string, + msg: RecordStartMessage, + deps: RecordFrameSendDeps, +): Promise { + let lastError: unknown; + for (let attempt = 0; attempt < RECORD_START_RETRIES; attempt += 1) { + try { + const response = await deps.sendToDocument(tabId, documentId, msg); + if (isRecordStartAck(response)) return; + lastError = new Error("content script did not ack RECORD_START"); + } catch (err) { + lastError = err; + } + if (attempt + 1 < RECORD_START_RETRIES) { + await sleep(RECORD_START_RETRY_DELAY_MS); + } + } + throw lastError ?? new Error("failed to start recording in content script"); +} + +export async function armDocumentCapture( + tabId: number, + frame: WebNavigationFrameInfo, + startMsg: RecordStartMessage, + deps: RecordFrameSendDeps, +): Promise<{ armed?: ArmedDocument; failure?: FrameCaptureFailure }> { + if (!frame.documentId) { + return { + failure: makeFrameCaptureFailure("not_injectable", { + frameId: frame.frameId, + url: frame.url, + detail: "missing documentId", + }), + }; + } + if (isFrameUrlRestricted(frame.url)) { + return { + failure: makeFrameCaptureFailure("not_injectable", { + frameId: frame.frameId, + documentId: frame.documentId, + url: frame.url, + }), + }; + } + try { + await sendRecordStartWithAckToDocument(tabId, frame.documentId, startMsg, deps); + return { + armed: { + tabId, + frameId: frame.frameId, + documentId: frame.documentId, + url: frame.url, + parentFrameId: frame.parentFrameId, + }, + }; + } catch (err) { + return { + failure: makeFrameCaptureFailure("arm_failed", { + frameId: frame.frameId, + documentId: frame.documentId, + url: frame.url, + detail: err instanceof Error ? err.message : "RECORD_START failed", + }), + }; + } +} + +export async function armAllDocumentsForTab( + tabId: number, + startMsg: RecordStartMessage, + deps: RecordFrameSendDeps, +): Promise<{ armed: ArmedDocument[]; failures: FrameCaptureFailure[] }> { + const { injectable, failures } = await listInjectableFrames(tabId, deps); + const armed: ArmedDocument[] = []; + for (const frame of injectable) { + const result = await armDocumentCapture(tabId, frame, startMsg, deps); + if (result.armed) armed.push(result.armed); + if (result.failure) failures.push(result.failure); + } + return { armed, failures }; +} + +export async function stopDocumentCapture( + armed: ArmedDocument, + stopMsg: RecordStopMessage, + deps: RecordFrameSendDeps, +): Promise<{ ok: true } | FrameCaptureFailure> { + try { + const response = await deps.sendToDocument(armed.tabId, armed.documentId, stopMsg); + if (isRecordStopAckOk(response)) return { ok: true }; + return makeFrameCaptureFailure("flush_failed", { + frameId: armed.frameId, + documentId: armed.documentId, + url: armed.url, + detail: "content script did not confirm recorded steps", + }); + } catch (err) { + return makeFrameCaptureFailure("flush_failed", { + frameId: armed.frameId, + documentId: armed.documentId, + url: armed.url, + detail: err instanceof Error ? err.message : "RECORD_STOP failed", + }); + } +} + +export async function cancelDocumentCapture( + armed: ArmedDocument, + cancelMsg: RecordCancelMessage, + deps: RecordFrameSendDeps, +): Promise { + try { + await deps.sendToDocument(armed.tabId, armed.documentId, cancelMsg); + } catch { + // Best-effort cleanup. + } +} diff --git a/apps/extension/src/lib/record-observation.ts b/apps/extension/src/lib/record-observation.ts index 97e15de..78a1e7c 100644 --- a/apps/extension/src/lib/record-observation.ts +++ b/apps/extension/src/lib/record-observation.ts @@ -17,6 +17,8 @@ import { reduceTraceSteps, resolveTraceStartUrl } from "./trace-reducer"; export interface LastSettledObservation { stateId: string; captured: CapturedNode[]; + matchNodes: CapturedNode[]; + documentTokenOwners: Map; refs: RenderedRef[]; url: string; title?: string; @@ -40,6 +42,8 @@ export interface PendingRedirectLanding { export interface RecordingObservationState { stateRegistry: Map; lastSettled: LastSettledObservation | null; + /** Latest token→owner map from the most recent observation capture. */ + documentTokenOwners: Map; maxPageTokens: number; redactValues: boolean; lastCaptureAtMs: number; @@ -67,6 +71,7 @@ export function createObservationState(options?: { return { stateRegistry: new Map(), lastSettled: null, + documentTokenOwners: new Map(), maxPageTokens: options?.maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS, redactValues: options?.redactValues ?? false, lastCaptureAtMs: 0, @@ -125,9 +130,12 @@ export async function captureAndRegisterObservation( }); obs.lastCaptureAtMs = Date.now(); + obs.documentTokenOwners = rendered.documentTokenOwners; const settled: LastSettledObservation = { stateId, captured: rendered.captured, + matchNodes: rendered.matchNodes, + documentTokenOwners: rendered.documentTokenOwners, refs: rendered.refs, url, title: resolvedTitle, @@ -164,7 +172,7 @@ export function applyTargetMatching( "geometry" in draft && draft.geometry ? matchTarget({ geometry: draft.geometry, - captured: obs.lastSettled.captured, + captured: obs.lastSettled.matchNodes, refs: obs.lastSettled.refs, fallback: draft.captureTarget, }) @@ -636,6 +644,7 @@ export function buildTraceV3(input: { startUrl?: string; stoppedBy: StopReason; bskVersion: string; + frameCapture?: Trace["frame_capture"]; }): Trace { const { steps, stepIdByDraftId } = reduceTraceSteps(input.steps, input.obs.stateRegistry); const published = selectPublishedStates(input.obs.stateRegistry, steps); @@ -657,6 +666,7 @@ export function buildTraceV3(input: { stopped_by: input.stoppedBy, entry: { start_url: startUrl }, recorder: { bsk: input.bskVersion, vom: 1 }, + ...(input.frameCapture ? { frame_capture: input.frameCapture } : {}), states, steps, }; diff --git a/apps/extension/src/tools/__tests__/capture-vom-observation.test.ts b/apps/extension/src/tools/__tests__/capture-vom-observation.test.ts new file mode 100644 index 0000000..4253c8d --- /dev/null +++ b/apps/extension/src/tools/__tests__/capture-vom-observation.test.ts @@ -0,0 +1,86 @@ +import { describe, expect, it } from "vitest"; +import { RECORD_DOCUMENT_TOKEN_ATTR } from "@/lib/record-document-token"; +import type { CapturedNode } from "@/tools/vom/capture"; +import { collectDocumentTokenOwners } from "../capture-vom-observation"; + +function htmlNode( + backendNodeId: number, + token: string, + ownerFrameBackendNodeId: number | null = null, +): CapturedNode { + return { + backendNodeId, + parentBackendNodeId: null, + tag: "html", + attrs: { [RECORD_DOCUMENT_TOKEN_ATTR]: token }, + rect: { x: 0, y: 0, w: 1280, h: 720 }, + paintOrder: 0, + position: "static", + pointerEvents: "auto", + ...(ownerFrameBackendNodeId === null ? {} : { ownerFrameBackendNodeId }), + }; +} + +describe("collectDocumentTokenOwners", () => { + it("maps document tokens to their immediate iframe owner backend node", () => { + const topToken = "top-doc-token"; + const childToken = "child-doc-token"; + const owners = collectDocumentTokenOwners({ + viewport: { width: 1280, height: 720 }, + nodes: [htmlNode(1, topToken, null)], + iframeNodes: new Map([ + [ + 13, + [ + htmlNode(101, childToken, 13), + { + backendNodeId: 102, + parentBackendNodeId: 101, + tag: "input", + attrs: {}, + rect: { x: 10, y: 20, w: 200, h: 30 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + ownerFrameBackendNodeId: 13, + }, + ], + ], + ]), + excludedBackendNodeIds: new Set(), + }); + + expect(owners.get(topToken)).toBeNull(); + expect(owners.get(childToken)).toBe(13); + }); + + it("maps nested iframe documents to their direct owner, not the outer iframe", () => { + const nestedToken = "nested-doc-token"; + const owners = collectDocumentTokenOwners({ + viewport: { width: 1280, height: 720 }, + nodes: [htmlNode(1, "top-token", null)], + iframeNodes: new Map([ + [ + 13, + [ + { + backendNodeId: 23, + parentBackendNodeId: 101, + tag: "iframe", + attrs: {}, + rect: { x: 0, y: 0, w: 400, h: 300 }, + paintOrder: 1, + position: "static", + pointerEvents: "auto", + ownerFrameBackendNodeId: 13, + }, + ], + ], + [23, [htmlNode(201, nestedToken, 23)]], + ]), + excludedBackendNodeIds: new Set(), + }); + + expect(owners.get(nestedToken)).toBe(23); + }); +}); diff --git a/apps/extension/src/tools/__tests__/record-steps.test.ts b/apps/extension/src/tools/__tests__/record-steps.test.ts index b235304..790c48a 100644 --- a/apps/extension/src/tools/__tests__/record-steps.test.ts +++ b/apps/extension/src/tools/__tests__/record-steps.test.ts @@ -18,11 +18,13 @@ import { handleRecordStart, handleRecordStop, resetBrowserObservationForTests, + type RecordDeps, } from "../record"; const AGENT_WINDOW_ID = 100; const TAB_ID = 4; const START_URL = "https://example.com/"; +const TOP_DOCUMENT_ID = "doc-top"; type RuntimeListener = ( message: unknown, @@ -63,9 +65,18 @@ function installChrome() { webNavigation: { onCompleted: webNavigationOnCompleted, onCommitted: webNavigationOnCommitted, + getAllFrames: vi.fn(async ({ tabId }: { tabId: number }) => [ + { + frameId: 0, + documentId: TOP_DOCUMENT_ID, + url: START_URL, + parentFrameId: -1, + tabId, + }, + ]), }, }); - return { runtimeOnMessage, webNavigationOnCompleted, webNavigationOnCommitted }; + return { runtimeOnMessage, webNavigationOnCompleted, webNavigationOnCommitted, TOP_DOCUMENT_ID }; } function fakeManager() { @@ -206,6 +217,57 @@ function makeTabsApi() { }; } +function makeRecordDeps( + tabsApi: ReturnType | ReturnType, + sendToTabMock: (tabId: number, msg: unknown) => Promise, + cdp?: ReturnType, +): RecordDeps { + return { + tabsApi, + sendToTab: sendToTabMock as RecordDeps["sendToTab"], + sendToDocument: (async (tabId, _documentId, msg) => + sendToTabMock(tabId, msg)) as RecordDeps["sendToDocument"], + getAllFrames: (async () => [ + { + frameId: 0, + documentId: TOP_DOCUMENT_ID, + url: START_URL, + parentFrameId: -1, + }, + ]) as RecordDeps["getAllFrames"], + ...(cdp ? { cdp } : {}), + }; +} + +function makeMultiTabRecordDeps( + tabsApi: ReturnType, + sendToTabMock: (tabId: number, msg: unknown) => Promise, + tabIds: number[], + cdp?: ReturnType, +): RecordDeps { + const framesByTab = new Map( + tabIds.map((tabId) => [ + tabId, + [ + { + frameId: 0, + documentId: tabId === TAB_ID ? TOP_DOCUMENT_ID : `doc-tab-${tabId}`, + url: tabId === TAB_ID ? START_URL : `https://example.com/tab-${tabId}`, + parentFrameId: -1, + }, + ], + ]), + ); + return { + tabsApi, + sendToTab: sendToTabMock as RecordDeps["sendToTab"], + sendToDocument: (async (tabId, _documentId, msg) => + sendToTabMock(tabId, msg)) as RecordDeps["sendToDocument"], + getAllFrames: (async (tabId) => framesByTab.get(tabId) ?? []) as RecordDeps["getAllFrames"], + ...(cdp ? { cdp } : {}), + }; +} + function makeMultiTabsApi(tabs: chrome.tabs.Tab[]) { const byId = new Map(tabs.flatMap((tab) => (typeof tab.id === "number" ? [[tab.id, tab]] : []))); return { @@ -1171,25 +1233,25 @@ describe("recorded user steps reach the exported trace", () => { } return { ok: true }; }); - const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() }; + const deps = makeMultiTabRecordDeps(tabsApi, sendToTab, [TAB_ID, 5], makeFakeCdp()); await handleRecordStart(manager, { session_id: "abcd", url: START_URL }, deps); attachRecordQueryListener(deps); await new Promise((resolve) => { chromeApi.runtimeOnMessage.emit( { type: RECORD_QUERY }, - { tab: secondTab } as chrome.runtime.MessageSender, + { tab: secondTab, documentId: "doc-tab-5" } as chrome.runtime.MessageSender, () => resolve(), ); }); const firstStop = await handleRecordStop(manager, { session_id: "abcd" }, deps); - expect(firstStop).toMatchObject({ code: "protocol_error" }); + expect((firstStop as RecordStopResult).trace.version).toBe(3); + expect((firstStop as RecordStopResult).trace.frame_capture?.status).toBe("partial"); expect(stopTabs).toEqual([TAB_ID, 5]); const retry = await handleRecordStop(manager, { session_id: "abcd" }, deps); - expect((retry as RecordStopResult).trace.version).toBe(3); - expect(stopTabs).toEqual([TAB_ID, 5, 5]); + expect(retry).toMatchObject({ code: "not_found" }); }); it("does not let a closed armed tab block finish", async () => { @@ -1215,14 +1277,14 @@ describe("recorded user steps reach the exported trace", () => { if ((msg as { type?: string }).type === RECORD_STOP) stopTabs.push(tabId); return { ok: true }; }); - const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() }; + const deps = makeMultiTabRecordDeps(tabsApi, sendToTab, [TAB_ID, 5], makeFakeCdp()); await handleRecordStart(manager, { session_id: "abcd", url: START_URL }, deps); attachRecordQueryListener(deps); await new Promise((resolve) => { chromeApi.runtimeOnMessage.emit( { type: RECORD_QUERY }, - { tab: secondTab } as chrome.runtime.MessageSender, + { tab: secondTab, documentId: "doc-tab-5" } as chrome.runtime.MessageSender, () => resolve(), ); }); @@ -1231,7 +1293,7 @@ describe("recorded user steps reach the exported trace", () => { const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps); expect((stopped as RecordStopResult).trace.version).toBe(3); - expect(stopTabs).toEqual([TAB_ID]); + expect(stopTabs).toContain(TAB_ID); }); it("waits for an acknowledged in-flight rearm before stopping armed tabs", async () => { @@ -1270,13 +1332,13 @@ describe("recorded user steps reach the exported trace", () => { if (type === RECORD_STOP) stopTabs.push(tabId); return { ok: true }; }); - const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() }; + const deps = makeMultiTabRecordDeps(tabsApi, sendToTab, [TAB_ID, 5], makeFakeCdp()); await handleRecordStart(manager, { session_id: "abcd", url: START_URL }, deps); attachRecordQueryListener(deps); chromeApi.runtimeOnMessage.emit( { type: RECORD_QUERY }, - { tab: secondTab } as chrome.runtime.MessageSender, + { tab: secondTab, documentId: "doc-tab-5" } as chrome.runtime.MessageSender, () => {}, ); await secondStartBegan; diff --git a/apps/extension/src/tools/capture-vom-observation.ts b/apps/extension/src/tools/capture-vom-observation.ts index 41bd205..ad4c345 100644 --- a/apps/extension/src/tools/capture-vom-observation.ts +++ b/apps/extension/src/tools/capture-vom-observation.ts @@ -1,4 +1,5 @@ import { type RenderedRef, renderVom, type VomOptions } from "@browser-skill/vom"; +import { readRecordDocumentTokenFromAttrs } from "@/lib/record-document-token"; import type { CdpAxNode } from "./observation"; import { buildVomScene } from "./observation"; import type { CdpRunner } from "./shared"; @@ -20,9 +21,28 @@ export interface CaptureVomObservationResult { refs: RenderedRef[]; truncated: boolean; captured: CapturedNode[]; + /** All nodes including iframe sub-documents, for geometric target matching. */ + matchNodes: CapturedNode[]; + /** Maps ephemeral record document tokens to immediate iframe owner backend ids. */ + documentTokenOwners: Map; surfaceProbes?: CapturedSurfaceProbe[]; } +export function flattenCapturedNodes(vm: CapturedViewModel): CapturedNode[] { + return [...vm.nodes, ...[...vm.iframeNodes.values()].flat()]; +} + +export function collectDocumentTokenOwners(vm: CapturedViewModel): Map { + const owners = new Map(); + for (const node of flattenCapturedNodes(vm)) { + if (node.tag !== "html") continue; + const token = readRecordDocumentTokenFromAttrs(node.attrs); + if (!token) continue; + owners.set(token, node.ownerFrameBackendNodeId ?? null); + } + return owners; +} + function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel { return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() }; } @@ -91,11 +111,14 @@ export async function captureVomObservation( redactValues: opts.redactValues, activeRegionPolicy: opts.activeRegionPolicy ?? true, }); + const matchNodes = flattenCapturedNodes(captured); return { text: rendered.text, refs: rendered.refs, truncated: rendered.truncated, captured: captured.nodes, + matchNodes, + documentTokenOwners: collectDocumentTokenOwners(captured), surfaceProbes: captured.surfaceProbes, }; } diff --git a/apps/extension/src/tools/record.ts b/apps/extension/src/tools/record.ts index 5438cf7..ff970ea 100644 --- a/apps/extension/src/tools/record.ts +++ b/apps/extension/src/tools/record.ts @@ -7,16 +7,28 @@ import { isRecordQueryMessage, isRecordStepMessage, RECORD_CANCEL, + RECORD_CAPTURE_STATUS, RECORD_START, RECORD_STEP, RECORD_STOP, type RecordCancelMessage, + type RecordCaptureStatusMessage, type RecordFinishMessage, type RecordQueryResponse, type RecordStartAck, type RecordStartMessage, + type RecordStepPayload, type RecordStopMessage, } from "@/lib/record-bridge"; +import { + armAllDocumentsForTab, + armDocumentCapture, + cancelDocumentCapture, + type ArmedDocument, + listInjectableFrames, + type RecordFrameSendDeps, + stopDocumentCapture, +} from "@/lib/record-frame-arming"; import { DEFAULT_MAX_PAGE_TOKENS } from "@/lib/record-constants"; import { applyTargetMatching, @@ -39,6 +51,9 @@ import type { SessionManager } from "@/session-manager/manager"; import { EXTENSION_VERSION } from "@/transport/handshake"; import type { DraftTraceStep, + FrameCaptureFailure, + FrameCaptureInfo, + FrameCaptureStatus, RecordAwaitParams, RecordAwaitResult, RecordStartParams, @@ -80,8 +95,12 @@ interface ActiveRecording { stoppedBy: StopReason; maxPageTokens: number; redactValues: boolean; - /** Tabs whose content capture acknowledged START and still need STOP. */ - armedTabIds: Set; + /** Documents whose capture acknowledged START and still need STOP. */ + armedDocuments: Map; + frameCaptureFailures: FrameCaptureFailure[]; + frameCaptureStatus: FrameCaptureStatus; + /** When false, ignore subframe rearm requests during finish. */ + acceptingNewFrames: boolean; /** Rearms already sending START when finish begins. */ rearmCallbacks: Set>; /** Navigation callbacks tracked from event receipt through action enqueue. */ @@ -203,6 +222,15 @@ async function sendRecordStartWithAck( } function buildTrace(recording: ActiveRecording): Trace { + const frameCapture: FrameCaptureInfo | undefined = + recording.frameCaptureFailures.length > 0 || recording.frameCaptureStatus === "partial" + ? { + status: recording.frameCaptureStatus, + ...(recording.frameCaptureFailures.length > 0 + ? { failures: recording.frameCaptureFailures } + : {}), + } + : undefined; return buildTraceV3({ obs: recording.observation, steps: recording.steps, @@ -211,9 +239,51 @@ function buildTrace(recording: ActiveRecording): Trace { startUrl: recording.startUrl, stoppedBy: recording.stoppedBy, bskVersion: BSK_TRACE_VERSION, + ...(frameCapture ? { frameCapture } : {}), }); } +function recordFrameFailure(recording: ActiveRecording, failure: FrameCaptureFailure): void { + recording.frameCaptureFailures.push(failure); + recording.frameCaptureStatus = "partial"; +} + +function enrichRecordedStep( + recording: ActiveRecording, + step: RecordStepPayload, +): RecordStepPayload { + const enriched: RecordStepPayload = { ...step }; + if (enriched.geometry && enriched.documentToken) { + const owner = recording.observation.documentTokenOwners.get(enriched.documentToken); + if (owner !== undefined) { + enriched.geometry = { ...enriched.geometry, ownerFrameBackendNodeId: owner }; + } + } + if (!enriched.page_url && recording.currentUrl) { + enriched.page_url = recording.currentUrl; + } + return enriched; +} + +async function publishCaptureStatus( + recording: ActiveRecording, + deps: RecordDeps, +): Promise { + if (!deps.broadcastCaptureStatus) return; + const msg: RecordCaptureStatusMessage = { + type: RECORD_CAPTURE_STATUS, + status: recording.frameCaptureStatus, + ...(recording.frameCaptureFailures.length > 0 + ? { failures: recording.frameCaptureFailures } + : {}), + }; + try { + await deps.broadcastCaptureStatus(recording.tabId, msg); + } catch { + // Overlay is best-effort. + } +} + async function processRecordedStep( recording: ActiveRecording, deps: RecordDeps, @@ -249,17 +319,67 @@ export interface RecordDeps { tabId: number, msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage, ): Promise; + sendToDocument?( + tabId: number, + documentId: string, + msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage, + ): Promise; + getAllFrames?(tabId: number): Promise< + Array<{ + frameId: number; + documentId?: string; + url?: string; + parentFrameId: number; + }> + >; + broadcastCaptureStatus?( + tabId: number, + msg: RecordCaptureStatusMessage, + ): Promise; bypassOverlay?: (tabId: number, enabled: boolean) => Promise; cdp?: CdpRunner; signal?: AbortSignal; } +function frameSendDeps(deps: RecordDeps): RecordFrameSendDeps { + return { + sendToDocument: (tabId, documentId, msg) => { + if (deps.sendToDocument) return deps.sendToDocument(tabId, documentId, msg); + return deps.sendToTab(tabId, msg); + }, + getAllFrames: async (tabId) => { + if (deps.getAllFrames) return deps.getAllFrames(tabId); + if (!chrome.webNavigation?.getAllFrames) return []; + const frames = await chrome.webNavigation.getAllFrames({ tabId }); + return (frames ?? []).map((frame) => ({ + frameId: frame.frameId, + documentId: frame.documentId, + url: frame.url, + parentFrameId: frame.parentFrameId, + })); + }, + }; +} + let defaultDeps: RecordDeps | null = null; function getDefaultDeps(): RecordDeps { if (!defaultDeps) { defaultDeps = { tabsApi: chromeTabsApi, sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg), + sendToDocument: (tabId, documentId, msg) => + chrome.tabs.sendMessage(tabId, msg, { documentId }), + getAllFrames: async (tabId) => { + if (!chrome.webNavigation?.getAllFrames) return []; + const frames = await chrome.webNavigation.getAllFrames({ tabId }); + return (frames ?? []).map((frame) => ({ + frameId: frame.frameId, + documentId: frame.documentId, + url: frame.url, + parentFrameId: frame.parentFrameId, + })); + }, + broadcastCaptureStatus: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg, { frameId: 0 }), }; } return defaultDeps; @@ -320,7 +440,7 @@ export function releaseBrowserObservationListenersIfIdle(): void { export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): () => void { const listener = ( message: unknown, - _sender: chrome.runtime.MessageSender, + sender: chrome.runtime.MessageSender, _sendResponse: () => void, ) => { if (!isRecordStepMessage(message)) return false; @@ -328,9 +448,6 @@ export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): ( if (recording.requestId !== message.requestId) continue; enqueueRecordingAction(recording, async () => { const cdp = recording.cdp ?? deps.cdp; - // Flush before appending so a coalesced OAuth navigate is ordered - // ahead of the click that landed on the destination page. The flush - // itself waits for that navigate's observation before returning. await flushPendingRedirectLanding( recording.observation, recording.steps, @@ -339,7 +456,8 @@ export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): ( deps.tabsApi, { cancelled: () => recording.settled }, ); - const draftIndex = appendRecordedPayload(recording, message.step); + const enriched = enrichRecordedStep(recording, message.step); + const draftIndex = appendRecordedPayload(recording, enriched); if (draftIndex !== null) await processRecordedStep(recording, deps, draftIndex); }); return false; @@ -368,8 +486,10 @@ export function attachRecordFinishListener(deps: RecordDeps = getDefaultDeps()): function findRecordingByTabId(tabId: number): ActiveRecording | null { for (const recording of recordings.values()) { - if (!recording.settled && (recording.tabId === tabId || recording.armedTabIds.has(tabId))) { - return recording; + if (recording.settled) continue; + if (recording.tabId === tabId) return recording; + for (const armed of recording.armedDocuments.values()) { + if (armed.tabId === tabId) return recording; } } return null; @@ -418,47 +538,64 @@ async function clearRearmTimersForRecording( } } -async function stopRecordingOnAllAgentTabs( +async function stopRecordingOnAllDocuments( recording: ActiveRecording, deps: RecordDeps, ): Promise { + recording.acceptingNewFrames = false; const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId }; - let failed = false; - for (const tabId of [...recording.armedTabIds]) { - try { - await deps.tabsApi.get(tabId); - } catch { - // A closed tab no longer has content capture to flush. - recording.armedTabIds.delete(tabId); + const frameDeps = frameSendDeps(deps); + const armed = [...recording.armedDocuments.values()]; + for (const doc of armed) { + const result = await stopDocumentCapture(doc, stopMsg, frameDeps); + if ("ok" in result) { + recording.armedDocuments.delete(doc.documentId); continue; } - try { - const response = await deps.sendToTab(tabId, stopMsg); - if (!isRecordStartAck(response)) { - throw new Error("content script did not confirm recorded steps"); - } - recording.armedTabIds.delete(tabId); - } catch { - failed = true; - continue; - } - if (deps.bypassOverlay) { - try { - await deps.bypassOverlay(tabId, false); - } catch { - // Best-effort cleanup. - } - } + recordFrameFailure(recording, result); } - if (failed || recording.armedTabIds.size > 0) { - throw new Error("failed to flush recorded steps"); + if (recording.armedDocuments.size > 0) { + for (const doc of [...recording.armedDocuments.values()]) { + recordFrameFailure( + recording, + { + reason: "flush_failed", + frame_id: doc.frameId, + document_id: doc.documentId, + url: doc.url, + detail: "document still armed after STOP", + }, + ); + } + recording.armedDocuments.clear(); + } + await publishCaptureStatus(recording, deps); + if (deps.bypassOverlay) { + try { + await deps.bypassOverlay(recording.tabId, false); + } catch { + // Best-effort cleanup. + } } } -async function rearmRecording( +async function cancelRecordingOnAllDocuments( + recording: ActiveRecording, + deps: RecordDeps, +): Promise { + const cancelMsg: RecordCancelMessage = { type: RECORD_CANCEL, requestId: recording.requestId }; + const frameDeps = frameSendDeps(deps); + for (const doc of [...recording.armedDocuments.values()]) { + await cancelDocumentCapture(doc, cancelMsg, frameDeps); + } + recording.armedDocuments.clear(); +} + +async function rearmRecordingDocuments( recording: ActiveRecording, targetTabId: number, deps: RecordDeps, + targetDocumentId?: string, ): Promise { let resolveTracked!: () => void; const tracked = new Promise((resolve) => { @@ -466,29 +603,65 @@ async function rearmRecording( }); recording.rearmCallbacks.add(tracked); - // Do NOT toggle automation-bypass here: each retry used to increment the - // content-script counter, and a single stop decrement left the ControlOverlay - // stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay - // already hides the control chrome while activeRecord is set. + const frameDeps = frameSendDeps(deps); try { - for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) { - if (recording.settled || recording.finishing) return false; - const startMsg: RecordStartMessage = { - type: RECORD_START, - requestId: recording.requestId, - startedAtMs: recording.startedAtMs, - }; - try { - await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab); - recording.armedTabIds.add(targetTabId); - recording.tabId = targetTabId; - return true; - } catch { - if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) { - await sleep(RECORD_REARM_RETRY_DELAY_MS); + const attemptLimit = targetDocumentId ? 1 : RECORD_REARM_MAX_ATTEMPTS; + for (let attempt = 0; attempt < attemptLimit; attempt += 1) { + if (recording.settled || recording.finishing || !recording.acceptingNewFrames) return false; + const { injectable, failures } = await listInjectableFrames(targetTabId, frameDeps); + for (const failure of failures) recordFrameFailure(recording, failure); + const targets = targetDocumentId + ? injectable.filter((frame) => frame.documentId === targetDocumentId) + : injectable.filter((frame) => !recording.armedDocuments.has(frame.documentId ?? "")); + if (targets.length === 0) { + if (targetDocumentId) { + recordFrameFailure(recording, { + reason: "rearm_failed", + frame_id: 0, + document_id: targetDocumentId, + detail: "document not injectable", + }); + await publishCaptureStatus(recording, deps); + return false; } + recording.tabId = targetTabId; + return injectable.some((frame) => recording.armedDocuments.has(frame.documentId ?? "")); + } + + let armedAny = false; + for (const frame of targets) { + const startMsg: RecordStartMessage = { + type: RECORD_START, + requestId: recording.requestId, + startedAtMs: recording.startedAtMs, + showOverlay: frame.frameId === 0, + frameMode: frame.frameId === 0 ? "top" : "child", + }; + const result = await armDocumentCapture(targetTabId, frame, startMsg, frameDeps); + if (result.armed) { + recording.armedDocuments.set(result.armed.documentId, result.armed); + armedAny = true; + } + if (result.failure) recordFrameFailure(recording, result.failure); + } + if (armedAny) { + recording.tabId = targetTabId; + await publishCaptureStatus(recording, deps); + if (deps.cdp && recording.acceptingNewFrames) { + void captureAndRegisterObservation( + recording.observation, + deps.cdp, + targetTabId, + deps.tabsApi, + ).catch(() => {}); + } + return true; + } + if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) { + await sleep(RECORD_REARM_RETRY_DELAY_MS); } } + await publishCaptureStatus(recording, deps); return false; } finally { recording.rearmCallbacks.delete(tracked); @@ -505,7 +678,7 @@ function scheduleRearmForTab(tabId: number, deps: RecordDeps): void { rearmTimers.delete(tabId); void (async () => { const current = await findRecordingForTab(tabId, deps); - if (current) await rearmRecording(current, tabId, deps); + if (current) await rearmRecordingDocuments(current, tabId, deps); })(); }, RECORD_REARM_DEBOUNCE_MS), ); @@ -626,12 +799,24 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps })(); }; + const onSubFrameComplete = (tabId: number, documentId?: string) => { + if (!documentId) return; + void (async () => { + const recording = await findRecordingForTab(tabId, deps); + if (!recording || !recording.acceptingNewFrames) return; + await rearmRecordingDocuments(recording, tabId, deps, documentId); + })(); + }; + if (chrome.webNavigation?.onCompleted) { const completedListener = ( details: chrome.webNavigation.WebNavigationFramedCallbackDetails, ) => { - if (details.frameId !== 0) return; - onMainFrameComplete(details.tabId, details.url); + if (details.frameId === 0) { + onMainFrameComplete(details.tabId, details.url); + return; + } + onSubFrameComplete(details.tabId, details.documentId); }; const committedListener = ( details: chrome.webNavigation.WebNavigationTransitionCallbackDetails, @@ -719,11 +904,15 @@ export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()): sendResponse({ active: false }); return; } - await rearmRecording(recording, tabId, deps); + await rearmRecordingDocuments(recording, tabId, deps, sender.documentId); sendResponse({ active: true, requestId: recording.requestId, startedAtMs: recording.startedAtMs, + captureStatus: recording.frameCaptureStatus, + ...(recording.frameCaptureFailures.length > 0 + ? { captureFailures: recording.frameCaptureFailures } + : {}), }); })(); return true; @@ -760,10 +949,7 @@ async function finishRecording( await clearRearmTimersForRecording(recording, deps); await Promise.all([...recording.rearmCallbacks]); try { - // Disposing content capture may commit a final dirty fill. Flush content - // first so all resulting RECORD_STEP messages enter actionQueue before it - // and the observation queues are drained. - await stopRecordingOnAllAgentTabs(recording, deps); + await stopRecordingOnAllDocuments(recording, deps); } catch { recording.finishing = false; return null; @@ -845,7 +1031,10 @@ export async function handleRecordStart( stoppedBy: "user_finish", maxPageTokens: maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS, redactValues, - armedTabIds: new Set(), + armedDocuments: new Map(), + frameCaptureFailures: [], + frameCaptureStatus: "complete", + acceptingNewFrames: true, rearmCallbacks: new Set(), navigationCallbacks: new Set(), acceptingNavigation: true, @@ -857,17 +1046,11 @@ export async function handleRecordStart( ensureBrowserObservationListeners(deps); const abortPending = async (notifyContent: boolean) => { + const pending = recordings.get(params.session_id); recordings.delete(params.session_id); releaseBrowserObservationListenersIfIdle(); - if (notifyContent) { - try { - await deps.sendToTab(target.tabId, { - type: RECORD_CANCEL, - requestId, - }); - } catch { - // Content may never have received RECORD_START. - } + if (notifyContent && pending) { + await cancelRecordingOnAllDocuments(pending, deps); } if (deps.bypassOverlay) { try { @@ -970,11 +1153,28 @@ export async function handleRecordStart( if (cancelled) return cancelled; } - const startMsg: RecordStartMessage = { type: RECORD_START, requestId, startedAtMs }; + const startMsg: RecordStartMessage = { + type: RECORD_START, + requestId, + startedAtMs, + showOverlay: true, + frameMode: "top", + }; try { - await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab); - active.armedTabIds.add(target.tabId); + const frameDeps = frameSendDeps(deps); + const { armed, failures } = await armAllDocumentsForTab(target.tabId, startMsg, frameDeps); + for (const failure of failures) recordFrameFailure(active, failure); + for (const doc of armed) active.armedDocuments.set(doc.documentId, doc); + if (active.armedDocuments.size === 0) { + await abortPending(true); + return { + code: "protocol_error", + message: + "failed to start recording in any frame — reload the BrowserSkill extension, then retry", + }; + } + await publishCaptureStatus(active, deps); } catch { await abortPending(true); return { diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index 009a199..77a55e3 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -681,6 +681,28 @@ export interface RecorderInfo { vom: number; } +export type FrameCaptureStatus = "complete" | "partial"; + +export type FrameCaptureFailureReason = + | "not_injectable" + | "arm_failed" + | "rearm_failed" + | "drain_failed" + | "flush_failed"; + +export interface FrameCaptureFailure { + reason: FrameCaptureFailureReason; + frame_id: number; + document_id?: string; + url?: string; + detail?: string; +} + +export interface FrameCaptureInfo { + status: FrameCaptureStatus; + failures?: FrameCaptureFailure[]; +} + export type StopReason = "user_finish" | "cli_stop"; export interface TraceState { @@ -864,6 +886,8 @@ export interface Trace { stopped_by: StopReason; entry: TraceEntry; recorder: RecorderInfo; + /** Present when one or more frames could not be armed, rearmed, or flushed. */ + frame_capture?: FrameCaptureInfo; states: TraceState[]; steps: Step[]; } diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index 8546a15..edcbb3e 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -285,6 +285,7 @@ bsk record stop [--output trace] # terminal fallback if the browser panel is u - `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead. - `--purpose` is optional context metadata; it does **not** change what gets captured. - `--redact-values` masks all form values in page files as `[filled]` / `[empty]`. +- Recording captures interactions in the **top frame and injectable child frames** (same-origin and cross-origin where the extension can attach). Embedded frames that cannot be injected are omitted; the trace may include `frame_capture.status: "partial"` with failure details. - Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text. ## Error handling diff --git a/crates/bsk-cli/src/cli/record.rs b/crates/bsk-cli/src/cli/record.rs index cb4dc56..c41944d 100644 --- a/crates/bsk-cli/src/cli/record.rs +++ b/crates/bsk-cli/src/cli/record.rs @@ -8,8 +8,8 @@ use std::time::Duration; use anyhow::Context; use bsk_protocol::tools::{ - RecordAwaitParams, RecordAwaitResult, RecordStartParams, RecordStartResult, RecordStopParams, - RecordStopResult, TRACE_VERSION, Trace, TraceState, + FrameCaptureStatus, RecordAwaitParams, RecordAwaitResult, RecordStartParams, RecordStartResult, + RecordStopParams, RecordStopResult, TRACE_VERSION, Trace, TraceState, }; use bsk_protocol::{ErrorCode, Method}; use clap::{Args, Subcommand}; @@ -550,6 +550,13 @@ fn render_finish( trace_path.display(), pages_dir.display() ); + if let Some(frame_capture) = &trace.frame_capture { + if matches!(frame_capture.status, FrameCaptureStatus::Partial) { + eprintln!( + "warning: trace is partial — some embedded frames were not recorded (see frame_capture.failures in trace.json)" + ); + } + } } } Ok(()) @@ -584,6 +591,7 @@ mod tests { bsk: "0.1.10".into(), vom: VOM_FORMAT_VERSION, }, + frame_capture: None, states: vec![TraceState { id: state_id.into(), url: "https://example.com/".into(), diff --git a/crates/bsk-cli/tests/record_stop_retry.rs b/crates/bsk-cli/tests/record_stop_retry.rs index 4c5e6af..a0f2de7 100644 --- a/crates/bsk-cli/tests/record_stop_retry.rs +++ b/crates/bsk-cli/tests/record_stop_retry.rs @@ -96,6 +96,7 @@ fn sample_trace() -> Trace { bsk: "0.1.10".into(), vom: VOM_FORMAT_VERSION, }, + frame_capture: None, states: vec![TraceState { id: "s1".into(), url: "https://example.com/".into(), diff --git a/crates/bsk-protocol/schema/trace.json b/crates/bsk-protocol/schema/trace.json index f17a028..ca7b649 100644 --- a/crates/bsk-protocol/schema/trace.json +++ b/crates/bsk-protocol/schema/trace.json @@ -16,6 +16,16 @@ "entry": { "$ref": "#/definitions/TraceEntry" }, + "frame_capture": { + "anyOf": [ + { + "$ref": "#/definitions/FrameCaptureInfo" + }, + { + "type": "null" + } + ] + }, "purpose": { "type": [ "string", @@ -64,6 +74,77 @@ "blur" ] }, + "FrameCaptureFailure": { + "type": "object", + "required": [ + "frame_id", + "reason" + ], + "properties": { + "detail": { + "type": [ + "string", + "null" + ] + }, + "document_id": { + "type": [ + "string", + "null" + ] + }, + "frame_id": { + "type": "integer", + "format": "int64" + }, + "reason": { + "$ref": "#/definitions/FrameCaptureFailureReason" + }, + "url": { + "type": [ + "string", + "null" + ] + } + } + }, + "FrameCaptureFailureReason": { + "type": "string", + "enum": [ + "not_injectable", + "arm_failed", + "rearm_failed", + "drain_failed", + "flush_failed" + ] + }, + "FrameCaptureInfo": { + "type": "object", + "required": [ + "status" + ], + "properties": { + "failures": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/FrameCaptureFailure" + } + }, + "status": { + "$ref": "#/definitions/FrameCaptureStatus" + } + } + }, + "FrameCaptureStatus": { + "type": "string", + "enum": [ + "complete", + "partial" + ] + }, "KeyModifier": { "description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).", "type": "string", diff --git a/crates/bsk-protocol/src/tools/record.rs b/crates/bsk-protocol/src/tools/record.rs index 5598a51..86f819e 100644 --- a/crates/bsk-protocol/src/tools/record.rs +++ b/crates/bsk-protocol/src/tools/record.rs @@ -167,6 +167,46 @@ pub enum Step { }, } +// --------------------------------------------------------------------------- +// Frame capture metadata +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum FrameCaptureStatus { + Complete, + Partial, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "snake_case")] +pub enum FrameCaptureFailureReason { + NotInjectable, + ArmFailed, + RearmFailed, + DrainFailed, + FlushFailed, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct FrameCaptureFailure { + pub reason: FrameCaptureFailureReason, + pub frame_id: i64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub document_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub url: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detail: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct FrameCaptureInfo { + pub status: FrameCaptureStatus, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub failures: Option>, +} + // --------------------------------------------------------------------------- // Trace root // --------------------------------------------------------------------------- @@ -183,6 +223,8 @@ pub struct Trace { pub stopped_by: StopReason, pub entry: TraceEntry, pub recorder: RecorderInfo, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub frame_capture: Option, pub states: Vec, pub steps: Vec, } @@ -277,6 +319,7 @@ mod tests { bsk: "0.1.10".into(), vom: VOM_FORMAT_VERSION, }, + frame_capture: None, states: vec![ TraceState { id: "s1".into(), @@ -479,4 +522,39 @@ mod tests { assert_eq!(trace.states.len(), 2); assert_eq!(trace.steps.len(), 2); } + + #[test] + fn trace_deserializes_optional_frame_capture_metadata() { + let v = json!({ + "version": 3, + "recorded_at": "2026-07-21T08:00:00Z", + "stopped_by": "cli_stop", + "entry": { "start_url": "https://example.com/editor" }, + "recorder": { "bsk": "0.1.10", "vom": 1 }, + "frame_capture": { + "status": "partial", + "failures": [ + { + "reason": "not_injectable", + "frame_id": 2, + "url": "chrome://settings", + "detail": "restricted frame url" + } + ] + }, + "states": [], + "steps": [] + }); + let trace: Trace = serde_json::from_value(v).unwrap(); + let frame_capture = trace.frame_capture.expect("frame_capture"); + assert_eq!(frame_capture.status, FrameCaptureStatus::Partial); + let failures = frame_capture.failures.expect("failures"); + assert_eq!(failures.len(), 1); + assert_eq!( + failures[0].reason, + FrameCaptureFailureReason::NotInjectable + ); + assert_eq!(failures[0].frame_id, 2); + assert_eq!(failures[0].url.as_deref(), Some("chrome://settings")); + } } diff --git a/packages/i18n/src/locales/en-US/extension.json b/packages/i18n/src/locales/en-US/extension.json index 5d73423..9a3acb1 100644 --- a/packages/i18n/src/locales/en-US/extension.json +++ b/packages/i18n/src/locales/en-US/extension.json @@ -90,6 +90,7 @@ }, "recordOverlay": { "recording": "Recording your actions", - "finish": "Finish" + "finish": "Finish", + "partialCapture": "Some embedded frames may not be recorded." } } diff --git a/packages/i18n/src/locales/zh-CN/extension.json b/packages/i18n/src/locales/zh-CN/extension.json index 8a5b9f9..fd1fec6 100644 --- a/packages/i18n/src/locales/zh-CN/extension.json +++ b/packages/i18n/src/locales/zh-CN/extension.json @@ -90,6 +90,7 @@ }, "recordOverlay": { "recording": "正在录制你的操作", - "finish": "结束" + "finish": "结束", + "partialCapture": "部分嵌入页面可能未被录制。" } } diff --git a/skill/SKILL.md b/skill/SKILL.md index 8546a15..edcbb3e 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -285,6 +285,7 @@ bsk record stop [--output trace] # terminal fallback if the browser panel is u - `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead. - `--purpose` is optional context metadata; it does **not** change what gets captured. - `--redact-values` masks all form values in page files as `[filled]` / `[empty]`. +- Recording captures interactions in the **top frame and injectable child frames** (same-origin and cross-origin where the extension can attach). Embedded frames that cannot be injected are omitted; the trace may include `frame_capture.status: "partial"` with failure details. - Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text. ## Error handling