mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(record): capture interactions in embedded frames
Track recording per document so injectable child frames are captured while partial frame failures remain visible in exported traces. Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
“haonan”
@@ -17,3 +17,7 @@ dist/
|
||||
# Environment
|
||||
.env.local
|
||||
.env.*.local
|
||||
.pnpm-store/
|
||||
|
||||
# Local recording output
|
||||
/trace/
|
||||
@@ -1,4 +1,5 @@
|
||||
import { useTranslation } from "@browser-skill/i18n/react";
|
||||
import type { FrameCaptureFailure, FrameCaptureStatus } from "@/transport/types";
|
||||
import {
|
||||
type PointerEvent as ReactPointerEvent,
|
||||
useCallback,
|
||||
@@ -21,6 +22,8 @@ export interface RecordRequestData {
|
||||
|
||||
type Props = {
|
||||
request: RecordRequestData | null;
|
||||
captureStatus?: FrameCaptureStatus;
|
||||
captureFailures?: FrameCaptureFailure[];
|
||||
};
|
||||
|
||||
const VIEWPORT_MARGIN = 16;
|
||||
@@ -52,7 +55,11 @@ function clampDragPos(
|
||||
};
|
||||
}
|
||||
|
||||
export function RecordOverlay({ request }: Props) {
|
||||
export function RecordOverlay({
|
||||
request,
|
||||
captureStatus = "complete",
|
||||
captureFailures = [],
|
||||
}: Props) {
|
||||
const { t } = useTranslation("extension");
|
||||
const [show, setShow] = useState(false);
|
||||
const [elapsedSeconds, setElapsedSeconds] = useState(0);
|
||||
@@ -154,6 +161,7 @@ export function RecordOverlay({ request }: Props) {
|
||||
|
||||
const introVisible = elapsedSeconds * 1_000 < INTRO_DURATION_MS;
|
||||
const positioned = dragPos !== null;
|
||||
const partialCapture = captureStatus === "partial" || captureFailures.length > 0;
|
||||
const defaultTransform = show
|
||||
? "translateX(-50%) translateY(0)"
|
||||
: "translateX(-50%) translateY(8px)";
|
||||
@@ -208,8 +216,9 @@ export function RecordOverlay({ request }: Props) {
|
||||
data-slot="record-overlay-status"
|
||||
style={{
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 8,
|
||||
flexDirection: "column",
|
||||
alignItems: "flex-start",
|
||||
gap: 2,
|
||||
minWidth: 0,
|
||||
flex: 1,
|
||||
borderRadius: 9999,
|
||||
@@ -218,41 +227,57 @@ export function RecordOverlay({ request }: Props) {
|
||||
paddingRight: 4,
|
||||
}}
|
||||
>
|
||||
<span
|
||||
data-slot="record-overlay-indicator"
|
||||
style={{
|
||||
width: 9,
|
||||
height: 9,
|
||||
borderRadius: "50%",
|
||||
backgroundColor: "#ef4444",
|
||||
flexShrink: 0,
|
||||
animation: "bsk-rec-pulse 1.4s ease-in-out infinite",
|
||||
}}
|
||||
/>
|
||||
<span
|
||||
data-slot="record-overlay-label"
|
||||
style={{
|
||||
display: "block",
|
||||
flex: "0 1 auto",
|
||||
fontSize: 16,
|
||||
fontWeight: 500,
|
||||
color: "#333",
|
||||
whiteSpace: "nowrap",
|
||||
userSelect: "none",
|
||||
overflow: "hidden",
|
||||
textOverflow: "ellipsis",
|
||||
width: introVisible ? "auto" : 48,
|
||||
maxWidth: introVisible ? 280 : 48,
|
||||
fontVariantNumeric: introVisible ? undefined : "tabular-nums",
|
||||
textAlign: introVisible ? "left" : "center",
|
||||
margin: 0,
|
||||
padding: 0,
|
||||
minWidth: 0,
|
||||
transition: "max-width 180ms ease-out",
|
||||
}}
|
||||
>
|
||||
{introVisible ? t("recordOverlay.recording") : formatElapsed(elapsedSeconds)}
|
||||
</span>
|
||||
<div style={{ display: "flex", alignItems: "center", gap: 8, minWidth: 0, width: "100%" }}>
|
||||
<span
|
||||
data-slot="record-overlay-indicator"
|
||||
style={{
|
||||
width: 9,
|
||||
height: 9,
|
||||
borderRadius: "50%",
|
||||
backgroundColor: "#ef4444",
|
||||
flexShrink: 0,
|
||||
animation: "bsk-rec-pulse 1.4s ease-in-out infinite",
|
||||
}}
|
||||
/>
|
||||
<span
|
||||
data-slot="record-overlay-label"
|
||||
style={{
|
||||
display: "block",
|
||||
flex: "0 1 auto",
|
||||
fontSize: 16,
|
||||
fontWeight: 500,
|
||||
color: "#333",
|
||||
whiteSpace: "nowrap",
|
||||
userSelect: "none",
|
||||
overflow: "hidden",
|
||||
textOverflow: "ellipsis",
|
||||
width: introVisible ? "auto" : 48,
|
||||
maxWidth: introVisible ? 280 : 48,
|
||||
fontVariantNumeric: introVisible ? undefined : "tabular-nums",
|
||||
textAlign: introVisible ? "left" : "center",
|
||||
margin: 0,
|
||||
padding: 0,
|
||||
minWidth: 0,
|
||||
transition: "max-width 180ms ease-out",
|
||||
}}
|
||||
>
|
||||
{introVisible ? t("recordOverlay.recording") : formatElapsed(elapsedSeconds)}
|
||||
</span>
|
||||
</div>
|
||||
{partialCapture ? (
|
||||
<span
|
||||
data-slot="record-overlay-partial-warning"
|
||||
style={{
|
||||
fontSize: 12,
|
||||
lineHeight: "16px",
|
||||
color: "#b45309",
|
||||
whiteSpace: "normal",
|
||||
maxWidth: 280,
|
||||
}}
|
||||
>
|
||||
{t("recordOverlay.partialCapture")}
|
||||
</span>
|
||||
) : null}
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
|
||||
@@ -178,4 +178,30 @@ describe("RecordOverlay", () => {
|
||||
expect(pill.style.bottom).toBe("auto");
|
||||
expect(pill.style.transform).toBe("none");
|
||||
});
|
||||
|
||||
it("shows a persistent partial-capture warning when frame capture is incomplete", () => {
|
||||
const { container } = render(
|
||||
createElement(
|
||||
I18nextProvider,
|
||||
{ i18n },
|
||||
createElement(RecordOverlay, {
|
||||
request: { id: "rec-1", onFinish: vi.fn() },
|
||||
captureStatus: "partial",
|
||||
captureFailures: [
|
||||
{
|
||||
reason: "not_injectable",
|
||||
frame_id: 2,
|
||||
url: "chrome://settings",
|
||||
},
|
||||
],
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
const warning = container.querySelector("[data-slot='record-overlay-partial-warning']");
|
||||
expect(warning).toBeTruthy();
|
||||
expect(warning?.textContent).toBe(
|
||||
i18n.t("recordOverlay.partialCapture", { ns: "extension" }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1614,3 +1614,79 @@ describe("record-capture semantic", () => {
|
||||
expect(steps.map((s) => s.op)).toEqual(["click"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("record-capture child frame", () => {
|
||||
let steps: RecordStepPayload[];
|
||||
|
||||
beforeEach(() => {
|
||||
steps = [];
|
||||
document.body.innerHTML = `
|
||||
<label for="child-q">Child query</label>
|
||||
<input id="child-q" name="child-q" />
|
||||
<button type="button" aria-label="Child search">Search</button>
|
||||
<select aria-label="Child status">
|
||||
<option value="on">On</option>
|
||||
<option value="off">Off</option>
|
||||
</select>
|
||||
`;
|
||||
});
|
||||
|
||||
it("stamps a document token on start and clears it on dispose", () => {
|
||||
const capture = startRecordCapture("rec-child-token", (step) => steps.push(step), {
|
||||
frameMode: "child",
|
||||
});
|
||||
const token = document.documentElement.getAttribute("data-bsk-record-document");
|
||||
expect(token).toBeTruthy();
|
||||
click(document.querySelector("button")!);
|
||||
expect(steps[0]?.documentToken).toBe(token);
|
||||
expect(steps[0]?.frame_url).toBeTruthy();
|
||||
capture.dispose();
|
||||
expect(document.documentElement.hasAttribute("data-bsk-record-document")).toBe(false);
|
||||
});
|
||||
|
||||
it("records click, fill, select, and press in child frame mode", () => {
|
||||
const capture = startRecordCapture("rec-child-ops", (step) => steps.push(step), {
|
||||
frameMode: "child",
|
||||
});
|
||||
const input = document.querySelector("input")!;
|
||||
const select = document.querySelector("select")!;
|
||||
|
||||
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||
input.value = "child value";
|
||||
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
input.dispatchEvent(new FocusEvent("focusout", { bubbles: true }));
|
||||
select.value = "off";
|
||||
select.dispatchEvent(new Event("change", { bubbles: true }));
|
||||
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||
input.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", bubbles: true }));
|
||||
click(document.querySelector("button")!);
|
||||
capture.dispose();
|
||||
|
||||
expect(steps.map((s) => s.op)).toEqual(
|
||||
expect.arrayContaining(["fill", "select", "press", "click"]),
|
||||
);
|
||||
expect(steps.every((s) => s.documentToken)).toBe(true);
|
||||
});
|
||||
|
||||
it("commits the last fill when dispose runs in child frame mode", () => {
|
||||
const capture = startRecordCapture("rec-child-fill-stop", (step) => steps.push(step), {
|
||||
frameMode: "child",
|
||||
});
|
||||
const input = document.querySelector("input")!;
|
||||
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||
input.value = "pending child fill";
|
||||
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
capture.dispose();
|
||||
|
||||
expect(steps.some((s) => s.op === "fill" && s.value === "pending child fill")).toBe(true);
|
||||
});
|
||||
|
||||
it("does not emit navigate steps for child frame navigations", () => {
|
||||
const capture = startRecordCapture("rec-child-nav", (step) => steps.push(step), {
|
||||
frameMode: "child",
|
||||
});
|
||||
history.pushState({}, "", "/child-frame-path");
|
||||
capture.dispose();
|
||||
expect(steps.filter((s) => s.op === "navigate")).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -21,12 +21,18 @@ import {
|
||||
RECORD_STEP,
|
||||
RECORD_STOP,
|
||||
type RecordCancelMessage,
|
||||
type RecordFrameMode,
|
||||
type RecordStartAck,
|
||||
type RecordStartMessage,
|
||||
type RecordStepPayload,
|
||||
type RecordStopAck,
|
||||
type RecordStopMessage,
|
||||
} from "@/lib/record-bridge";
|
||||
import {
|
||||
clearRecordDocumentToken,
|
||||
createRecordDocumentToken,
|
||||
stampRecordDocumentToken,
|
||||
} from "@/lib/record-document-token";
|
||||
import { shouldRecordPress } from "@/lib/trace-reducer";
|
||||
import {
|
||||
closestHoverSurfaceCandidate,
|
||||
@@ -402,12 +408,37 @@ function scheduleInputCompletionCommit(
|
||||
setTimeout(syncAndCommit, 0);
|
||||
}
|
||||
|
||||
export interface RecordCaptureOptions {
|
||||
frameMode?: RecordFrameMode;
|
||||
}
|
||||
|
||||
export function startRecordCapture(
|
||||
_requestId: string,
|
||||
sendStep: (step: RecordStepPayload) => void,
|
||||
options: RecordCaptureOptions = {},
|
||||
): RecordCaptureController {
|
||||
const frameMode = options.frameMode ?? "top";
|
||||
const emitNavigate = frameMode === "top";
|
||||
const documentToken = createRecordDocumentToken();
|
||||
stampRecordDocumentToken(documentToken);
|
||||
const capturedAtMs = () => Date.now();
|
||||
|
||||
const resolveTopPageUrl = (): string => {
|
||||
try {
|
||||
return window.top?.location.href ?? location.href;
|
||||
} catch {
|
||||
return location.href;
|
||||
}
|
||||
};
|
||||
|
||||
const emitStep = (step: RecordStepPayload) => {
|
||||
sendStep({ page_url: location.href, ...step });
|
||||
sendStep({
|
||||
page_url: resolveTopPageUrl(),
|
||||
frame_url: location.href,
|
||||
documentToken,
|
||||
capturedAtMs: capturedAtMs(),
|
||||
...step,
|
||||
});
|
||||
};
|
||||
const hoverSurfaceStateMap = (states: HoverSurfaceState[]): Map<Element, string> =>
|
||||
new Map(states.map((state) => [state.element, state.signature]));
|
||||
@@ -478,6 +509,7 @@ export function startRecordCapture(
|
||||
};
|
||||
|
||||
const emitNavigateIfChanged = (causedByAction?: boolean) => {
|
||||
if (!emitNavigate) return;
|
||||
if (location.href === lastUrl) return;
|
||||
commitFillSession();
|
||||
lastUrl = location.href;
|
||||
@@ -965,6 +997,7 @@ export function startRecordCapture(
|
||||
return {
|
||||
dispose() {
|
||||
commitFillSession();
|
||||
clearRecordDocumentToken();
|
||||
document.removeEventListener("click", onClick, true);
|
||||
document.removeEventListener("mouseover", onMouseOver, true);
|
||||
document.removeEventListener("focusin", onFocusIn, true);
|
||||
@@ -997,8 +1030,8 @@ export function handleRecordContentMessage(
|
||||
capture: RecordCaptureController | null;
|
||||
setActiveRequestId(id: string | null): void;
|
||||
setCapture(capture: RecordCaptureController | null): void;
|
||||
onStart(requestId: string, startedAtMs?: number): void;
|
||||
onStop(): void;
|
||||
onStart?(requestId: string, startedAtMs?: number): void;
|
||||
onStop?(): void;
|
||||
},
|
||||
sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
|
||||
): boolean {
|
||||
@@ -1009,13 +1042,20 @@ export function handleRecordContentMessage(
|
||||
failedStepPayloads.delete(message.requestId);
|
||||
}
|
||||
state.capture?.dispose();
|
||||
const frameMode = message.frameMode ?? "top";
|
||||
state.setCapture(
|
||||
startRecordCapture(message.requestId, (step) => {
|
||||
sendRecordStep(message.requestId, step);
|
||||
}),
|
||||
startRecordCapture(
|
||||
message.requestId,
|
||||
(step) => {
|
||||
sendRecordStep(message.requestId, step);
|
||||
},
|
||||
{ frameMode },
|
||||
),
|
||||
);
|
||||
state.setActiveRequestId(message.requestId);
|
||||
state.onStart(message.requestId, message.startedAtMs);
|
||||
if (message.showOverlay !== false && frameMode === "top") {
|
||||
state.onStart?.(message.requestId, message.startedAtMs);
|
||||
}
|
||||
sendResponse?.({ ok: true });
|
||||
return sendResponse !== undefined;
|
||||
}
|
||||
@@ -1030,7 +1070,7 @@ export function handleRecordContentMessage(
|
||||
state.capture?.dispose();
|
||||
state.setCapture(null);
|
||||
const finishStop = () => {
|
||||
state.onStop();
|
||||
state.onStop?.();
|
||||
state.setActiveRequestId(null);
|
||||
};
|
||||
if (isRecordStopMessage(message) && sendResponse) {
|
||||
|
||||
@@ -33,6 +33,9 @@ import {
|
||||
defaultBorrowChromeWindows,
|
||||
requestBorrowConfirmation,
|
||||
} from "@/tools/borrow-confirmation";
|
||||
import {
|
||||
type RecordCaptureStatusMessage,
|
||||
} from "@/lib/record-bridge";
|
||||
import { ToolDispatcher } from "@/tools/dispatcher";
|
||||
import {
|
||||
attachRecordFinishListener,
|
||||
@@ -175,6 +178,24 @@ export default defineBackground(() => {
|
||||
cdp,
|
||||
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
|
||||
chrome.tabs.sendMessage(tabId, msg),
|
||||
sendToDocument: (
|
||||
tabId: number,
|
||||
documentId: string,
|
||||
msg: Parameters<typeof chrome.tabs.sendMessage>[1],
|
||||
) => chrome.tabs.sendMessage(tabId, msg, { documentId }),
|
||||
getAllFrames: async (tabId: number) => {
|
||||
if (!chrome.webNavigation?.getAllFrames) return [];
|
||||
const frames = await chrome.webNavigation.getAllFrames({ tabId });
|
||||
return (frames ?? []).map((frame) => ({
|
||||
frameId: frame.frameId,
|
||||
documentId: frame.documentId,
|
||||
url: frame.url,
|
||||
parentFrameId: frame.parentFrameId,
|
||||
}));
|
||||
},
|
||||
broadcastCaptureStatus: async (tabId: number, msg: RecordCaptureStatusMessage) => {
|
||||
await chrome.tabs.sendMessage(tabId, msg, { frameId: 0 });
|
||||
},
|
||||
bypassOverlay: async (tabId: number, enabled: boolean) => {
|
||||
try {
|
||||
await chrome.tabs.sendMessage(tabId, {
|
||||
|
||||
@@ -44,12 +44,15 @@ import {
|
||||
} from "@/lib/overlay-bridge";
|
||||
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
|
||||
import {
|
||||
isRecordCaptureStatusMessage,
|
||||
RECORD_FINISH,
|
||||
RECORD_QUERY,
|
||||
type RecordCaptureStatusMessage,
|
||||
type RecordQueryResponse,
|
||||
type RecordStartAck,
|
||||
type RecordStopAck,
|
||||
} from "@/lib/record-bridge";
|
||||
import type { FrameCaptureFailure, FrameCaptureStatus } from "@/transport/types";
|
||||
import type {
|
||||
BorrowCancelMessage,
|
||||
BorrowRequestMessage,
|
||||
@@ -70,6 +73,8 @@ export default defineContentScript({
|
||||
const overlays = new OverlayController();
|
||||
let recordCapture: RecordCaptureController | null = null;
|
||||
let activeRecordRequestId: string | null = null;
|
||||
let recordCaptureStatus: FrameCaptureStatus = "complete";
|
||||
let recordCaptureFailures: FrameCaptureFailure[] = [];
|
||||
let reactRoot: ReactDOM.Root | null = null;
|
||||
let overlayHost: HTMLElement | null = null;
|
||||
let overlayContainer: HTMLElement | null = null;
|
||||
@@ -188,7 +193,11 @@ export default defineContentScript({
|
||||
requests: overlayState.borrowRequests,
|
||||
}),
|
||||
React.createElement(HelpRequestOverlay, { request: overlayState.activeHelp }),
|
||||
React.createElement(RecordOverlay, { request: overlayState.activeRecord }),
|
||||
React.createElement(RecordOverlay, {
|
||||
request: overlayState.activeRecord,
|
||||
captureStatus: recordCaptureStatus,
|
||||
captureFailures: recordCaptureFailures,
|
||||
}),
|
||||
React.createElement(ControlOverlay, {
|
||||
visible: controlOverlayVisible,
|
||||
interrupting: overlayState.interrupting,
|
||||
@@ -260,7 +269,8 @@ export default defineContentScript({
|
||||
| CaptureSuppressMessage
|
||||
| OverlayAgentOverlayResetMessage
|
||||
| OverlayAgentStateMessage
|
||||
| OverlayAutomationBypassMessage,
|
||||
| OverlayAutomationBypassMessage
|
||||
| RecordCaptureStatusMessage,
|
||||
_sender: chrome.runtime.MessageSender,
|
||||
sendResponse: (response: BorrowResponseMessage | HelpAckMessage | CaptureSuppressAck) => void,
|
||||
) => {
|
||||
@@ -268,6 +278,13 @@ export default defineContentScript({
|
||||
return captureSuppress.handleMessage(message, sendResponse);
|
||||
}
|
||||
|
||||
if (isRecordCaptureStatusMessage(message)) {
|
||||
recordCaptureStatus = message.status;
|
||||
recordCaptureFailures = message.failures ?? [];
|
||||
renderAll();
|
||||
return false;
|
||||
}
|
||||
|
||||
if (isRecordContentMessage(message)) {
|
||||
const needsAsync = handleRecordContentMessage(
|
||||
message,
|
||||
@@ -295,6 +312,8 @@ export default defineContentScript({
|
||||
},
|
||||
onStop: () => {
|
||||
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
|
||||
recordCaptureStatus = "complete";
|
||||
recordCaptureFailures = [];
|
||||
renderAll();
|
||||
},
|
||||
},
|
||||
@@ -449,6 +468,8 @@ export default defineContentScript({
|
||||
) {
|
||||
const requestId = recordQuery.requestId;
|
||||
const startedAtMs = recordQuery.startedAtMs;
|
||||
recordCaptureStatus = recordQuery.captureStatus ?? "complete";
|
||||
recordCaptureFailures = recordQuery.captureFailures ?? [];
|
||||
overlays.setAgentRecordRequest({
|
||||
id: requestId,
|
||||
...(typeof startedAtMs === "number" ? { startedAtMs } : {}),
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import {
|
||||
handleRecordContentMessage,
|
||||
isRecordContentMessage,
|
||||
type RecordCaptureController,
|
||||
} from "@/content/record-capture";
|
||||
import {
|
||||
RECORD_QUERY,
|
||||
type RecordQueryResponse,
|
||||
} from "@/lib/record-bridge";
|
||||
|
||||
/** Capture-only content script for child frames (no React overlay). */
|
||||
export default defineContentScript({
|
||||
matches: ["<all_urls>"],
|
||||
runAt: "document_end",
|
||||
allFrames: true,
|
||||
matchAboutBlank: true,
|
||||
matchOriginAsFallback: true,
|
||||
|
||||
main(ctx) {
|
||||
if (window.top === window) return;
|
||||
|
||||
let recordCapture: RecordCaptureController | null = null;
|
||||
let activeRecordRequestId: string | null = null;
|
||||
|
||||
const onMessage = (
|
||||
message: unknown,
|
||||
_sender: chrome.runtime.MessageSender,
|
||||
sendResponse: (response: unknown) => void,
|
||||
) => {
|
||||
if (isRecordContentMessage(message)) {
|
||||
const needsAsync = handleRecordContentMessage(
|
||||
message,
|
||||
{
|
||||
activeRequestId: activeRecordRequestId,
|
||||
capture: recordCapture,
|
||||
setActiveRequestId: (id) => {
|
||||
activeRecordRequestId = id;
|
||||
},
|
||||
setCapture: (capture) => {
|
||||
recordCapture = capture;
|
||||
},
|
||||
},
|
||||
sendResponse as never,
|
||||
);
|
||||
return needsAsync;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
async function queryActiveRecord(): Promise<void> {
|
||||
try {
|
||||
const recordQuery = (await chrome.runtime.sendMessage({
|
||||
type: RECORD_QUERY,
|
||||
})) as RecordQueryResponse | undefined;
|
||||
if (
|
||||
recordQuery?.active &&
|
||||
typeof recordQuery.requestId === "string" &&
|
||||
activeRecordRequestId === null
|
||||
) {
|
||||
// Background rearm sends RECORD_START; nothing to mount locally.
|
||||
}
|
||||
} catch (err) {
|
||||
console.debug("[bsk record-frame] record query failed", err);
|
||||
}
|
||||
}
|
||||
|
||||
chrome.runtime.onMessage.addListener(onMessage);
|
||||
void queryActiveRecord();
|
||||
|
||||
ctx.onInvalidated(() => {
|
||||
chrome.runtime.onMessage.removeListener(onMessage);
|
||||
recordCapture?.dispose();
|
||||
recordCapture = null;
|
||||
activeRecordRequestId = null;
|
||||
});
|
||||
},
|
||||
});
|
||||
@@ -113,4 +113,60 @@ describe("matchTarget", () => {
|
||||
});
|
||||
expect(target.ref).toBe("e1");
|
||||
});
|
||||
|
||||
it("isolates geometry matching to the owner iframe backend node", () => {
|
||||
const sharedRect = { x: 10, y: 20, w: 100, h: 30 };
|
||||
const iframeRefs: RenderedRef[] = [
|
||||
{ ref: "e1", backendNodeId: 42, role: "button", name: "Top", line: 1 },
|
||||
{ ref: "e2", backendNodeId: 99, role: "button", name: "Iframe", line: 2 },
|
||||
];
|
||||
const target = matchTarget({
|
||||
geometry: {
|
||||
rect: sharedRect,
|
||||
scrollX: 0,
|
||||
scrollY: 0,
|
||||
position: "static",
|
||||
tag: "button",
|
||||
ownerFrameBackendNodeId: 13,
|
||||
},
|
||||
captured: [
|
||||
node(42, { rect: sharedRect, documentRect: sharedRect }),
|
||||
node(99, {
|
||||
rect: sharedRect,
|
||||
documentRect: sharedRect,
|
||||
ownerFrameBackendNodeId: 13,
|
||||
}),
|
||||
],
|
||||
refs: iframeRefs,
|
||||
fallback: { tag: "button", name: "Iframe" },
|
||||
});
|
||||
expect(target).toEqual({ ref: "e2", role: "button", name: "Iframe" });
|
||||
});
|
||||
|
||||
it("matches iframe static elements using frame scroll and document coordinates", () => {
|
||||
const iframeRefs: RenderedRef[] = [
|
||||
{ ref: "e2", backendNodeId: 99, role: "button", name: "Iframe", line: 2 },
|
||||
];
|
||||
const target = matchTarget({
|
||||
geometry: {
|
||||
rect: { x: 10, y: 50, w: 100, h: 30 },
|
||||
scrollX: 0,
|
||||
scrollY: 120,
|
||||
position: "static",
|
||||
tag: "button",
|
||||
ownerFrameBackendNodeId: 13,
|
||||
},
|
||||
captured: [
|
||||
node(99, {
|
||||
ownerFrameBackendNodeId: 13,
|
||||
documentRect: { x: 10, y: 170, w: 100, h: 30 },
|
||||
localRect: { x: 10, y: 50, w: 100, h: 30 },
|
||||
rect: { x: 10, y: 50, w: 100, h: 30 },
|
||||
}),
|
||||
],
|
||||
refs: iframeRefs,
|
||||
fallback: { tag: "button", name: "Iframe" },
|
||||
});
|
||||
expect(target).toEqual({ ref: "e2", role: "button", name: "Iframe" });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -33,6 +33,8 @@ function finalizedFillBody(value: string, redactValues: boolean): string {
|
||||
obs.lastSettled = {
|
||||
stateId,
|
||||
captured: [capturedInput()],
|
||||
matchNodes: [capturedInput()],
|
||||
documentTokenOwners: new Map(),
|
||||
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
|
||||
url: URL,
|
||||
vomText: rawVomText,
|
||||
|
||||
@@ -40,9 +40,8 @@ function matchingRects(
|
||||
target: { x: number; y: number; w: number; h: number };
|
||||
node: { x: number; y: number; w: number; h: number } | null;
|
||||
} {
|
||||
const topFrame = (geometry.ownerFrameBackendNodeId ?? null) === null;
|
||||
const viewportPosition = geometry.position === "fixed" || geometry.position === "sticky";
|
||||
if (!topFrame || viewportPosition || !node.documentRect) {
|
||||
if (viewportPosition || !node.documentRect) {
|
||||
return { target: geometry.rect, node: nodeViewportRect(node) };
|
||||
}
|
||||
return {
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
*/
|
||||
|
||||
import type { CaptureTargetDescriptor } from "./describe-target";
|
||||
import type { FrameCaptureFailure, FrameCaptureStatus } from "@/transport/types";
|
||||
|
||||
export const RECORD_START = "bsk-record-start";
|
||||
export const RECORD_STEP = "bsk-record-step";
|
||||
@@ -11,6 +12,7 @@ export const RECORD_STOP = "bsk-record-stop";
|
||||
export const RECORD_CANCEL = "bsk-record-cancel";
|
||||
export const RECORD_FINISH = "bsk-record-finish";
|
||||
export const RECORD_QUERY = "bsk-record-query";
|
||||
export const RECORD_CAPTURE_STATUS = "bsk-record-capture-status";
|
||||
|
||||
export interface RecordStartAck {
|
||||
ok: true;
|
||||
@@ -27,8 +29,12 @@ export interface RecordQueryResponse {
|
||||
requestId?: string;
|
||||
/** Epoch ms when the recording began; see `RecordStartMessage.startedAtMs`. */
|
||||
startedAtMs?: number;
|
||||
captureStatus?: FrameCaptureStatus;
|
||||
captureFailures?: FrameCaptureFailure[];
|
||||
}
|
||||
|
||||
export type RecordFrameMode = "top" | "child";
|
||||
|
||||
export interface RecordStartMessage {
|
||||
type: typeof RECORD_START;
|
||||
requestId: string;
|
||||
@@ -38,6 +44,10 @@ export interface RecordStartMessage {
|
||||
* content-script remounts instead of restarting per page.
|
||||
*/
|
||||
startedAtMs?: number;
|
||||
/** When true, the top-frame overlay should mount. Child frames omit this. */
|
||||
showOverlay?: boolean;
|
||||
/** Whether this document runs top-frame or child-frame capture semantics. */
|
||||
frameMode?: RecordFrameMode;
|
||||
}
|
||||
|
||||
export interface RecordStepPayload {
|
||||
@@ -51,8 +61,14 @@ export interface RecordStepPayload {
|
||||
url?: string;
|
||||
redacted?: boolean;
|
||||
commit?: "enter" | "suggestion" | "blur";
|
||||
/** Page URL when the step was captured. */
|
||||
/** Top-level tab URL when the step was captured. */
|
||||
page_url?: string;
|
||||
/** URL of the frame document where the interaction occurred. */
|
||||
frame_url?: string;
|
||||
/** Ephemeral token stamped on the active document root; not persisted in trace steps. */
|
||||
documentToken?: string;
|
||||
/** Epoch ms when the step was captured in the content script. */
|
||||
capturedAtMs?: number;
|
||||
/** Document geometry at capture time for background geometric matching. */
|
||||
geometry?: {
|
||||
rect: { x: number; y: number; w: number; h: number };
|
||||
@@ -92,6 +108,12 @@ export interface RecordFinishMessage {
|
||||
requestId: string;
|
||||
}
|
||||
|
||||
export interface RecordCaptureStatusMessage {
|
||||
type: typeof RECORD_CAPTURE_STATUS;
|
||||
status: FrameCaptureStatus;
|
||||
failures?: FrameCaptureFailure[];
|
||||
}
|
||||
|
||||
export function isRecordStartMessage(msg: unknown): msg is RecordStartMessage {
|
||||
if (typeof msg !== "object" || msg === null) return false;
|
||||
const m = msg as Record<string, unknown>;
|
||||
@@ -122,6 +144,16 @@ export function isRecordQueryMessage(msg: unknown): msg is RecordQueryMessage {
|
||||
return m.type === RECORD_QUERY;
|
||||
}
|
||||
|
||||
export function isRecordCaptureStatusMessage(msg: unknown): msg is RecordCaptureStatusMessage {
|
||||
if (typeof msg !== "object" || msg === null) return false;
|
||||
const m = msg as Record<string, unknown>;
|
||||
return (
|
||||
m.type === RECORD_CAPTURE_STATUS &&
|
||||
(m.status === "complete" || m.status === "partial") &&
|
||||
(m.failures === undefined || Array.isArray(m.failures))
|
||||
);
|
||||
}
|
||||
|
||||
export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage {
|
||||
if (typeof msg !== "object" || msg === null) return false;
|
||||
const m = msg as Record<string, unknown>;
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
/** Attribute written on `<html>` while a document is actively recording. */
|
||||
export const RECORD_DOCUMENT_TOKEN_ATTR = "data-bsk-record-document";
|
||||
|
||||
export function createRecordDocumentToken(): string {
|
||||
if (typeof crypto !== "undefined" && typeof crypto.randomUUID === "function") {
|
||||
return crypto.randomUUID();
|
||||
}
|
||||
return `doc-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
|
||||
}
|
||||
|
||||
export function stampRecordDocumentToken(token: string): void {
|
||||
document.documentElement.setAttribute(RECORD_DOCUMENT_TOKEN_ATTR, token);
|
||||
}
|
||||
|
||||
export function clearRecordDocumentToken(): void {
|
||||
document.documentElement.removeAttribute(RECORD_DOCUMENT_TOKEN_ATTR);
|
||||
}
|
||||
|
||||
export function readRecordDocumentTokenFromAttrs(
|
||||
attrs: Record<string, string> | undefined,
|
||||
): string | undefined {
|
||||
const token = attrs?.[RECORD_DOCUMENT_TOKEN_ATTR];
|
||||
return token?.trim() ? token : undefined;
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
import {
|
||||
RECORD_CANCEL,
|
||||
RECORD_START,
|
||||
RECORD_STOP,
|
||||
type RecordCancelMessage,
|
||||
type RecordStartAck,
|
||||
type RecordStartMessage,
|
||||
type RecordStopAck,
|
||||
type RecordStopMessage,
|
||||
} from "@/lib/record-bridge";
|
||||
import type { FrameCaptureFailure, FrameCaptureFailureReason } from "@/transport/types";
|
||||
|
||||
export interface WebNavigationFrameInfo {
|
||||
frameId: number;
|
||||
documentId?: string;
|
||||
url?: string;
|
||||
parentFrameId: number;
|
||||
}
|
||||
|
||||
export interface ArmedDocument {
|
||||
tabId: number;
|
||||
frameId: number;
|
||||
documentId: string;
|
||||
url?: string;
|
||||
parentFrameId: number;
|
||||
}
|
||||
|
||||
export interface RecordFrameSendDeps {
|
||||
sendToDocument(
|
||||
tabId: number,
|
||||
documentId: string,
|
||||
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
|
||||
): Promise<unknown>;
|
||||
getAllFrames(tabId: number): Promise<WebNavigationFrameInfo[]>;
|
||||
}
|
||||
|
||||
const RECORD_START_RETRIES = 3;
|
||||
const RECORD_START_RETRY_DELAY_MS = 500;
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
export function isFrameUrlRestricted(url: string | undefined): boolean {
|
||||
if (!url) return false;
|
||||
const lower = url.toLowerCase();
|
||||
return (
|
||||
lower.startsWith("chrome://") ||
|
||||
lower.startsWith("chrome-extension://") ||
|
||||
lower.startsWith("edge://") ||
|
||||
lower.startsWith("devtools://") ||
|
||||
lower.startsWith("devtools:") ||
|
||||
lower.startsWith("https://chrome.google.com/webstore")
|
||||
);
|
||||
}
|
||||
|
||||
function isRecordStartAck(response: unknown): response is RecordStartAck {
|
||||
return (
|
||||
typeof response === "object" &&
|
||||
response !== null &&
|
||||
"ok" in response &&
|
||||
(response as RecordStartAck).ok === true
|
||||
);
|
||||
}
|
||||
|
||||
function isRecordStopAckOk(response: unknown): boolean {
|
||||
return (
|
||||
typeof response === "object" &&
|
||||
response !== null &&
|
||||
"ok" in response &&
|
||||
(response as RecordStopAck).ok === true
|
||||
);
|
||||
}
|
||||
|
||||
export function makeFrameCaptureFailure(
|
||||
reason: FrameCaptureFailureReason,
|
||||
input: { frameId: number; documentId?: string; url?: string; detail?: string },
|
||||
): FrameCaptureFailure {
|
||||
return {
|
||||
reason,
|
||||
frame_id: input.frameId,
|
||||
...(input.documentId ? { document_id: input.documentId } : {}),
|
||||
...(input.url ? { url: input.url } : {}),
|
||||
...(input.detail ? { detail: input.detail } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function listInjectableFrames(
|
||||
tabId: number,
|
||||
deps: RecordFrameSendDeps,
|
||||
): Promise<{ injectable: WebNavigationFrameInfo[]; failures: FrameCaptureFailure[] }> {
|
||||
const failures: FrameCaptureFailure[] = [];
|
||||
let frames: WebNavigationFrameInfo[] = [];
|
||||
try {
|
||||
frames = await deps.getAllFrames(tabId);
|
||||
} catch (err) {
|
||||
failures.push(
|
||||
makeFrameCaptureFailure("arm_failed", {
|
||||
frameId: 0,
|
||||
detail: err instanceof Error ? err.message : "webNavigation.getAllFrames failed",
|
||||
}),
|
||||
);
|
||||
return { injectable: [], failures };
|
||||
}
|
||||
|
||||
const injectable = frames.filter((frame) => {
|
||||
if (!frame.documentId) {
|
||||
failures.push(
|
||||
makeFrameCaptureFailure("not_injectable", {
|
||||
frameId: frame.frameId,
|
||||
url: frame.url,
|
||||
detail: "missing documentId",
|
||||
}),
|
||||
);
|
||||
return false;
|
||||
}
|
||||
if (isFrameUrlRestricted(frame.url)) {
|
||||
failures.push(
|
||||
makeFrameCaptureFailure("not_injectable", {
|
||||
frameId: frame.frameId,
|
||||
documentId: frame.documentId,
|
||||
url: frame.url,
|
||||
}),
|
||||
);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
return { injectable, failures };
|
||||
}
|
||||
|
||||
export async function sendRecordStartWithAckToDocument(
|
||||
tabId: number,
|
||||
documentId: string,
|
||||
msg: RecordStartMessage,
|
||||
deps: RecordFrameSendDeps,
|
||||
): Promise<void> {
|
||||
let lastError: unknown;
|
||||
for (let attempt = 0; attempt < RECORD_START_RETRIES; attempt += 1) {
|
||||
try {
|
||||
const response = await deps.sendToDocument(tabId, documentId, msg);
|
||||
if (isRecordStartAck(response)) return;
|
||||
lastError = new Error("content script did not ack RECORD_START");
|
||||
} catch (err) {
|
||||
lastError = err;
|
||||
}
|
||||
if (attempt + 1 < RECORD_START_RETRIES) {
|
||||
await sleep(RECORD_START_RETRY_DELAY_MS);
|
||||
}
|
||||
}
|
||||
throw lastError ?? new Error("failed to start recording in content script");
|
||||
}
|
||||
|
||||
export async function armDocumentCapture(
|
||||
tabId: number,
|
||||
frame: WebNavigationFrameInfo,
|
||||
startMsg: RecordStartMessage,
|
||||
deps: RecordFrameSendDeps,
|
||||
): Promise<{ armed?: ArmedDocument; failure?: FrameCaptureFailure }> {
|
||||
if (!frame.documentId) {
|
||||
return {
|
||||
failure: makeFrameCaptureFailure("not_injectable", {
|
||||
frameId: frame.frameId,
|
||||
url: frame.url,
|
||||
detail: "missing documentId",
|
||||
}),
|
||||
};
|
||||
}
|
||||
if (isFrameUrlRestricted(frame.url)) {
|
||||
return {
|
||||
failure: makeFrameCaptureFailure("not_injectable", {
|
||||
frameId: frame.frameId,
|
||||
documentId: frame.documentId,
|
||||
url: frame.url,
|
||||
}),
|
||||
};
|
||||
}
|
||||
try {
|
||||
await sendRecordStartWithAckToDocument(tabId, frame.documentId, startMsg, deps);
|
||||
return {
|
||||
armed: {
|
||||
tabId,
|
||||
frameId: frame.frameId,
|
||||
documentId: frame.documentId,
|
||||
url: frame.url,
|
||||
parentFrameId: frame.parentFrameId,
|
||||
},
|
||||
};
|
||||
} catch (err) {
|
||||
return {
|
||||
failure: makeFrameCaptureFailure("arm_failed", {
|
||||
frameId: frame.frameId,
|
||||
documentId: frame.documentId,
|
||||
url: frame.url,
|
||||
detail: err instanceof Error ? err.message : "RECORD_START failed",
|
||||
}),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function armAllDocumentsForTab(
|
||||
tabId: number,
|
||||
startMsg: RecordStartMessage,
|
||||
deps: RecordFrameSendDeps,
|
||||
): Promise<{ armed: ArmedDocument[]; failures: FrameCaptureFailure[] }> {
|
||||
const { injectable, failures } = await listInjectableFrames(tabId, deps);
|
||||
const armed: ArmedDocument[] = [];
|
||||
for (const frame of injectable) {
|
||||
const result = await armDocumentCapture(tabId, frame, startMsg, deps);
|
||||
if (result.armed) armed.push(result.armed);
|
||||
if (result.failure) failures.push(result.failure);
|
||||
}
|
||||
return { armed, failures };
|
||||
}
|
||||
|
||||
export async function stopDocumentCapture(
|
||||
armed: ArmedDocument,
|
||||
stopMsg: RecordStopMessage,
|
||||
deps: RecordFrameSendDeps,
|
||||
): Promise<{ ok: true } | FrameCaptureFailure> {
|
||||
try {
|
||||
const response = await deps.sendToDocument(armed.tabId, armed.documentId, stopMsg);
|
||||
if (isRecordStopAckOk(response)) return { ok: true };
|
||||
return makeFrameCaptureFailure("flush_failed", {
|
||||
frameId: armed.frameId,
|
||||
documentId: armed.documentId,
|
||||
url: armed.url,
|
||||
detail: "content script did not confirm recorded steps",
|
||||
});
|
||||
} catch (err) {
|
||||
return makeFrameCaptureFailure("flush_failed", {
|
||||
frameId: armed.frameId,
|
||||
documentId: armed.documentId,
|
||||
url: armed.url,
|
||||
detail: err instanceof Error ? err.message : "RECORD_STOP failed",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function cancelDocumentCapture(
|
||||
armed: ArmedDocument,
|
||||
cancelMsg: RecordCancelMessage,
|
||||
deps: RecordFrameSendDeps,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await deps.sendToDocument(armed.tabId, armed.documentId, cancelMsg);
|
||||
} catch {
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,8 @@ import { reduceTraceSteps, resolveTraceStartUrl } from "./trace-reducer";
|
||||
export interface LastSettledObservation {
|
||||
stateId: string;
|
||||
captured: CapturedNode[];
|
||||
matchNodes: CapturedNode[];
|
||||
documentTokenOwners: Map<string, number | null>;
|
||||
refs: RenderedRef[];
|
||||
url: string;
|
||||
title?: string;
|
||||
@@ -40,6 +42,8 @@ export interface PendingRedirectLanding {
|
||||
export interface RecordingObservationState {
|
||||
stateRegistry: Map<string, StateRegistryEntry>;
|
||||
lastSettled: LastSettledObservation | null;
|
||||
/** Latest token→owner map from the most recent observation capture. */
|
||||
documentTokenOwners: Map<string, number | null>;
|
||||
maxPageTokens: number;
|
||||
redactValues: boolean;
|
||||
lastCaptureAtMs: number;
|
||||
@@ -67,6 +71,7 @@ export function createObservationState(options?: {
|
||||
return {
|
||||
stateRegistry: new Map(),
|
||||
lastSettled: null,
|
||||
documentTokenOwners: new Map(),
|
||||
maxPageTokens: options?.maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS,
|
||||
redactValues: options?.redactValues ?? false,
|
||||
lastCaptureAtMs: 0,
|
||||
@@ -125,9 +130,12 @@ export async function captureAndRegisterObservation(
|
||||
});
|
||||
|
||||
obs.lastCaptureAtMs = Date.now();
|
||||
obs.documentTokenOwners = rendered.documentTokenOwners;
|
||||
const settled: LastSettledObservation = {
|
||||
stateId,
|
||||
captured: rendered.captured,
|
||||
matchNodes: rendered.matchNodes,
|
||||
documentTokenOwners: rendered.documentTokenOwners,
|
||||
refs: rendered.refs,
|
||||
url,
|
||||
title: resolvedTitle,
|
||||
@@ -164,7 +172,7 @@ export function applyTargetMatching(
|
||||
"geometry" in draft && draft.geometry
|
||||
? matchTarget({
|
||||
geometry: draft.geometry,
|
||||
captured: obs.lastSettled.captured,
|
||||
captured: obs.lastSettled.matchNodes,
|
||||
refs: obs.lastSettled.refs,
|
||||
fallback: draft.captureTarget,
|
||||
})
|
||||
@@ -636,6 +644,7 @@ export function buildTraceV3(input: {
|
||||
startUrl?: string;
|
||||
stoppedBy: StopReason;
|
||||
bskVersion: string;
|
||||
frameCapture?: Trace["frame_capture"];
|
||||
}): Trace {
|
||||
const { steps, stepIdByDraftId } = reduceTraceSteps(input.steps, input.obs.stateRegistry);
|
||||
const published = selectPublishedStates(input.obs.stateRegistry, steps);
|
||||
@@ -657,6 +666,7 @@ export function buildTraceV3(input: {
|
||||
stopped_by: input.stoppedBy,
|
||||
entry: { start_url: startUrl },
|
||||
recorder: { bsk: input.bskVersion, vom: 1 },
|
||||
...(input.frameCapture ? { frame_capture: input.frameCapture } : {}),
|
||||
states,
|
||||
steps,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { RECORD_DOCUMENT_TOKEN_ATTR } from "@/lib/record-document-token";
|
||||
import type { CapturedNode } from "@/tools/vom/capture";
|
||||
import { collectDocumentTokenOwners } from "../capture-vom-observation";
|
||||
|
||||
function htmlNode(
|
||||
backendNodeId: number,
|
||||
token: string,
|
||||
ownerFrameBackendNodeId: number | null = null,
|
||||
): CapturedNode {
|
||||
return {
|
||||
backendNodeId,
|
||||
parentBackendNodeId: null,
|
||||
tag: "html",
|
||||
attrs: { [RECORD_DOCUMENT_TOKEN_ATTR]: token },
|
||||
rect: { x: 0, y: 0, w: 1280, h: 720 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
...(ownerFrameBackendNodeId === null ? {} : { ownerFrameBackendNodeId }),
|
||||
};
|
||||
}
|
||||
|
||||
describe("collectDocumentTokenOwners", () => {
|
||||
it("maps document tokens to their immediate iframe owner backend node", () => {
|
||||
const topToken = "top-doc-token";
|
||||
const childToken = "child-doc-token";
|
||||
const owners = collectDocumentTokenOwners({
|
||||
viewport: { width: 1280, height: 720 },
|
||||
nodes: [htmlNode(1, topToken, null)],
|
||||
iframeNodes: new Map([
|
||||
[
|
||||
13,
|
||||
[
|
||||
htmlNode(101, childToken, 13),
|
||||
{
|
||||
backendNodeId: 102,
|
||||
parentBackendNodeId: 101,
|
||||
tag: "input",
|
||||
attrs: {},
|
||||
rect: { x: 10, y: 20, w: 200, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
ownerFrameBackendNodeId: 13,
|
||||
},
|
||||
],
|
||||
],
|
||||
]),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
});
|
||||
|
||||
expect(owners.get(topToken)).toBeNull();
|
||||
expect(owners.get(childToken)).toBe(13);
|
||||
});
|
||||
|
||||
it("maps nested iframe documents to their direct owner, not the outer iframe", () => {
|
||||
const nestedToken = "nested-doc-token";
|
||||
const owners = collectDocumentTokenOwners({
|
||||
viewport: { width: 1280, height: 720 },
|
||||
nodes: [htmlNode(1, "top-token", null)],
|
||||
iframeNodes: new Map([
|
||||
[
|
||||
13,
|
||||
[
|
||||
{
|
||||
backendNodeId: 23,
|
||||
parentBackendNodeId: 101,
|
||||
tag: "iframe",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 400, h: 300 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
ownerFrameBackendNodeId: 13,
|
||||
},
|
||||
],
|
||||
],
|
||||
[23, [htmlNode(201, nestedToken, 23)]],
|
||||
]),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
});
|
||||
|
||||
expect(owners.get(nestedToken)).toBe(23);
|
||||
});
|
||||
});
|
||||
@@ -18,11 +18,13 @@ import {
|
||||
handleRecordStart,
|
||||
handleRecordStop,
|
||||
resetBrowserObservationForTests,
|
||||
type RecordDeps,
|
||||
} from "../record";
|
||||
|
||||
const AGENT_WINDOW_ID = 100;
|
||||
const TAB_ID = 4;
|
||||
const START_URL = "https://example.com/";
|
||||
const TOP_DOCUMENT_ID = "doc-top";
|
||||
|
||||
type RuntimeListener = (
|
||||
message: unknown,
|
||||
@@ -63,9 +65,18 @@ function installChrome() {
|
||||
webNavigation: {
|
||||
onCompleted: webNavigationOnCompleted,
|
||||
onCommitted: webNavigationOnCommitted,
|
||||
getAllFrames: vi.fn(async ({ tabId }: { tabId: number }) => [
|
||||
{
|
||||
frameId: 0,
|
||||
documentId: TOP_DOCUMENT_ID,
|
||||
url: START_URL,
|
||||
parentFrameId: -1,
|
||||
tabId,
|
||||
},
|
||||
]),
|
||||
},
|
||||
});
|
||||
return { runtimeOnMessage, webNavigationOnCompleted, webNavigationOnCommitted };
|
||||
return { runtimeOnMessage, webNavigationOnCompleted, webNavigationOnCommitted, TOP_DOCUMENT_ID };
|
||||
}
|
||||
|
||||
function fakeManager() {
|
||||
@@ -206,6 +217,57 @@ function makeTabsApi() {
|
||||
};
|
||||
}
|
||||
|
||||
function makeRecordDeps(
|
||||
tabsApi: ReturnType<typeof makeTabsApi> | ReturnType<typeof makeMultiTabsApi>,
|
||||
sendToTabMock: (tabId: number, msg: unknown) => Promise<unknown>,
|
||||
cdp?: ReturnType<typeof makeFakeCdp>,
|
||||
): RecordDeps {
|
||||
return {
|
||||
tabsApi,
|
||||
sendToTab: sendToTabMock as RecordDeps["sendToTab"],
|
||||
sendToDocument: (async (tabId, _documentId, msg) =>
|
||||
sendToTabMock(tabId, msg)) as RecordDeps["sendToDocument"],
|
||||
getAllFrames: (async () => [
|
||||
{
|
||||
frameId: 0,
|
||||
documentId: TOP_DOCUMENT_ID,
|
||||
url: START_URL,
|
||||
parentFrameId: -1,
|
||||
},
|
||||
]) as RecordDeps["getAllFrames"],
|
||||
...(cdp ? { cdp } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function makeMultiTabRecordDeps(
|
||||
tabsApi: ReturnType<typeof makeMultiTabsApi>,
|
||||
sendToTabMock: (tabId: number, msg: unknown) => Promise<unknown>,
|
||||
tabIds: number[],
|
||||
cdp?: ReturnType<typeof makeFakeCdp>,
|
||||
): RecordDeps {
|
||||
const framesByTab = new Map(
|
||||
tabIds.map((tabId) => [
|
||||
tabId,
|
||||
[
|
||||
{
|
||||
frameId: 0,
|
||||
documentId: tabId === TAB_ID ? TOP_DOCUMENT_ID : `doc-tab-${tabId}`,
|
||||
url: tabId === TAB_ID ? START_URL : `https://example.com/tab-${tabId}`,
|
||||
parentFrameId: -1,
|
||||
},
|
||||
],
|
||||
]),
|
||||
);
|
||||
return {
|
||||
tabsApi,
|
||||
sendToTab: sendToTabMock as RecordDeps["sendToTab"],
|
||||
sendToDocument: (async (tabId, _documentId, msg) =>
|
||||
sendToTabMock(tabId, msg)) as RecordDeps["sendToDocument"],
|
||||
getAllFrames: (async (tabId) => framesByTab.get(tabId) ?? []) as RecordDeps["getAllFrames"],
|
||||
...(cdp ? { cdp } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function makeMultiTabsApi(tabs: chrome.tabs.Tab[]) {
|
||||
const byId = new Map(tabs.flatMap((tab) => (typeof tab.id === "number" ? [[tab.id, tab]] : [])));
|
||||
return {
|
||||
@@ -1171,25 +1233,25 @@ describe("recorded user steps reach the exported trace", () => {
|
||||
}
|
||||
return { ok: true };
|
||||
});
|
||||
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
|
||||
const deps = makeMultiTabRecordDeps(tabsApi, sendToTab, [TAB_ID, 5], makeFakeCdp());
|
||||
|
||||
await handleRecordStart(manager, { session_id: "abcd", url: START_URL }, deps);
|
||||
attachRecordQueryListener(deps);
|
||||
await new Promise<void>((resolve) => {
|
||||
chromeApi.runtimeOnMessage.emit(
|
||||
{ type: RECORD_QUERY },
|
||||
{ tab: secondTab } as chrome.runtime.MessageSender,
|
||||
{ tab: secondTab, documentId: "doc-tab-5" } as chrome.runtime.MessageSender,
|
||||
() => resolve(),
|
||||
);
|
||||
});
|
||||
|
||||
const firstStop = await handleRecordStop(manager, { session_id: "abcd" }, deps);
|
||||
expect(firstStop).toMatchObject({ code: "protocol_error" });
|
||||
expect((firstStop as RecordStopResult).trace.version).toBe(3);
|
||||
expect((firstStop as RecordStopResult).trace.frame_capture?.status).toBe("partial");
|
||||
expect(stopTabs).toEqual([TAB_ID, 5]);
|
||||
|
||||
const retry = await handleRecordStop(manager, { session_id: "abcd" }, deps);
|
||||
expect((retry as RecordStopResult).trace.version).toBe(3);
|
||||
expect(stopTabs).toEqual([TAB_ID, 5, 5]);
|
||||
expect(retry).toMatchObject({ code: "not_found" });
|
||||
});
|
||||
|
||||
it("does not let a closed armed tab block finish", async () => {
|
||||
@@ -1215,14 +1277,14 @@ describe("recorded user steps reach the exported trace", () => {
|
||||
if ((msg as { type?: string }).type === RECORD_STOP) stopTabs.push(tabId);
|
||||
return { ok: true };
|
||||
});
|
||||
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
|
||||
const deps = makeMultiTabRecordDeps(tabsApi, sendToTab, [TAB_ID, 5], makeFakeCdp());
|
||||
|
||||
await handleRecordStart(manager, { session_id: "abcd", url: START_URL }, deps);
|
||||
attachRecordQueryListener(deps);
|
||||
await new Promise<void>((resolve) => {
|
||||
chromeApi.runtimeOnMessage.emit(
|
||||
{ type: RECORD_QUERY },
|
||||
{ tab: secondTab } as chrome.runtime.MessageSender,
|
||||
{ tab: secondTab, documentId: "doc-tab-5" } as chrome.runtime.MessageSender,
|
||||
() => resolve(),
|
||||
);
|
||||
});
|
||||
@@ -1231,7 +1293,7 @@ describe("recorded user steps reach the exported trace", () => {
|
||||
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
|
||||
|
||||
expect((stopped as RecordStopResult).trace.version).toBe(3);
|
||||
expect(stopTabs).toEqual([TAB_ID]);
|
||||
expect(stopTabs).toContain(TAB_ID);
|
||||
});
|
||||
|
||||
it("waits for an acknowledged in-flight rearm before stopping armed tabs", async () => {
|
||||
@@ -1270,13 +1332,13 @@ describe("recorded user steps reach the exported trace", () => {
|
||||
if (type === RECORD_STOP) stopTabs.push(tabId);
|
||||
return { ok: true };
|
||||
});
|
||||
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
|
||||
const deps = makeMultiTabRecordDeps(tabsApi, sendToTab, [TAB_ID, 5], makeFakeCdp());
|
||||
|
||||
await handleRecordStart(manager, { session_id: "abcd", url: START_URL }, deps);
|
||||
attachRecordQueryListener(deps);
|
||||
chromeApi.runtimeOnMessage.emit(
|
||||
{ type: RECORD_QUERY },
|
||||
{ tab: secondTab } as chrome.runtime.MessageSender,
|
||||
{ tab: secondTab, documentId: "doc-tab-5" } as chrome.runtime.MessageSender,
|
||||
() => {},
|
||||
);
|
||||
await secondStartBegan;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { type RenderedRef, renderVom, type VomOptions } from "@browser-skill/vom";
|
||||
import { readRecordDocumentTokenFromAttrs } from "@/lib/record-document-token";
|
||||
import type { CdpAxNode } from "./observation";
|
||||
import { buildVomScene } from "./observation";
|
||||
import type { CdpRunner } from "./shared";
|
||||
@@ -20,9 +21,28 @@ export interface CaptureVomObservationResult {
|
||||
refs: RenderedRef[];
|
||||
truncated: boolean;
|
||||
captured: CapturedNode[];
|
||||
/** All nodes including iframe sub-documents, for geometric target matching. */
|
||||
matchNodes: CapturedNode[];
|
||||
/** Maps ephemeral record document tokens to immediate iframe owner backend ids. */
|
||||
documentTokenOwners: Map<string, number | null>;
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
}
|
||||
|
||||
export function flattenCapturedNodes(vm: CapturedViewModel): CapturedNode[] {
|
||||
return [...vm.nodes, ...[...vm.iframeNodes.values()].flat()];
|
||||
}
|
||||
|
||||
export function collectDocumentTokenOwners(vm: CapturedViewModel): Map<string, number | null> {
|
||||
const owners = new Map<string, number | null>();
|
||||
for (const node of flattenCapturedNodes(vm)) {
|
||||
if (node.tag !== "html") continue;
|
||||
const token = readRecordDocumentTokenFromAttrs(node.attrs);
|
||||
if (!token) continue;
|
||||
owners.set(token, node.ownerFrameBackendNodeId ?? null);
|
||||
}
|
||||
return owners;
|
||||
}
|
||||
|
||||
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
|
||||
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
|
||||
}
|
||||
@@ -91,11 +111,14 @@ export async function captureVomObservation(
|
||||
redactValues: opts.redactValues,
|
||||
activeRegionPolicy: opts.activeRegionPolicy ?? true,
|
||||
});
|
||||
const matchNodes = flattenCapturedNodes(captured);
|
||||
return {
|
||||
text: rendered.text,
|
||||
refs: rendered.refs,
|
||||
truncated: rendered.truncated,
|
||||
captured: captured.nodes,
|
||||
matchNodes,
|
||||
documentTokenOwners: collectDocumentTokenOwners(captured),
|
||||
surfaceProbes: captured.surfaceProbes,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -7,16 +7,28 @@ import {
|
||||
isRecordQueryMessage,
|
||||
isRecordStepMessage,
|
||||
RECORD_CANCEL,
|
||||
RECORD_CAPTURE_STATUS,
|
||||
RECORD_START,
|
||||
RECORD_STEP,
|
||||
RECORD_STOP,
|
||||
type RecordCancelMessage,
|
||||
type RecordCaptureStatusMessage,
|
||||
type RecordFinishMessage,
|
||||
type RecordQueryResponse,
|
||||
type RecordStartAck,
|
||||
type RecordStartMessage,
|
||||
type RecordStepPayload,
|
||||
type RecordStopMessage,
|
||||
} from "@/lib/record-bridge";
|
||||
import {
|
||||
armAllDocumentsForTab,
|
||||
armDocumentCapture,
|
||||
cancelDocumentCapture,
|
||||
type ArmedDocument,
|
||||
listInjectableFrames,
|
||||
type RecordFrameSendDeps,
|
||||
stopDocumentCapture,
|
||||
} from "@/lib/record-frame-arming";
|
||||
import { DEFAULT_MAX_PAGE_TOKENS } from "@/lib/record-constants";
|
||||
import {
|
||||
applyTargetMatching,
|
||||
@@ -39,6 +51,9 @@ import type { SessionManager } from "@/session-manager/manager";
|
||||
import { EXTENSION_VERSION } from "@/transport/handshake";
|
||||
import type {
|
||||
DraftTraceStep,
|
||||
FrameCaptureFailure,
|
||||
FrameCaptureInfo,
|
||||
FrameCaptureStatus,
|
||||
RecordAwaitParams,
|
||||
RecordAwaitResult,
|
||||
RecordStartParams,
|
||||
@@ -80,8 +95,12 @@ interface ActiveRecording {
|
||||
stoppedBy: StopReason;
|
||||
maxPageTokens: number;
|
||||
redactValues: boolean;
|
||||
/** Tabs whose content capture acknowledged START and still need STOP. */
|
||||
armedTabIds: Set<number>;
|
||||
/** Documents whose capture acknowledged START and still need STOP. */
|
||||
armedDocuments: Map<string, ArmedDocument>;
|
||||
frameCaptureFailures: FrameCaptureFailure[];
|
||||
frameCaptureStatus: FrameCaptureStatus;
|
||||
/** When false, ignore subframe rearm requests during finish. */
|
||||
acceptingNewFrames: boolean;
|
||||
/** Rearms already sending START when finish begins. */
|
||||
rearmCallbacks: Set<Promise<void>>;
|
||||
/** Navigation callbacks tracked from event receipt through action enqueue. */
|
||||
@@ -203,6 +222,15 @@ async function sendRecordStartWithAck(
|
||||
}
|
||||
|
||||
function buildTrace(recording: ActiveRecording): Trace {
|
||||
const frameCapture: FrameCaptureInfo | undefined =
|
||||
recording.frameCaptureFailures.length > 0 || recording.frameCaptureStatus === "partial"
|
||||
? {
|
||||
status: recording.frameCaptureStatus,
|
||||
...(recording.frameCaptureFailures.length > 0
|
||||
? { failures: recording.frameCaptureFailures }
|
||||
: {}),
|
||||
}
|
||||
: undefined;
|
||||
return buildTraceV3({
|
||||
obs: recording.observation,
|
||||
steps: recording.steps,
|
||||
@@ -211,9 +239,51 @@ function buildTrace(recording: ActiveRecording): Trace {
|
||||
startUrl: recording.startUrl,
|
||||
stoppedBy: recording.stoppedBy,
|
||||
bskVersion: BSK_TRACE_VERSION,
|
||||
...(frameCapture ? { frameCapture } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
function recordFrameFailure(recording: ActiveRecording, failure: FrameCaptureFailure): void {
|
||||
recording.frameCaptureFailures.push(failure);
|
||||
recording.frameCaptureStatus = "partial";
|
||||
}
|
||||
|
||||
function enrichRecordedStep(
|
||||
recording: ActiveRecording,
|
||||
step: RecordStepPayload,
|
||||
): RecordStepPayload {
|
||||
const enriched: RecordStepPayload = { ...step };
|
||||
if (enriched.geometry && enriched.documentToken) {
|
||||
const owner = recording.observation.documentTokenOwners.get(enriched.documentToken);
|
||||
if (owner !== undefined) {
|
||||
enriched.geometry = { ...enriched.geometry, ownerFrameBackendNodeId: owner };
|
||||
}
|
||||
}
|
||||
if (!enriched.page_url && recording.currentUrl) {
|
||||
enriched.page_url = recording.currentUrl;
|
||||
}
|
||||
return enriched;
|
||||
}
|
||||
|
||||
async function publishCaptureStatus(
|
||||
recording: ActiveRecording,
|
||||
deps: RecordDeps,
|
||||
): Promise<void> {
|
||||
if (!deps.broadcastCaptureStatus) return;
|
||||
const msg: RecordCaptureStatusMessage = {
|
||||
type: RECORD_CAPTURE_STATUS,
|
||||
status: recording.frameCaptureStatus,
|
||||
...(recording.frameCaptureFailures.length > 0
|
||||
? { failures: recording.frameCaptureFailures }
|
||||
: {}),
|
||||
};
|
||||
try {
|
||||
await deps.broadcastCaptureStatus(recording.tabId, msg);
|
||||
} catch {
|
||||
// Overlay is best-effort.
|
||||
}
|
||||
}
|
||||
|
||||
async function processRecordedStep(
|
||||
recording: ActiveRecording,
|
||||
deps: RecordDeps,
|
||||
@@ -249,17 +319,67 @@ export interface RecordDeps {
|
||||
tabId: number,
|
||||
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
|
||||
): Promise<unknown>;
|
||||
sendToDocument?(
|
||||
tabId: number,
|
||||
documentId: string,
|
||||
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
|
||||
): Promise<unknown>;
|
||||
getAllFrames?(tabId: number): Promise<
|
||||
Array<{
|
||||
frameId: number;
|
||||
documentId?: string;
|
||||
url?: string;
|
||||
parentFrameId: number;
|
||||
}>
|
||||
>;
|
||||
broadcastCaptureStatus?(
|
||||
tabId: number,
|
||||
msg: RecordCaptureStatusMessage,
|
||||
): Promise<void>;
|
||||
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
cdp?: CdpRunner;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
function frameSendDeps(deps: RecordDeps): RecordFrameSendDeps {
|
||||
return {
|
||||
sendToDocument: (tabId, documentId, msg) => {
|
||||
if (deps.sendToDocument) return deps.sendToDocument(tabId, documentId, msg);
|
||||
return deps.sendToTab(tabId, msg);
|
||||
},
|
||||
getAllFrames: async (tabId) => {
|
||||
if (deps.getAllFrames) return deps.getAllFrames(tabId);
|
||||
if (!chrome.webNavigation?.getAllFrames) return [];
|
||||
const frames = await chrome.webNavigation.getAllFrames({ tabId });
|
||||
return (frames ?? []).map((frame) => ({
|
||||
frameId: frame.frameId,
|
||||
documentId: frame.documentId,
|
||||
url: frame.url,
|
||||
parentFrameId: frame.parentFrameId,
|
||||
}));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
let defaultDeps: RecordDeps | null = null;
|
||||
function getDefaultDeps(): RecordDeps {
|
||||
if (!defaultDeps) {
|
||||
defaultDeps = {
|
||||
tabsApi: chromeTabsApi,
|
||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||
sendToDocument: (tabId, documentId, msg) =>
|
||||
chrome.tabs.sendMessage(tabId, msg, { documentId }),
|
||||
getAllFrames: async (tabId) => {
|
||||
if (!chrome.webNavigation?.getAllFrames) return [];
|
||||
const frames = await chrome.webNavigation.getAllFrames({ tabId });
|
||||
return (frames ?? []).map((frame) => ({
|
||||
frameId: frame.frameId,
|
||||
documentId: frame.documentId,
|
||||
url: frame.url,
|
||||
parentFrameId: frame.parentFrameId,
|
||||
}));
|
||||
},
|
||||
broadcastCaptureStatus: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg, { frameId: 0 }),
|
||||
};
|
||||
}
|
||||
return defaultDeps;
|
||||
@@ -320,7 +440,7 @@ export function releaseBrowserObservationListenersIfIdle(): void {
|
||||
export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||
const listener = (
|
||||
message: unknown,
|
||||
_sender: chrome.runtime.MessageSender,
|
||||
sender: chrome.runtime.MessageSender,
|
||||
_sendResponse: () => void,
|
||||
) => {
|
||||
if (!isRecordStepMessage(message)) return false;
|
||||
@@ -328,9 +448,6 @@ export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): (
|
||||
if (recording.requestId !== message.requestId) continue;
|
||||
enqueueRecordingAction(recording, async () => {
|
||||
const cdp = recording.cdp ?? deps.cdp;
|
||||
// Flush before appending so a coalesced OAuth navigate is ordered
|
||||
// ahead of the click that landed on the destination page. The flush
|
||||
// itself waits for that navigate's observation before returning.
|
||||
await flushPendingRedirectLanding(
|
||||
recording.observation,
|
||||
recording.steps,
|
||||
@@ -339,7 +456,8 @@ export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): (
|
||||
deps.tabsApi,
|
||||
{ cancelled: () => recording.settled },
|
||||
);
|
||||
const draftIndex = appendRecordedPayload(recording, message.step);
|
||||
const enriched = enrichRecordedStep(recording, message.step);
|
||||
const draftIndex = appendRecordedPayload(recording, enriched);
|
||||
if (draftIndex !== null) await processRecordedStep(recording, deps, draftIndex);
|
||||
});
|
||||
return false;
|
||||
@@ -368,8 +486,10 @@ export function attachRecordFinishListener(deps: RecordDeps = getDefaultDeps()):
|
||||
|
||||
function findRecordingByTabId(tabId: number): ActiveRecording | null {
|
||||
for (const recording of recordings.values()) {
|
||||
if (!recording.settled && (recording.tabId === tabId || recording.armedTabIds.has(tabId))) {
|
||||
return recording;
|
||||
if (recording.settled) continue;
|
||||
if (recording.tabId === tabId) return recording;
|
||||
for (const armed of recording.armedDocuments.values()) {
|
||||
if (armed.tabId === tabId) return recording;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
@@ -418,47 +538,64 @@ async function clearRearmTimersForRecording(
|
||||
}
|
||||
}
|
||||
|
||||
async function stopRecordingOnAllAgentTabs(
|
||||
async function stopRecordingOnAllDocuments(
|
||||
recording: ActiveRecording,
|
||||
deps: RecordDeps,
|
||||
): Promise<void> {
|
||||
recording.acceptingNewFrames = false;
|
||||
const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
|
||||
let failed = false;
|
||||
for (const tabId of [...recording.armedTabIds]) {
|
||||
try {
|
||||
await deps.tabsApi.get(tabId);
|
||||
} catch {
|
||||
// A closed tab no longer has content capture to flush.
|
||||
recording.armedTabIds.delete(tabId);
|
||||
const frameDeps = frameSendDeps(deps);
|
||||
const armed = [...recording.armedDocuments.values()];
|
||||
for (const doc of armed) {
|
||||
const result = await stopDocumentCapture(doc, stopMsg, frameDeps);
|
||||
if ("ok" in result) {
|
||||
recording.armedDocuments.delete(doc.documentId);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const response = await deps.sendToTab(tabId, stopMsg);
|
||||
if (!isRecordStartAck(response)) {
|
||||
throw new Error("content script did not confirm recorded steps");
|
||||
}
|
||||
recording.armedTabIds.delete(tabId);
|
||||
} catch {
|
||||
failed = true;
|
||||
continue;
|
||||
}
|
||||
if (deps.bypassOverlay) {
|
||||
try {
|
||||
await deps.bypassOverlay(tabId, false);
|
||||
} catch {
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
}
|
||||
recordFrameFailure(recording, result);
|
||||
}
|
||||
if (failed || recording.armedTabIds.size > 0) {
|
||||
throw new Error("failed to flush recorded steps");
|
||||
if (recording.armedDocuments.size > 0) {
|
||||
for (const doc of [...recording.armedDocuments.values()]) {
|
||||
recordFrameFailure(
|
||||
recording,
|
||||
{
|
||||
reason: "flush_failed",
|
||||
frame_id: doc.frameId,
|
||||
document_id: doc.documentId,
|
||||
url: doc.url,
|
||||
detail: "document still armed after STOP",
|
||||
},
|
||||
);
|
||||
}
|
||||
recording.armedDocuments.clear();
|
||||
}
|
||||
await publishCaptureStatus(recording, deps);
|
||||
if (deps.bypassOverlay) {
|
||||
try {
|
||||
await deps.bypassOverlay(recording.tabId, false);
|
||||
} catch {
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function rearmRecording(
|
||||
async function cancelRecordingOnAllDocuments(
|
||||
recording: ActiveRecording,
|
||||
deps: RecordDeps,
|
||||
): Promise<void> {
|
||||
const cancelMsg: RecordCancelMessage = { type: RECORD_CANCEL, requestId: recording.requestId };
|
||||
const frameDeps = frameSendDeps(deps);
|
||||
for (const doc of [...recording.armedDocuments.values()]) {
|
||||
await cancelDocumentCapture(doc, cancelMsg, frameDeps);
|
||||
}
|
||||
recording.armedDocuments.clear();
|
||||
}
|
||||
|
||||
async function rearmRecordingDocuments(
|
||||
recording: ActiveRecording,
|
||||
targetTabId: number,
|
||||
deps: RecordDeps,
|
||||
targetDocumentId?: string,
|
||||
): Promise<boolean> {
|
||||
let resolveTracked!: () => void;
|
||||
const tracked = new Promise<void>((resolve) => {
|
||||
@@ -466,29 +603,65 @@ async function rearmRecording(
|
||||
});
|
||||
recording.rearmCallbacks.add(tracked);
|
||||
|
||||
// Do NOT toggle automation-bypass here: each retry used to increment the
|
||||
// content-script counter, and a single stop decrement left the ControlOverlay
|
||||
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
|
||||
// already hides the control chrome while activeRecord is set.
|
||||
const frameDeps = frameSendDeps(deps);
|
||||
try {
|
||||
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
|
||||
if (recording.settled || recording.finishing) return false;
|
||||
const startMsg: RecordStartMessage = {
|
||||
type: RECORD_START,
|
||||
requestId: recording.requestId,
|
||||
startedAtMs: recording.startedAtMs,
|
||||
};
|
||||
try {
|
||||
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
|
||||
recording.armedTabIds.add(targetTabId);
|
||||
recording.tabId = targetTabId;
|
||||
return true;
|
||||
} catch {
|
||||
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
|
||||
await sleep(RECORD_REARM_RETRY_DELAY_MS);
|
||||
const attemptLimit = targetDocumentId ? 1 : RECORD_REARM_MAX_ATTEMPTS;
|
||||
for (let attempt = 0; attempt < attemptLimit; attempt += 1) {
|
||||
if (recording.settled || recording.finishing || !recording.acceptingNewFrames) return false;
|
||||
const { injectable, failures } = await listInjectableFrames(targetTabId, frameDeps);
|
||||
for (const failure of failures) recordFrameFailure(recording, failure);
|
||||
const targets = targetDocumentId
|
||||
? injectable.filter((frame) => frame.documentId === targetDocumentId)
|
||||
: injectable.filter((frame) => !recording.armedDocuments.has(frame.documentId ?? ""));
|
||||
if (targets.length === 0) {
|
||||
if (targetDocumentId) {
|
||||
recordFrameFailure(recording, {
|
||||
reason: "rearm_failed",
|
||||
frame_id: 0,
|
||||
document_id: targetDocumentId,
|
||||
detail: "document not injectable",
|
||||
});
|
||||
await publishCaptureStatus(recording, deps);
|
||||
return false;
|
||||
}
|
||||
recording.tabId = targetTabId;
|
||||
return injectable.some((frame) => recording.armedDocuments.has(frame.documentId ?? ""));
|
||||
}
|
||||
|
||||
let armedAny = false;
|
||||
for (const frame of targets) {
|
||||
const startMsg: RecordStartMessage = {
|
||||
type: RECORD_START,
|
||||
requestId: recording.requestId,
|
||||
startedAtMs: recording.startedAtMs,
|
||||
showOverlay: frame.frameId === 0,
|
||||
frameMode: frame.frameId === 0 ? "top" : "child",
|
||||
};
|
||||
const result = await armDocumentCapture(targetTabId, frame, startMsg, frameDeps);
|
||||
if (result.armed) {
|
||||
recording.armedDocuments.set(result.armed.documentId, result.armed);
|
||||
armedAny = true;
|
||||
}
|
||||
if (result.failure) recordFrameFailure(recording, result.failure);
|
||||
}
|
||||
if (armedAny) {
|
||||
recording.tabId = targetTabId;
|
||||
await publishCaptureStatus(recording, deps);
|
||||
if (deps.cdp && recording.acceptingNewFrames) {
|
||||
void captureAndRegisterObservation(
|
||||
recording.observation,
|
||||
deps.cdp,
|
||||
targetTabId,
|
||||
deps.tabsApi,
|
||||
).catch(() => {});
|
||||
}
|
||||
return true;
|
||||
}
|
||||
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
|
||||
await sleep(RECORD_REARM_RETRY_DELAY_MS);
|
||||
}
|
||||
}
|
||||
await publishCaptureStatus(recording, deps);
|
||||
return false;
|
||||
} finally {
|
||||
recording.rearmCallbacks.delete(tracked);
|
||||
@@ -505,7 +678,7 @@ function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
|
||||
rearmTimers.delete(tabId);
|
||||
void (async () => {
|
||||
const current = await findRecordingForTab(tabId, deps);
|
||||
if (current) await rearmRecording(current, tabId, deps);
|
||||
if (current) await rearmRecordingDocuments(current, tabId, deps);
|
||||
})();
|
||||
}, RECORD_REARM_DEBOUNCE_MS),
|
||||
);
|
||||
@@ -626,12 +799,24 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
|
||||
})();
|
||||
};
|
||||
|
||||
const onSubFrameComplete = (tabId: number, documentId?: string) => {
|
||||
if (!documentId) return;
|
||||
void (async () => {
|
||||
const recording = await findRecordingForTab(tabId, deps);
|
||||
if (!recording || !recording.acceptingNewFrames) return;
|
||||
await rearmRecordingDocuments(recording, tabId, deps, documentId);
|
||||
})();
|
||||
};
|
||||
|
||||
if (chrome.webNavigation?.onCompleted) {
|
||||
const completedListener = (
|
||||
details: chrome.webNavigation.WebNavigationFramedCallbackDetails,
|
||||
) => {
|
||||
if (details.frameId !== 0) return;
|
||||
onMainFrameComplete(details.tabId, details.url);
|
||||
if (details.frameId === 0) {
|
||||
onMainFrameComplete(details.tabId, details.url);
|
||||
return;
|
||||
}
|
||||
onSubFrameComplete(details.tabId, details.documentId);
|
||||
};
|
||||
const committedListener = (
|
||||
details: chrome.webNavigation.WebNavigationTransitionCallbackDetails,
|
||||
@@ -719,11 +904,15 @@ export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()):
|
||||
sendResponse({ active: false });
|
||||
return;
|
||||
}
|
||||
await rearmRecording(recording, tabId, deps);
|
||||
await rearmRecordingDocuments(recording, tabId, deps, sender.documentId);
|
||||
sendResponse({
|
||||
active: true,
|
||||
requestId: recording.requestId,
|
||||
startedAtMs: recording.startedAtMs,
|
||||
captureStatus: recording.frameCaptureStatus,
|
||||
...(recording.frameCaptureFailures.length > 0
|
||||
? { captureFailures: recording.frameCaptureFailures }
|
||||
: {}),
|
||||
});
|
||||
})();
|
||||
return true;
|
||||
@@ -760,10 +949,7 @@ async function finishRecording(
|
||||
await clearRearmTimersForRecording(recording, deps);
|
||||
await Promise.all([...recording.rearmCallbacks]);
|
||||
try {
|
||||
// Disposing content capture may commit a final dirty fill. Flush content
|
||||
// first so all resulting RECORD_STEP messages enter actionQueue before it
|
||||
// and the observation queues are drained.
|
||||
await stopRecordingOnAllAgentTabs(recording, deps);
|
||||
await stopRecordingOnAllDocuments(recording, deps);
|
||||
} catch {
|
||||
recording.finishing = false;
|
||||
return null;
|
||||
@@ -845,7 +1031,10 @@ export async function handleRecordStart(
|
||||
stoppedBy: "user_finish",
|
||||
maxPageTokens: maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS,
|
||||
redactValues,
|
||||
armedTabIds: new Set(),
|
||||
armedDocuments: new Map(),
|
||||
frameCaptureFailures: [],
|
||||
frameCaptureStatus: "complete",
|
||||
acceptingNewFrames: true,
|
||||
rearmCallbacks: new Set(),
|
||||
navigationCallbacks: new Set(),
|
||||
acceptingNavigation: true,
|
||||
@@ -857,17 +1046,11 @@ export async function handleRecordStart(
|
||||
ensureBrowserObservationListeners(deps);
|
||||
|
||||
const abortPending = async (notifyContent: boolean) => {
|
||||
const pending = recordings.get(params.session_id);
|
||||
recordings.delete(params.session_id);
|
||||
releaseBrowserObservationListenersIfIdle();
|
||||
if (notifyContent) {
|
||||
try {
|
||||
await deps.sendToTab(target.tabId, {
|
||||
type: RECORD_CANCEL,
|
||||
requestId,
|
||||
});
|
||||
} catch {
|
||||
// Content may never have received RECORD_START.
|
||||
}
|
||||
if (notifyContent && pending) {
|
||||
await cancelRecordingOnAllDocuments(pending, deps);
|
||||
}
|
||||
if (deps.bypassOverlay) {
|
||||
try {
|
||||
@@ -970,11 +1153,28 @@ export async function handleRecordStart(
|
||||
if (cancelled) return cancelled;
|
||||
}
|
||||
|
||||
const startMsg: RecordStartMessage = { type: RECORD_START, requestId, startedAtMs };
|
||||
const startMsg: RecordStartMessage = {
|
||||
type: RECORD_START,
|
||||
requestId,
|
||||
startedAtMs,
|
||||
showOverlay: true,
|
||||
frameMode: "top",
|
||||
};
|
||||
|
||||
try {
|
||||
await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
|
||||
active.armedTabIds.add(target.tabId);
|
||||
const frameDeps = frameSendDeps(deps);
|
||||
const { armed, failures } = await armAllDocumentsForTab(target.tabId, startMsg, frameDeps);
|
||||
for (const failure of failures) recordFrameFailure(active, failure);
|
||||
for (const doc of armed) active.armedDocuments.set(doc.documentId, doc);
|
||||
if (active.armedDocuments.size === 0) {
|
||||
await abortPending(true);
|
||||
return {
|
||||
code: "protocol_error",
|
||||
message:
|
||||
"failed to start recording in any frame — reload the BrowserSkill extension, then retry",
|
||||
};
|
||||
}
|
||||
await publishCaptureStatus(active, deps);
|
||||
} catch {
|
||||
await abortPending(true);
|
||||
return {
|
||||
|
||||
@@ -681,6 +681,28 @@ export interface RecorderInfo {
|
||||
vom: number;
|
||||
}
|
||||
|
||||
export type FrameCaptureStatus = "complete" | "partial";
|
||||
|
||||
export type FrameCaptureFailureReason =
|
||||
| "not_injectable"
|
||||
| "arm_failed"
|
||||
| "rearm_failed"
|
||||
| "drain_failed"
|
||||
| "flush_failed";
|
||||
|
||||
export interface FrameCaptureFailure {
|
||||
reason: FrameCaptureFailureReason;
|
||||
frame_id: number;
|
||||
document_id?: string;
|
||||
url?: string;
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
export interface FrameCaptureInfo {
|
||||
status: FrameCaptureStatus;
|
||||
failures?: FrameCaptureFailure[];
|
||||
}
|
||||
|
||||
export type StopReason = "user_finish" | "cli_stop";
|
||||
|
||||
export interface TraceState {
|
||||
@@ -864,6 +886,8 @@ export interface Trace {
|
||||
stopped_by: StopReason;
|
||||
entry: TraceEntry;
|
||||
recorder: RecorderInfo;
|
||||
/** Present when one or more frames could not be armed, rearmed, or flushed. */
|
||||
frame_capture?: FrameCaptureInfo;
|
||||
states: TraceState[];
|
||||
steps: Step[];
|
||||
}
|
||||
|
||||
@@ -285,6 +285,7 @@ bsk record stop [--output trace] # terminal fallback if the browser panel is u
|
||||
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
|
||||
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
||||
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
|
||||
- Recording captures interactions in the **top frame and injectable child frames** (same-origin and cross-origin where the extension can attach). Embedded frames that cannot be injected are omitted; the trace may include `frame_capture.status: "partial"` with failure details.
|
||||
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
||||
|
||||
## Error handling
|
||||
|
||||
@@ -8,8 +8,8 @@ use std::time::Duration;
|
||||
|
||||
use anyhow::Context;
|
||||
use bsk_protocol::tools::{
|
||||
RecordAwaitParams, RecordAwaitResult, RecordStartParams, RecordStartResult, RecordStopParams,
|
||||
RecordStopResult, TRACE_VERSION, Trace, TraceState,
|
||||
FrameCaptureStatus, RecordAwaitParams, RecordAwaitResult, RecordStartParams, RecordStartResult,
|
||||
RecordStopParams, RecordStopResult, TRACE_VERSION, Trace, TraceState,
|
||||
};
|
||||
use bsk_protocol::{ErrorCode, Method};
|
||||
use clap::{Args, Subcommand};
|
||||
@@ -550,6 +550,13 @@ fn render_finish(
|
||||
trace_path.display(),
|
||||
pages_dir.display()
|
||||
);
|
||||
if let Some(frame_capture) = &trace.frame_capture {
|
||||
if matches!(frame_capture.status, FrameCaptureStatus::Partial) {
|
||||
eprintln!(
|
||||
"warning: trace is partial — some embedded frames were not recorded (see frame_capture.failures in trace.json)"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
@@ -584,6 +591,7 @@ mod tests {
|
||||
bsk: "0.1.10".into(),
|
||||
vom: VOM_FORMAT_VERSION,
|
||||
},
|
||||
frame_capture: None,
|
||||
states: vec![TraceState {
|
||||
id: state_id.into(),
|
||||
url: "https://example.com/".into(),
|
||||
|
||||
@@ -96,6 +96,7 @@ fn sample_trace() -> Trace {
|
||||
bsk: "0.1.10".into(),
|
||||
vom: VOM_FORMAT_VERSION,
|
||||
},
|
||||
frame_capture: None,
|
||||
states: vec![TraceState {
|
||||
id: "s1".into(),
|
||||
url: "https://example.com/".into(),
|
||||
|
||||
@@ -16,6 +16,16 @@
|
||||
"entry": {
|
||||
"$ref": "#/definitions/TraceEntry"
|
||||
},
|
||||
"frame_capture": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/FrameCaptureInfo"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"purpose": {
|
||||
"type": [
|
||||
"string",
|
||||
@@ -64,6 +74,77 @@
|
||||
"blur"
|
||||
]
|
||||
},
|
||||
"FrameCaptureFailure": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"frame_id",
|
||||
"reason"
|
||||
],
|
||||
"properties": {
|
||||
"detail": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"document_id": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"frame_id": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
},
|
||||
"reason": {
|
||||
"$ref": "#/definitions/FrameCaptureFailureReason"
|
||||
},
|
||||
"url": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"FrameCaptureFailureReason": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"not_injectable",
|
||||
"arm_failed",
|
||||
"rearm_failed",
|
||||
"drain_failed",
|
||||
"flush_failed"
|
||||
]
|
||||
},
|
||||
"FrameCaptureInfo": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"status"
|
||||
],
|
||||
"properties": {
|
||||
"failures": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/FrameCaptureFailure"
|
||||
}
|
||||
},
|
||||
"status": {
|
||||
"$ref": "#/definitions/FrameCaptureStatus"
|
||||
}
|
||||
}
|
||||
},
|
||||
"FrameCaptureStatus": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"complete",
|
||||
"partial"
|
||||
]
|
||||
},
|
||||
"KeyModifier": {
|
||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||
"type": "string",
|
||||
|
||||
@@ -167,6 +167,46 @@ pub enum Step {
|
||||
},
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Frame capture metadata
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum FrameCaptureStatus {
|
||||
Complete,
|
||||
Partial,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum FrameCaptureFailureReason {
|
||||
NotInjectable,
|
||||
ArmFailed,
|
||||
RearmFailed,
|
||||
DrainFailed,
|
||||
FlushFailed,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct FrameCaptureFailure {
|
||||
pub reason: FrameCaptureFailureReason,
|
||||
pub frame_id: i64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub document_id: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub url: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub detail: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct FrameCaptureInfo {
|
||||
pub status: FrameCaptureStatus,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub failures: Option<Vec<FrameCaptureFailure>>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Trace root
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -183,6 +223,8 @@ pub struct Trace {
|
||||
pub stopped_by: StopReason,
|
||||
pub entry: TraceEntry,
|
||||
pub recorder: RecorderInfo,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub frame_capture: Option<FrameCaptureInfo>,
|
||||
pub states: Vec<TraceState>,
|
||||
pub steps: Vec<Step>,
|
||||
}
|
||||
@@ -277,6 +319,7 @@ mod tests {
|
||||
bsk: "0.1.10".into(),
|
||||
vom: VOM_FORMAT_VERSION,
|
||||
},
|
||||
frame_capture: None,
|
||||
states: vec![
|
||||
TraceState {
|
||||
id: "s1".into(),
|
||||
@@ -479,4 +522,39 @@ mod tests {
|
||||
assert_eq!(trace.states.len(), 2);
|
||||
assert_eq!(trace.steps.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_deserializes_optional_frame_capture_metadata() {
|
||||
let v = json!({
|
||||
"version": 3,
|
||||
"recorded_at": "2026-07-21T08:00:00Z",
|
||||
"stopped_by": "cli_stop",
|
||||
"entry": { "start_url": "https://example.com/editor" },
|
||||
"recorder": { "bsk": "0.1.10", "vom": 1 },
|
||||
"frame_capture": {
|
||||
"status": "partial",
|
||||
"failures": [
|
||||
{
|
||||
"reason": "not_injectable",
|
||||
"frame_id": 2,
|
||||
"url": "chrome://settings",
|
||||
"detail": "restricted frame url"
|
||||
}
|
||||
]
|
||||
},
|
||||
"states": [],
|
||||
"steps": []
|
||||
});
|
||||
let trace: Trace = serde_json::from_value(v).unwrap();
|
||||
let frame_capture = trace.frame_capture.expect("frame_capture");
|
||||
assert_eq!(frame_capture.status, FrameCaptureStatus::Partial);
|
||||
let failures = frame_capture.failures.expect("failures");
|
||||
assert_eq!(failures.len(), 1);
|
||||
assert_eq!(
|
||||
failures[0].reason,
|
||||
FrameCaptureFailureReason::NotInjectable
|
||||
);
|
||||
assert_eq!(failures[0].frame_id, 2);
|
||||
assert_eq!(failures[0].url.as_deref(), Some("chrome://settings"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,6 +90,7 @@
|
||||
},
|
||||
"recordOverlay": {
|
||||
"recording": "Recording your actions",
|
||||
"finish": "Finish"
|
||||
"finish": "Finish",
|
||||
"partialCapture": "Some embedded frames may not be recorded."
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,6 +90,7 @@
|
||||
},
|
||||
"recordOverlay": {
|
||||
"recording": "正在录制你的操作",
|
||||
"finish": "结束"
|
||||
"finish": "结束",
|
||||
"partialCapture": "部分嵌入页面可能未被录制。"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -285,6 +285,7 @@ bsk record stop [--output trace] # terminal fallback if the browser panel is u
|
||||
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
|
||||
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
||||
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
|
||||
- Recording captures interactions in the **top frame and injectable child frames** (same-origin and cross-origin where the extension can attach). Embedded frames that cannot be injected are omitted; the trace may include `frame_capture.status: "partial"` with failure details.
|
||||
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
||||
|
||||
## Error handling
|
||||
|
||||
Reference in New Issue
Block a user