mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f7e08f461b |
@@ -17,3 +17,7 @@ dist/
|
|||||||
# Environment
|
# Environment
|
||||||
.env.local
|
.env.local
|
||||||
.env.*.local
|
.env.*.local
|
||||||
|
.pnpm-store/
|
||||||
|
|
||||||
|
# Local recording output
|
||||||
|
/trace/
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
import { useTranslation } from "@browser-skill/i18n/react";
|
import { useTranslation } from "@browser-skill/i18n/react";
|
||||||
|
import type { FrameCaptureFailure, FrameCaptureStatus } from "@/transport/types";
|
||||||
import {
|
import {
|
||||||
type PointerEvent as ReactPointerEvent,
|
type PointerEvent as ReactPointerEvent,
|
||||||
useCallback,
|
useCallback,
|
||||||
@@ -21,6 +22,8 @@ export interface RecordRequestData {
|
|||||||
|
|
||||||
type Props = {
|
type Props = {
|
||||||
request: RecordRequestData | null;
|
request: RecordRequestData | null;
|
||||||
|
captureStatus?: FrameCaptureStatus;
|
||||||
|
captureFailures?: FrameCaptureFailure[];
|
||||||
};
|
};
|
||||||
|
|
||||||
const VIEWPORT_MARGIN = 16;
|
const VIEWPORT_MARGIN = 16;
|
||||||
@@ -52,7 +55,11 @@ function clampDragPos(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export function RecordOverlay({ request }: Props) {
|
export function RecordOverlay({
|
||||||
|
request,
|
||||||
|
captureStatus = "complete",
|
||||||
|
captureFailures = [],
|
||||||
|
}: Props) {
|
||||||
const { t } = useTranslation("extension");
|
const { t } = useTranslation("extension");
|
||||||
const [show, setShow] = useState(false);
|
const [show, setShow] = useState(false);
|
||||||
const [elapsedSeconds, setElapsedSeconds] = useState(0);
|
const [elapsedSeconds, setElapsedSeconds] = useState(0);
|
||||||
@@ -154,6 +161,7 @@ export function RecordOverlay({ request }: Props) {
|
|||||||
|
|
||||||
const introVisible = elapsedSeconds * 1_000 < INTRO_DURATION_MS;
|
const introVisible = elapsedSeconds * 1_000 < INTRO_DURATION_MS;
|
||||||
const positioned = dragPos !== null;
|
const positioned = dragPos !== null;
|
||||||
|
const partialCapture = captureStatus === "partial" || captureFailures.length > 0;
|
||||||
const defaultTransform = show
|
const defaultTransform = show
|
||||||
? "translateX(-50%) translateY(0)"
|
? "translateX(-50%) translateY(0)"
|
||||||
: "translateX(-50%) translateY(8px)";
|
: "translateX(-50%) translateY(8px)";
|
||||||
@@ -208,8 +216,9 @@ export function RecordOverlay({ request }: Props) {
|
|||||||
data-slot="record-overlay-status"
|
data-slot="record-overlay-status"
|
||||||
style={{
|
style={{
|
||||||
display: "flex",
|
display: "flex",
|
||||||
alignItems: "center",
|
flexDirection: "column",
|
||||||
gap: 8,
|
alignItems: "flex-start",
|
||||||
|
gap: 2,
|
||||||
minWidth: 0,
|
minWidth: 0,
|
||||||
flex: 1,
|
flex: 1,
|
||||||
borderRadius: 9999,
|
borderRadius: 9999,
|
||||||
@@ -218,41 +227,57 @@ export function RecordOverlay({ request }: Props) {
|
|||||||
paddingRight: 4,
|
paddingRight: 4,
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
<span
|
<div style={{ display: "flex", alignItems: "center", gap: 8, minWidth: 0, width: "100%" }}>
|
||||||
data-slot="record-overlay-indicator"
|
<span
|
||||||
style={{
|
data-slot="record-overlay-indicator"
|
||||||
width: 9,
|
style={{
|
||||||
height: 9,
|
width: 9,
|
||||||
borderRadius: "50%",
|
height: 9,
|
||||||
backgroundColor: "#ef4444",
|
borderRadius: "50%",
|
||||||
flexShrink: 0,
|
backgroundColor: "#ef4444",
|
||||||
animation: "bsk-rec-pulse 1.4s ease-in-out infinite",
|
flexShrink: 0,
|
||||||
}}
|
animation: "bsk-rec-pulse 1.4s ease-in-out infinite",
|
||||||
/>
|
}}
|
||||||
<span
|
/>
|
||||||
data-slot="record-overlay-label"
|
<span
|
||||||
style={{
|
data-slot="record-overlay-label"
|
||||||
display: "block",
|
style={{
|
||||||
flex: "0 1 auto",
|
display: "block",
|
||||||
fontSize: 16,
|
flex: "0 1 auto",
|
||||||
fontWeight: 500,
|
fontSize: 16,
|
||||||
color: "#333",
|
fontWeight: 500,
|
||||||
whiteSpace: "nowrap",
|
color: "#333",
|
||||||
userSelect: "none",
|
whiteSpace: "nowrap",
|
||||||
overflow: "hidden",
|
userSelect: "none",
|
||||||
textOverflow: "ellipsis",
|
overflow: "hidden",
|
||||||
width: introVisible ? "auto" : 48,
|
textOverflow: "ellipsis",
|
||||||
maxWidth: introVisible ? 280 : 48,
|
width: introVisible ? "auto" : 48,
|
||||||
fontVariantNumeric: introVisible ? undefined : "tabular-nums",
|
maxWidth: introVisible ? 280 : 48,
|
||||||
textAlign: introVisible ? "left" : "center",
|
fontVariantNumeric: introVisible ? undefined : "tabular-nums",
|
||||||
margin: 0,
|
textAlign: introVisible ? "left" : "center",
|
||||||
padding: 0,
|
margin: 0,
|
||||||
minWidth: 0,
|
padding: 0,
|
||||||
transition: "max-width 180ms ease-out",
|
minWidth: 0,
|
||||||
}}
|
transition: "max-width 180ms ease-out",
|
||||||
>
|
}}
|
||||||
{introVisible ? t("recordOverlay.recording") : formatElapsed(elapsedSeconds)}
|
>
|
||||||
</span>
|
{introVisible ? t("recordOverlay.recording") : formatElapsed(elapsedSeconds)}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
{partialCapture ? (
|
||||||
|
<span
|
||||||
|
data-slot="record-overlay-partial-warning"
|
||||||
|
style={{
|
||||||
|
fontSize: 12,
|
||||||
|
lineHeight: "16px",
|
||||||
|
color: "#b45309",
|
||||||
|
whiteSpace: "normal",
|
||||||
|
maxWidth: 280,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{t("recordOverlay.partialCapture")}
|
||||||
|
</span>
|
||||||
|
) : null}
|
||||||
</div>
|
</div>
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
|
|||||||
@@ -178,4 +178,30 @@ describe("RecordOverlay", () => {
|
|||||||
expect(pill.style.bottom).toBe("auto");
|
expect(pill.style.bottom).toBe("auto");
|
||||||
expect(pill.style.transform).toBe("none");
|
expect(pill.style.transform).toBe("none");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("shows a persistent partial-capture warning when frame capture is incomplete", () => {
|
||||||
|
const { container } = render(
|
||||||
|
createElement(
|
||||||
|
I18nextProvider,
|
||||||
|
{ i18n },
|
||||||
|
createElement(RecordOverlay, {
|
||||||
|
request: { id: "rec-1", onFinish: vi.fn() },
|
||||||
|
captureStatus: "partial",
|
||||||
|
captureFailures: [
|
||||||
|
{
|
||||||
|
reason: "not_injectable",
|
||||||
|
frame_id: 2,
|
||||||
|
url: "chrome://settings",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
const warning = container.querySelector("[data-slot='record-overlay-partial-warning']");
|
||||||
|
expect(warning).toBeTruthy();
|
||||||
|
expect(warning?.textContent).toBe(
|
||||||
|
i18n.t("recordOverlay.partialCapture", { ns: "extension" }),
|
||||||
|
);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1614,3 +1614,79 @@ describe("record-capture semantic", () => {
|
|||||||
expect(steps.map((s) => s.op)).toEqual(["click"]);
|
expect(steps.map((s) => s.op)).toEqual(["click"]);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("record-capture child frame", () => {
|
||||||
|
let steps: RecordStepPayload[];
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
steps = [];
|
||||||
|
document.body.innerHTML = `
|
||||||
|
<label for="child-q">Child query</label>
|
||||||
|
<input id="child-q" name="child-q" />
|
||||||
|
<button type="button" aria-label="Child search">Search</button>
|
||||||
|
<select aria-label="Child status">
|
||||||
|
<option value="on">On</option>
|
||||||
|
<option value="off">Off</option>
|
||||||
|
</select>
|
||||||
|
`;
|
||||||
|
});
|
||||||
|
|
||||||
|
it("stamps a document token on start and clears it on dispose", () => {
|
||||||
|
const capture = startRecordCapture("rec-child-token", (step) => steps.push(step), {
|
||||||
|
frameMode: "child",
|
||||||
|
});
|
||||||
|
const token = document.documentElement.getAttribute("data-bsk-record-document");
|
||||||
|
expect(token).toBeTruthy();
|
||||||
|
click(document.querySelector("button")!);
|
||||||
|
expect(steps[0]?.documentToken).toBe(token);
|
||||||
|
expect(steps[0]?.frame_url).toBeTruthy();
|
||||||
|
capture.dispose();
|
||||||
|
expect(document.documentElement.hasAttribute("data-bsk-record-document")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("records click, fill, select, and press in child frame mode", () => {
|
||||||
|
const capture = startRecordCapture("rec-child-ops", (step) => steps.push(step), {
|
||||||
|
frameMode: "child",
|
||||||
|
});
|
||||||
|
const input = document.querySelector("input")!;
|
||||||
|
const select = document.querySelector("select")!;
|
||||||
|
|
||||||
|
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||||
|
input.value = "child value";
|
||||||
|
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||||
|
input.dispatchEvent(new FocusEvent("focusout", { bubbles: true }));
|
||||||
|
select.value = "off";
|
||||||
|
select.dispatchEvent(new Event("change", { bubbles: true }));
|
||||||
|
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||||
|
input.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", bubbles: true }));
|
||||||
|
click(document.querySelector("button")!);
|
||||||
|
capture.dispose();
|
||||||
|
|
||||||
|
expect(steps.map((s) => s.op)).toEqual(
|
||||||
|
expect.arrayContaining(["fill", "select", "press", "click"]),
|
||||||
|
);
|
||||||
|
expect(steps.every((s) => s.documentToken)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("commits the last fill when dispose runs in child frame mode", () => {
|
||||||
|
const capture = startRecordCapture("rec-child-fill-stop", (step) => steps.push(step), {
|
||||||
|
frameMode: "child",
|
||||||
|
});
|
||||||
|
const input = document.querySelector("input")!;
|
||||||
|
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||||
|
input.value = "pending child fill";
|
||||||
|
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||||
|
capture.dispose();
|
||||||
|
|
||||||
|
expect(steps.some((s) => s.op === "fill" && s.value === "pending child fill")).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not emit navigate steps for child frame navigations", () => {
|
||||||
|
const capture = startRecordCapture("rec-child-nav", (step) => steps.push(step), {
|
||||||
|
frameMode: "child",
|
||||||
|
});
|
||||||
|
history.pushState({}, "", "/child-frame-path");
|
||||||
|
capture.dispose();
|
||||||
|
expect(steps.filter((s) => s.op === "navigate")).toHaveLength(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -21,12 +21,18 @@ import {
|
|||||||
RECORD_STEP,
|
RECORD_STEP,
|
||||||
RECORD_STOP,
|
RECORD_STOP,
|
||||||
type RecordCancelMessage,
|
type RecordCancelMessage,
|
||||||
|
type RecordFrameMode,
|
||||||
type RecordStartAck,
|
type RecordStartAck,
|
||||||
type RecordStartMessage,
|
type RecordStartMessage,
|
||||||
type RecordStepPayload,
|
type RecordStepPayload,
|
||||||
type RecordStopAck,
|
type RecordStopAck,
|
||||||
type RecordStopMessage,
|
type RecordStopMessage,
|
||||||
} from "@/lib/record-bridge";
|
} from "@/lib/record-bridge";
|
||||||
|
import {
|
||||||
|
clearRecordDocumentToken,
|
||||||
|
createRecordDocumentToken,
|
||||||
|
stampRecordDocumentToken,
|
||||||
|
} from "@/lib/record-document-token";
|
||||||
import { shouldRecordPress } from "@/lib/trace-reducer";
|
import { shouldRecordPress } from "@/lib/trace-reducer";
|
||||||
import {
|
import {
|
||||||
closestHoverSurfaceCandidate,
|
closestHoverSurfaceCandidate,
|
||||||
@@ -402,12 +408,37 @@ function scheduleInputCompletionCommit(
|
|||||||
setTimeout(syncAndCommit, 0);
|
setTimeout(syncAndCommit, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface RecordCaptureOptions {
|
||||||
|
frameMode?: RecordFrameMode;
|
||||||
|
}
|
||||||
|
|
||||||
export function startRecordCapture(
|
export function startRecordCapture(
|
||||||
_requestId: string,
|
_requestId: string,
|
||||||
sendStep: (step: RecordStepPayload) => void,
|
sendStep: (step: RecordStepPayload) => void,
|
||||||
|
options: RecordCaptureOptions = {},
|
||||||
): RecordCaptureController {
|
): RecordCaptureController {
|
||||||
|
const frameMode = options.frameMode ?? "top";
|
||||||
|
const emitNavigate = frameMode === "top";
|
||||||
|
const documentToken = createRecordDocumentToken();
|
||||||
|
stampRecordDocumentToken(documentToken);
|
||||||
|
const capturedAtMs = () => Date.now();
|
||||||
|
|
||||||
|
const resolveTopPageUrl = (): string => {
|
||||||
|
try {
|
||||||
|
return window.top?.location.href ?? location.href;
|
||||||
|
} catch {
|
||||||
|
return location.href;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
const emitStep = (step: RecordStepPayload) => {
|
const emitStep = (step: RecordStepPayload) => {
|
||||||
sendStep({ page_url: location.href, ...step });
|
sendStep({
|
||||||
|
page_url: resolveTopPageUrl(),
|
||||||
|
frame_url: location.href,
|
||||||
|
documentToken,
|
||||||
|
capturedAtMs: capturedAtMs(),
|
||||||
|
...step,
|
||||||
|
});
|
||||||
};
|
};
|
||||||
const hoverSurfaceStateMap = (states: HoverSurfaceState[]): Map<Element, string> =>
|
const hoverSurfaceStateMap = (states: HoverSurfaceState[]): Map<Element, string> =>
|
||||||
new Map(states.map((state) => [state.element, state.signature]));
|
new Map(states.map((state) => [state.element, state.signature]));
|
||||||
@@ -478,6 +509,7 @@ export function startRecordCapture(
|
|||||||
};
|
};
|
||||||
|
|
||||||
const emitNavigateIfChanged = (causedByAction?: boolean) => {
|
const emitNavigateIfChanged = (causedByAction?: boolean) => {
|
||||||
|
if (!emitNavigate) return;
|
||||||
if (location.href === lastUrl) return;
|
if (location.href === lastUrl) return;
|
||||||
commitFillSession();
|
commitFillSession();
|
||||||
lastUrl = location.href;
|
lastUrl = location.href;
|
||||||
@@ -965,6 +997,7 @@ export function startRecordCapture(
|
|||||||
return {
|
return {
|
||||||
dispose() {
|
dispose() {
|
||||||
commitFillSession();
|
commitFillSession();
|
||||||
|
clearRecordDocumentToken();
|
||||||
document.removeEventListener("click", onClick, true);
|
document.removeEventListener("click", onClick, true);
|
||||||
document.removeEventListener("mouseover", onMouseOver, true);
|
document.removeEventListener("mouseover", onMouseOver, true);
|
||||||
document.removeEventListener("focusin", onFocusIn, true);
|
document.removeEventListener("focusin", onFocusIn, true);
|
||||||
@@ -997,8 +1030,8 @@ export function handleRecordContentMessage(
|
|||||||
capture: RecordCaptureController | null;
|
capture: RecordCaptureController | null;
|
||||||
setActiveRequestId(id: string | null): void;
|
setActiveRequestId(id: string | null): void;
|
||||||
setCapture(capture: RecordCaptureController | null): void;
|
setCapture(capture: RecordCaptureController | null): void;
|
||||||
onStart(requestId: string, startedAtMs?: number): void;
|
onStart?(requestId: string, startedAtMs?: number): void;
|
||||||
onStop(): void;
|
onStop?(): void;
|
||||||
},
|
},
|
||||||
sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
|
sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
|
||||||
): boolean {
|
): boolean {
|
||||||
@@ -1009,13 +1042,20 @@ export function handleRecordContentMessage(
|
|||||||
failedStepPayloads.delete(message.requestId);
|
failedStepPayloads.delete(message.requestId);
|
||||||
}
|
}
|
||||||
state.capture?.dispose();
|
state.capture?.dispose();
|
||||||
|
const frameMode = message.frameMode ?? "top";
|
||||||
state.setCapture(
|
state.setCapture(
|
||||||
startRecordCapture(message.requestId, (step) => {
|
startRecordCapture(
|
||||||
sendRecordStep(message.requestId, step);
|
message.requestId,
|
||||||
}),
|
(step) => {
|
||||||
|
sendRecordStep(message.requestId, step);
|
||||||
|
},
|
||||||
|
{ frameMode },
|
||||||
|
),
|
||||||
);
|
);
|
||||||
state.setActiveRequestId(message.requestId);
|
state.setActiveRequestId(message.requestId);
|
||||||
state.onStart(message.requestId, message.startedAtMs);
|
if (message.showOverlay !== false && frameMode === "top") {
|
||||||
|
state.onStart?.(message.requestId, message.startedAtMs);
|
||||||
|
}
|
||||||
sendResponse?.({ ok: true });
|
sendResponse?.({ ok: true });
|
||||||
return sendResponse !== undefined;
|
return sendResponse !== undefined;
|
||||||
}
|
}
|
||||||
@@ -1030,7 +1070,7 @@ export function handleRecordContentMessage(
|
|||||||
state.capture?.dispose();
|
state.capture?.dispose();
|
||||||
state.setCapture(null);
|
state.setCapture(null);
|
||||||
const finishStop = () => {
|
const finishStop = () => {
|
||||||
state.onStop();
|
state.onStop?.();
|
||||||
state.setActiveRequestId(null);
|
state.setActiveRequestId(null);
|
||||||
};
|
};
|
||||||
if (isRecordStopMessage(message) && sendResponse) {
|
if (isRecordStopMessage(message) && sendResponse) {
|
||||||
|
|||||||
@@ -33,6 +33,9 @@ import {
|
|||||||
defaultBorrowChromeWindows,
|
defaultBorrowChromeWindows,
|
||||||
requestBorrowConfirmation,
|
requestBorrowConfirmation,
|
||||||
} from "@/tools/borrow-confirmation";
|
} from "@/tools/borrow-confirmation";
|
||||||
|
import {
|
||||||
|
type RecordCaptureStatusMessage,
|
||||||
|
} from "@/lib/record-bridge";
|
||||||
import { ToolDispatcher } from "@/tools/dispatcher";
|
import { ToolDispatcher } from "@/tools/dispatcher";
|
||||||
import {
|
import {
|
||||||
attachRecordFinishListener,
|
attachRecordFinishListener,
|
||||||
@@ -175,6 +178,24 @@ export default defineBackground(() => {
|
|||||||
cdp,
|
cdp,
|
||||||
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
|
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
|
||||||
chrome.tabs.sendMessage(tabId, msg),
|
chrome.tabs.sendMessage(tabId, msg),
|
||||||
|
sendToDocument: (
|
||||||
|
tabId: number,
|
||||||
|
documentId: string,
|
||||||
|
msg: Parameters<typeof chrome.tabs.sendMessage>[1],
|
||||||
|
) => chrome.tabs.sendMessage(tabId, msg, { documentId }),
|
||||||
|
getAllFrames: async (tabId: number) => {
|
||||||
|
if (!chrome.webNavigation?.getAllFrames) return [];
|
||||||
|
const frames = await chrome.webNavigation.getAllFrames({ tabId });
|
||||||
|
return (frames ?? []).map((frame) => ({
|
||||||
|
frameId: frame.frameId,
|
||||||
|
documentId: frame.documentId,
|
||||||
|
url: frame.url,
|
||||||
|
parentFrameId: frame.parentFrameId,
|
||||||
|
}));
|
||||||
|
},
|
||||||
|
broadcastCaptureStatus: async (tabId: number, msg: RecordCaptureStatusMessage) => {
|
||||||
|
await chrome.tabs.sendMessage(tabId, msg, { frameId: 0 });
|
||||||
|
},
|
||||||
bypassOverlay: async (tabId: number, enabled: boolean) => {
|
bypassOverlay: async (tabId: number, enabled: boolean) => {
|
||||||
try {
|
try {
|
||||||
await chrome.tabs.sendMessage(tabId, {
|
await chrome.tabs.sendMessage(tabId, {
|
||||||
|
|||||||
@@ -44,12 +44,15 @@ import {
|
|||||||
} from "@/lib/overlay-bridge";
|
} from "@/lib/overlay-bridge";
|
||||||
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
|
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
|
||||||
import {
|
import {
|
||||||
|
isRecordCaptureStatusMessage,
|
||||||
RECORD_FINISH,
|
RECORD_FINISH,
|
||||||
RECORD_QUERY,
|
RECORD_QUERY,
|
||||||
|
type RecordCaptureStatusMessage,
|
||||||
type RecordQueryResponse,
|
type RecordQueryResponse,
|
||||||
type RecordStartAck,
|
type RecordStartAck,
|
||||||
type RecordStopAck,
|
type RecordStopAck,
|
||||||
} from "@/lib/record-bridge";
|
} from "@/lib/record-bridge";
|
||||||
|
import type { FrameCaptureFailure, FrameCaptureStatus } from "@/transport/types";
|
||||||
import type {
|
import type {
|
||||||
BorrowCancelMessage,
|
BorrowCancelMessage,
|
||||||
BorrowRequestMessage,
|
BorrowRequestMessage,
|
||||||
@@ -70,6 +73,8 @@ export default defineContentScript({
|
|||||||
const overlays = new OverlayController();
|
const overlays = new OverlayController();
|
||||||
let recordCapture: RecordCaptureController | null = null;
|
let recordCapture: RecordCaptureController | null = null;
|
||||||
let activeRecordRequestId: string | null = null;
|
let activeRecordRequestId: string | null = null;
|
||||||
|
let recordCaptureStatus: FrameCaptureStatus = "complete";
|
||||||
|
let recordCaptureFailures: FrameCaptureFailure[] = [];
|
||||||
let reactRoot: ReactDOM.Root | null = null;
|
let reactRoot: ReactDOM.Root | null = null;
|
||||||
let overlayHost: HTMLElement | null = null;
|
let overlayHost: HTMLElement | null = null;
|
||||||
let overlayContainer: HTMLElement | null = null;
|
let overlayContainer: HTMLElement | null = null;
|
||||||
@@ -188,7 +193,11 @@ export default defineContentScript({
|
|||||||
requests: overlayState.borrowRequests,
|
requests: overlayState.borrowRequests,
|
||||||
}),
|
}),
|
||||||
React.createElement(HelpRequestOverlay, { request: overlayState.activeHelp }),
|
React.createElement(HelpRequestOverlay, { request: overlayState.activeHelp }),
|
||||||
React.createElement(RecordOverlay, { request: overlayState.activeRecord }),
|
React.createElement(RecordOverlay, {
|
||||||
|
request: overlayState.activeRecord,
|
||||||
|
captureStatus: recordCaptureStatus,
|
||||||
|
captureFailures: recordCaptureFailures,
|
||||||
|
}),
|
||||||
React.createElement(ControlOverlay, {
|
React.createElement(ControlOverlay, {
|
||||||
visible: controlOverlayVisible,
|
visible: controlOverlayVisible,
|
||||||
interrupting: overlayState.interrupting,
|
interrupting: overlayState.interrupting,
|
||||||
@@ -260,7 +269,8 @@ export default defineContentScript({
|
|||||||
| CaptureSuppressMessage
|
| CaptureSuppressMessage
|
||||||
| OverlayAgentOverlayResetMessage
|
| OverlayAgentOverlayResetMessage
|
||||||
| OverlayAgentStateMessage
|
| OverlayAgentStateMessage
|
||||||
| OverlayAutomationBypassMessage,
|
| OverlayAutomationBypassMessage
|
||||||
|
| RecordCaptureStatusMessage,
|
||||||
_sender: chrome.runtime.MessageSender,
|
_sender: chrome.runtime.MessageSender,
|
||||||
sendResponse: (response: BorrowResponseMessage | HelpAckMessage | CaptureSuppressAck) => void,
|
sendResponse: (response: BorrowResponseMessage | HelpAckMessage | CaptureSuppressAck) => void,
|
||||||
) => {
|
) => {
|
||||||
@@ -268,6 +278,13 @@ export default defineContentScript({
|
|||||||
return captureSuppress.handleMessage(message, sendResponse);
|
return captureSuppress.handleMessage(message, sendResponse);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (isRecordCaptureStatusMessage(message)) {
|
||||||
|
recordCaptureStatus = message.status;
|
||||||
|
recordCaptureFailures = message.failures ?? [];
|
||||||
|
renderAll();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
if (isRecordContentMessage(message)) {
|
if (isRecordContentMessage(message)) {
|
||||||
const needsAsync = handleRecordContentMessage(
|
const needsAsync = handleRecordContentMessage(
|
||||||
message,
|
message,
|
||||||
@@ -295,6 +312,8 @@ export default defineContentScript({
|
|||||||
},
|
},
|
||||||
onStop: () => {
|
onStop: () => {
|
||||||
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
|
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
|
||||||
|
recordCaptureStatus = "complete";
|
||||||
|
recordCaptureFailures = [];
|
||||||
renderAll();
|
renderAll();
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -449,6 +468,8 @@ export default defineContentScript({
|
|||||||
) {
|
) {
|
||||||
const requestId = recordQuery.requestId;
|
const requestId = recordQuery.requestId;
|
||||||
const startedAtMs = recordQuery.startedAtMs;
|
const startedAtMs = recordQuery.startedAtMs;
|
||||||
|
recordCaptureStatus = recordQuery.captureStatus ?? "complete";
|
||||||
|
recordCaptureFailures = recordQuery.captureFailures ?? [];
|
||||||
overlays.setAgentRecordRequest({
|
overlays.setAgentRecordRequest({
|
||||||
id: requestId,
|
id: requestId,
|
||||||
...(typeof startedAtMs === "number" ? { startedAtMs } : {}),
|
...(typeof startedAtMs === "number" ? { startedAtMs } : {}),
|
||||||
|
|||||||
@@ -0,0 +1,77 @@
|
|||||||
|
import {
|
||||||
|
handleRecordContentMessage,
|
||||||
|
isRecordContentMessage,
|
||||||
|
type RecordCaptureController,
|
||||||
|
} from "@/content/record-capture";
|
||||||
|
import {
|
||||||
|
RECORD_QUERY,
|
||||||
|
type RecordQueryResponse,
|
||||||
|
} from "@/lib/record-bridge";
|
||||||
|
|
||||||
|
/** Capture-only content script for child frames (no React overlay). */
|
||||||
|
export default defineContentScript({
|
||||||
|
matches: ["<all_urls>"],
|
||||||
|
runAt: "document_end",
|
||||||
|
allFrames: true,
|
||||||
|
matchAboutBlank: true,
|
||||||
|
matchOriginAsFallback: true,
|
||||||
|
|
||||||
|
main(ctx) {
|
||||||
|
if (window.top === window) return;
|
||||||
|
|
||||||
|
let recordCapture: RecordCaptureController | null = null;
|
||||||
|
let activeRecordRequestId: string | null = null;
|
||||||
|
|
||||||
|
const onMessage = (
|
||||||
|
message: unknown,
|
||||||
|
_sender: chrome.runtime.MessageSender,
|
||||||
|
sendResponse: (response: unknown) => void,
|
||||||
|
) => {
|
||||||
|
if (isRecordContentMessage(message)) {
|
||||||
|
const needsAsync = handleRecordContentMessage(
|
||||||
|
message,
|
||||||
|
{
|
||||||
|
activeRequestId: activeRecordRequestId,
|
||||||
|
capture: recordCapture,
|
||||||
|
setActiveRequestId: (id) => {
|
||||||
|
activeRecordRequestId = id;
|
||||||
|
},
|
||||||
|
setCapture: (capture) => {
|
||||||
|
recordCapture = capture;
|
||||||
|
},
|
||||||
|
},
|
||||||
|
sendResponse as never,
|
||||||
|
);
|
||||||
|
return needsAsync;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
|
||||||
|
async function queryActiveRecord(): Promise<void> {
|
||||||
|
try {
|
||||||
|
const recordQuery = (await chrome.runtime.sendMessage({
|
||||||
|
type: RECORD_QUERY,
|
||||||
|
})) as RecordQueryResponse | undefined;
|
||||||
|
if (
|
||||||
|
recordQuery?.active &&
|
||||||
|
typeof recordQuery.requestId === "string" &&
|
||||||
|
activeRecordRequestId === null
|
||||||
|
) {
|
||||||
|
// Background rearm sends RECORD_START; nothing to mount locally.
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.debug("[bsk record-frame] record query failed", err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
chrome.runtime.onMessage.addListener(onMessage);
|
||||||
|
void queryActiveRecord();
|
||||||
|
|
||||||
|
ctx.onInvalidated(() => {
|
||||||
|
chrome.runtime.onMessage.removeListener(onMessage);
|
||||||
|
recordCapture?.dispose();
|
||||||
|
recordCapture = null;
|
||||||
|
activeRecordRequestId = null;
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
@@ -113,4 +113,60 @@ describe("matchTarget", () => {
|
|||||||
});
|
});
|
||||||
expect(target.ref).toBe("e1");
|
expect(target.ref).toBe("e1");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("isolates geometry matching to the owner iframe backend node", () => {
|
||||||
|
const sharedRect = { x: 10, y: 20, w: 100, h: 30 };
|
||||||
|
const iframeRefs: RenderedRef[] = [
|
||||||
|
{ ref: "e1", backendNodeId: 42, role: "button", name: "Top", line: 1 },
|
||||||
|
{ ref: "e2", backendNodeId: 99, role: "button", name: "Iframe", line: 2 },
|
||||||
|
];
|
||||||
|
const target = matchTarget({
|
||||||
|
geometry: {
|
||||||
|
rect: sharedRect,
|
||||||
|
scrollX: 0,
|
||||||
|
scrollY: 0,
|
||||||
|
position: "static",
|
||||||
|
tag: "button",
|
||||||
|
ownerFrameBackendNodeId: 13,
|
||||||
|
},
|
||||||
|
captured: [
|
||||||
|
node(42, { rect: sharedRect, documentRect: sharedRect }),
|
||||||
|
node(99, {
|
||||||
|
rect: sharedRect,
|
||||||
|
documentRect: sharedRect,
|
||||||
|
ownerFrameBackendNodeId: 13,
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
refs: iframeRefs,
|
||||||
|
fallback: { tag: "button", name: "Iframe" },
|
||||||
|
});
|
||||||
|
expect(target).toEqual({ ref: "e2", role: "button", name: "Iframe" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("matches iframe static elements using frame scroll and document coordinates", () => {
|
||||||
|
const iframeRefs: RenderedRef[] = [
|
||||||
|
{ ref: "e2", backendNodeId: 99, role: "button", name: "Iframe", line: 2 },
|
||||||
|
];
|
||||||
|
const target = matchTarget({
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 10, y: 50, w: 100, h: 30 },
|
||||||
|
scrollX: 0,
|
||||||
|
scrollY: 120,
|
||||||
|
position: "static",
|
||||||
|
tag: "button",
|
||||||
|
ownerFrameBackendNodeId: 13,
|
||||||
|
},
|
||||||
|
captured: [
|
||||||
|
node(99, {
|
||||||
|
ownerFrameBackendNodeId: 13,
|
||||||
|
documentRect: { x: 10, y: 170, w: 100, h: 30 },
|
||||||
|
localRect: { x: 10, y: 50, w: 100, h: 30 },
|
||||||
|
rect: { x: 10, y: 50, w: 100, h: 30 },
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
refs: iframeRefs,
|
||||||
|
fallback: { tag: "button", name: "Iframe" },
|
||||||
|
});
|
||||||
|
expect(target).toEqual({ ref: "e2", role: "button", name: "Iframe" });
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -33,6 +33,8 @@ function finalizedFillBody(value: string, redactValues: boolean): string {
|
|||||||
obs.lastSettled = {
|
obs.lastSettled = {
|
||||||
stateId,
|
stateId,
|
||||||
captured: [capturedInput()],
|
captured: [capturedInput()],
|
||||||
|
matchNodes: [capturedInput()],
|
||||||
|
documentTokenOwners: new Map(),
|
||||||
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
|
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
|
||||||
url: URL,
|
url: URL,
|
||||||
vomText: rawVomText,
|
vomText: rawVomText,
|
||||||
|
|||||||
@@ -40,9 +40,8 @@ function matchingRects(
|
|||||||
target: { x: number; y: number; w: number; h: number };
|
target: { x: number; y: number; w: number; h: number };
|
||||||
node: { x: number; y: number; w: number; h: number } | null;
|
node: { x: number; y: number; w: number; h: number } | null;
|
||||||
} {
|
} {
|
||||||
const topFrame = (geometry.ownerFrameBackendNodeId ?? null) === null;
|
|
||||||
const viewportPosition = geometry.position === "fixed" || geometry.position === "sticky";
|
const viewportPosition = geometry.position === "fixed" || geometry.position === "sticky";
|
||||||
if (!topFrame || viewportPosition || !node.documentRect) {
|
if (viewportPosition || !node.documentRect) {
|
||||||
return { target: geometry.rect, node: nodeViewportRect(node) };
|
return { target: geometry.rect, node: nodeViewportRect(node) };
|
||||||
}
|
}
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -4,6 +4,7 @@
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
import type { CaptureTargetDescriptor } from "./describe-target";
|
import type { CaptureTargetDescriptor } from "./describe-target";
|
||||||
|
import type { FrameCaptureFailure, FrameCaptureStatus } from "@/transport/types";
|
||||||
|
|
||||||
export const RECORD_START = "bsk-record-start";
|
export const RECORD_START = "bsk-record-start";
|
||||||
export const RECORD_STEP = "bsk-record-step";
|
export const RECORD_STEP = "bsk-record-step";
|
||||||
@@ -11,6 +12,7 @@ export const RECORD_STOP = "bsk-record-stop";
|
|||||||
export const RECORD_CANCEL = "bsk-record-cancel";
|
export const RECORD_CANCEL = "bsk-record-cancel";
|
||||||
export const RECORD_FINISH = "bsk-record-finish";
|
export const RECORD_FINISH = "bsk-record-finish";
|
||||||
export const RECORD_QUERY = "bsk-record-query";
|
export const RECORD_QUERY = "bsk-record-query";
|
||||||
|
export const RECORD_CAPTURE_STATUS = "bsk-record-capture-status";
|
||||||
|
|
||||||
export interface RecordStartAck {
|
export interface RecordStartAck {
|
||||||
ok: true;
|
ok: true;
|
||||||
@@ -27,8 +29,12 @@ export interface RecordQueryResponse {
|
|||||||
requestId?: string;
|
requestId?: string;
|
||||||
/** Epoch ms when the recording began; see `RecordStartMessage.startedAtMs`. */
|
/** Epoch ms when the recording began; see `RecordStartMessage.startedAtMs`. */
|
||||||
startedAtMs?: number;
|
startedAtMs?: number;
|
||||||
|
captureStatus?: FrameCaptureStatus;
|
||||||
|
captureFailures?: FrameCaptureFailure[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type RecordFrameMode = "top" | "child";
|
||||||
|
|
||||||
export interface RecordStartMessage {
|
export interface RecordStartMessage {
|
||||||
type: typeof RECORD_START;
|
type: typeof RECORD_START;
|
||||||
requestId: string;
|
requestId: string;
|
||||||
@@ -38,6 +44,10 @@ export interface RecordStartMessage {
|
|||||||
* content-script remounts instead of restarting per page.
|
* content-script remounts instead of restarting per page.
|
||||||
*/
|
*/
|
||||||
startedAtMs?: number;
|
startedAtMs?: number;
|
||||||
|
/** When true, the top-frame overlay should mount. Child frames omit this. */
|
||||||
|
showOverlay?: boolean;
|
||||||
|
/** Whether this document runs top-frame or child-frame capture semantics. */
|
||||||
|
frameMode?: RecordFrameMode;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RecordStepPayload {
|
export interface RecordStepPayload {
|
||||||
@@ -51,8 +61,14 @@ export interface RecordStepPayload {
|
|||||||
url?: string;
|
url?: string;
|
||||||
redacted?: boolean;
|
redacted?: boolean;
|
||||||
commit?: "enter" | "suggestion" | "blur";
|
commit?: "enter" | "suggestion" | "blur";
|
||||||
/** Page URL when the step was captured. */
|
/** Top-level tab URL when the step was captured. */
|
||||||
page_url?: string;
|
page_url?: string;
|
||||||
|
/** URL of the frame document where the interaction occurred. */
|
||||||
|
frame_url?: string;
|
||||||
|
/** Ephemeral token stamped on the active document root; not persisted in trace steps. */
|
||||||
|
documentToken?: string;
|
||||||
|
/** Epoch ms when the step was captured in the content script. */
|
||||||
|
capturedAtMs?: number;
|
||||||
/** Document geometry at capture time for background geometric matching. */
|
/** Document geometry at capture time for background geometric matching. */
|
||||||
geometry?: {
|
geometry?: {
|
||||||
rect: { x: number; y: number; w: number; h: number };
|
rect: { x: number; y: number; w: number; h: number };
|
||||||
@@ -92,6 +108,12 @@ export interface RecordFinishMessage {
|
|||||||
requestId: string;
|
requestId: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface RecordCaptureStatusMessage {
|
||||||
|
type: typeof RECORD_CAPTURE_STATUS;
|
||||||
|
status: FrameCaptureStatus;
|
||||||
|
failures?: FrameCaptureFailure[];
|
||||||
|
}
|
||||||
|
|
||||||
export function isRecordStartMessage(msg: unknown): msg is RecordStartMessage {
|
export function isRecordStartMessage(msg: unknown): msg is RecordStartMessage {
|
||||||
if (typeof msg !== "object" || msg === null) return false;
|
if (typeof msg !== "object" || msg === null) return false;
|
||||||
const m = msg as Record<string, unknown>;
|
const m = msg as Record<string, unknown>;
|
||||||
@@ -122,6 +144,16 @@ export function isRecordQueryMessage(msg: unknown): msg is RecordQueryMessage {
|
|||||||
return m.type === RECORD_QUERY;
|
return m.type === RECORD_QUERY;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function isRecordCaptureStatusMessage(msg: unknown): msg is RecordCaptureStatusMessage {
|
||||||
|
if (typeof msg !== "object" || msg === null) return false;
|
||||||
|
const m = msg as Record<string, unknown>;
|
||||||
|
return (
|
||||||
|
m.type === RECORD_CAPTURE_STATUS &&
|
||||||
|
(m.status === "complete" || m.status === "partial") &&
|
||||||
|
(m.failures === undefined || Array.isArray(m.failures))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage {
|
export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage {
|
||||||
if (typeof msg !== "object" || msg === null) return false;
|
if (typeof msg !== "object" || msg === null) return false;
|
||||||
const m = msg as Record<string, unknown>;
|
const m = msg as Record<string, unknown>;
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
/** Attribute written on `<html>` while a document is actively recording. */
|
||||||
|
export const RECORD_DOCUMENT_TOKEN_ATTR = "data-bsk-record-document";
|
||||||
|
|
||||||
|
export function createRecordDocumentToken(): string {
|
||||||
|
if (typeof crypto !== "undefined" && typeof crypto.randomUUID === "function") {
|
||||||
|
return crypto.randomUUID();
|
||||||
|
}
|
||||||
|
return `doc-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function stampRecordDocumentToken(token: string): void {
|
||||||
|
document.documentElement.setAttribute(RECORD_DOCUMENT_TOKEN_ATTR, token);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function clearRecordDocumentToken(): void {
|
||||||
|
document.documentElement.removeAttribute(RECORD_DOCUMENT_TOKEN_ATTR);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function readRecordDocumentTokenFromAttrs(
|
||||||
|
attrs: Record<string, string> | undefined,
|
||||||
|
): string | undefined {
|
||||||
|
const token = attrs?.[RECORD_DOCUMENT_TOKEN_ATTR];
|
||||||
|
return token?.trim() ? token : undefined;
|
||||||
|
}
|
||||||
@@ -0,0 +1,250 @@
|
|||||||
|
import {
|
||||||
|
RECORD_CANCEL,
|
||||||
|
RECORD_START,
|
||||||
|
RECORD_STOP,
|
||||||
|
type RecordCancelMessage,
|
||||||
|
type RecordStartAck,
|
||||||
|
type RecordStartMessage,
|
||||||
|
type RecordStopAck,
|
||||||
|
type RecordStopMessage,
|
||||||
|
} from "@/lib/record-bridge";
|
||||||
|
import type { FrameCaptureFailure, FrameCaptureFailureReason } from "@/transport/types";
|
||||||
|
|
||||||
|
export interface WebNavigationFrameInfo {
|
||||||
|
frameId: number;
|
||||||
|
documentId?: string;
|
||||||
|
url?: string;
|
||||||
|
parentFrameId: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ArmedDocument {
|
||||||
|
tabId: number;
|
||||||
|
frameId: number;
|
||||||
|
documentId: string;
|
||||||
|
url?: string;
|
||||||
|
parentFrameId: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RecordFrameSendDeps {
|
||||||
|
sendToDocument(
|
||||||
|
tabId: number,
|
||||||
|
documentId: string,
|
||||||
|
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
|
||||||
|
): Promise<unknown>;
|
||||||
|
getAllFrames(tabId: number): Promise<WebNavigationFrameInfo[]>;
|
||||||
|
}
|
||||||
|
|
||||||
|
const RECORD_START_RETRIES = 3;
|
||||||
|
const RECORD_START_RETRY_DELAY_MS = 500;
|
||||||
|
|
||||||
|
function sleep(ms: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isFrameUrlRestricted(url: string | undefined): boolean {
|
||||||
|
if (!url) return false;
|
||||||
|
const lower = url.toLowerCase();
|
||||||
|
return (
|
||||||
|
lower.startsWith("chrome://") ||
|
||||||
|
lower.startsWith("chrome-extension://") ||
|
||||||
|
lower.startsWith("edge://") ||
|
||||||
|
lower.startsWith("devtools://") ||
|
||||||
|
lower.startsWith("devtools:") ||
|
||||||
|
lower.startsWith("https://chrome.google.com/webstore")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function isRecordStartAck(response: unknown): response is RecordStartAck {
|
||||||
|
return (
|
||||||
|
typeof response === "object" &&
|
||||||
|
response !== null &&
|
||||||
|
"ok" in response &&
|
||||||
|
(response as RecordStartAck).ok === true
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function isRecordStopAckOk(response: unknown): boolean {
|
||||||
|
return (
|
||||||
|
typeof response === "object" &&
|
||||||
|
response !== null &&
|
||||||
|
"ok" in response &&
|
||||||
|
(response as RecordStopAck).ok === true
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function makeFrameCaptureFailure(
|
||||||
|
reason: FrameCaptureFailureReason,
|
||||||
|
input: { frameId: number; documentId?: string; url?: string; detail?: string },
|
||||||
|
): FrameCaptureFailure {
|
||||||
|
return {
|
||||||
|
reason,
|
||||||
|
frame_id: input.frameId,
|
||||||
|
...(input.documentId ? { document_id: input.documentId } : {}),
|
||||||
|
...(input.url ? { url: input.url } : {}),
|
||||||
|
...(input.detail ? { detail: input.detail } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function listInjectableFrames(
|
||||||
|
tabId: number,
|
||||||
|
deps: RecordFrameSendDeps,
|
||||||
|
): Promise<{ injectable: WebNavigationFrameInfo[]; failures: FrameCaptureFailure[] }> {
|
||||||
|
const failures: FrameCaptureFailure[] = [];
|
||||||
|
let frames: WebNavigationFrameInfo[] = [];
|
||||||
|
try {
|
||||||
|
frames = await deps.getAllFrames(tabId);
|
||||||
|
} catch (err) {
|
||||||
|
failures.push(
|
||||||
|
makeFrameCaptureFailure("arm_failed", {
|
||||||
|
frameId: 0,
|
||||||
|
detail: err instanceof Error ? err.message : "webNavigation.getAllFrames failed",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return { injectable: [], failures };
|
||||||
|
}
|
||||||
|
|
||||||
|
const injectable = frames.filter((frame) => {
|
||||||
|
if (!frame.documentId) {
|
||||||
|
failures.push(
|
||||||
|
makeFrameCaptureFailure("not_injectable", {
|
||||||
|
frameId: frame.frameId,
|
||||||
|
url: frame.url,
|
||||||
|
detail: "missing documentId",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (isFrameUrlRestricted(frame.url)) {
|
||||||
|
failures.push(
|
||||||
|
makeFrameCaptureFailure("not_injectable", {
|
||||||
|
frameId: frame.frameId,
|
||||||
|
documentId: frame.documentId,
|
||||||
|
url: frame.url,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
return { injectable, failures };
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function sendRecordStartWithAckToDocument(
|
||||||
|
tabId: number,
|
||||||
|
documentId: string,
|
||||||
|
msg: RecordStartMessage,
|
||||||
|
deps: RecordFrameSendDeps,
|
||||||
|
): Promise<void> {
|
||||||
|
let lastError: unknown;
|
||||||
|
for (let attempt = 0; attempt < RECORD_START_RETRIES; attempt += 1) {
|
||||||
|
try {
|
||||||
|
const response = await deps.sendToDocument(tabId, documentId, msg);
|
||||||
|
if (isRecordStartAck(response)) return;
|
||||||
|
lastError = new Error("content script did not ack RECORD_START");
|
||||||
|
} catch (err) {
|
||||||
|
lastError = err;
|
||||||
|
}
|
||||||
|
if (attempt + 1 < RECORD_START_RETRIES) {
|
||||||
|
await sleep(RECORD_START_RETRY_DELAY_MS);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw lastError ?? new Error("failed to start recording in content script");
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function armDocumentCapture(
|
||||||
|
tabId: number,
|
||||||
|
frame: WebNavigationFrameInfo,
|
||||||
|
startMsg: RecordStartMessage,
|
||||||
|
deps: RecordFrameSendDeps,
|
||||||
|
): Promise<{ armed?: ArmedDocument; failure?: FrameCaptureFailure }> {
|
||||||
|
if (!frame.documentId) {
|
||||||
|
return {
|
||||||
|
failure: makeFrameCaptureFailure("not_injectable", {
|
||||||
|
frameId: frame.frameId,
|
||||||
|
url: frame.url,
|
||||||
|
detail: "missing documentId",
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (isFrameUrlRestricted(frame.url)) {
|
||||||
|
return {
|
||||||
|
failure: makeFrameCaptureFailure("not_injectable", {
|
||||||
|
frameId: frame.frameId,
|
||||||
|
documentId: frame.documentId,
|
||||||
|
url: frame.url,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await sendRecordStartWithAckToDocument(tabId, frame.documentId, startMsg, deps);
|
||||||
|
return {
|
||||||
|
armed: {
|
||||||
|
tabId,
|
||||||
|
frameId: frame.frameId,
|
||||||
|
documentId: frame.documentId,
|
||||||
|
url: frame.url,
|
||||||
|
parentFrameId: frame.parentFrameId,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
} catch (err) {
|
||||||
|
return {
|
||||||
|
failure: makeFrameCaptureFailure("arm_failed", {
|
||||||
|
frameId: frame.frameId,
|
||||||
|
documentId: frame.documentId,
|
||||||
|
url: frame.url,
|
||||||
|
detail: err instanceof Error ? err.message : "RECORD_START failed",
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function armAllDocumentsForTab(
|
||||||
|
tabId: number,
|
||||||
|
startMsg: RecordStartMessage,
|
||||||
|
deps: RecordFrameSendDeps,
|
||||||
|
): Promise<{ armed: ArmedDocument[]; failures: FrameCaptureFailure[] }> {
|
||||||
|
const { injectable, failures } = await listInjectableFrames(tabId, deps);
|
||||||
|
const armed: ArmedDocument[] = [];
|
||||||
|
for (const frame of injectable) {
|
||||||
|
const result = await armDocumentCapture(tabId, frame, startMsg, deps);
|
||||||
|
if (result.armed) armed.push(result.armed);
|
||||||
|
if (result.failure) failures.push(result.failure);
|
||||||
|
}
|
||||||
|
return { armed, failures };
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function stopDocumentCapture(
|
||||||
|
armed: ArmedDocument,
|
||||||
|
stopMsg: RecordStopMessage,
|
||||||
|
deps: RecordFrameSendDeps,
|
||||||
|
): Promise<{ ok: true } | FrameCaptureFailure> {
|
||||||
|
try {
|
||||||
|
const response = await deps.sendToDocument(armed.tabId, armed.documentId, stopMsg);
|
||||||
|
if (isRecordStopAckOk(response)) return { ok: true };
|
||||||
|
return makeFrameCaptureFailure("flush_failed", {
|
||||||
|
frameId: armed.frameId,
|
||||||
|
documentId: armed.documentId,
|
||||||
|
url: armed.url,
|
||||||
|
detail: "content script did not confirm recorded steps",
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
return makeFrameCaptureFailure("flush_failed", {
|
||||||
|
frameId: armed.frameId,
|
||||||
|
documentId: armed.documentId,
|
||||||
|
url: armed.url,
|
||||||
|
detail: err instanceof Error ? err.message : "RECORD_STOP failed",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function cancelDocumentCapture(
|
||||||
|
armed: ArmedDocument,
|
||||||
|
cancelMsg: RecordCancelMessage,
|
||||||
|
deps: RecordFrameSendDeps,
|
||||||
|
): Promise<void> {
|
||||||
|
try {
|
||||||
|
await deps.sendToDocument(armed.tabId, armed.documentId, cancelMsg);
|
||||||
|
} catch {
|
||||||
|
// Best-effort cleanup.
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -17,6 +17,8 @@ import { reduceTraceSteps, resolveTraceStartUrl } from "./trace-reducer";
|
|||||||
export interface LastSettledObservation {
|
export interface LastSettledObservation {
|
||||||
stateId: string;
|
stateId: string;
|
||||||
captured: CapturedNode[];
|
captured: CapturedNode[];
|
||||||
|
matchNodes: CapturedNode[];
|
||||||
|
documentTokenOwners: Map<string, number | null>;
|
||||||
refs: RenderedRef[];
|
refs: RenderedRef[];
|
||||||
url: string;
|
url: string;
|
||||||
title?: string;
|
title?: string;
|
||||||
@@ -40,6 +42,8 @@ export interface PendingRedirectLanding {
|
|||||||
export interface RecordingObservationState {
|
export interface RecordingObservationState {
|
||||||
stateRegistry: Map<string, StateRegistryEntry>;
|
stateRegistry: Map<string, StateRegistryEntry>;
|
||||||
lastSettled: LastSettledObservation | null;
|
lastSettled: LastSettledObservation | null;
|
||||||
|
/** Latest token→owner map from the most recent observation capture. */
|
||||||
|
documentTokenOwners: Map<string, number | null>;
|
||||||
maxPageTokens: number;
|
maxPageTokens: number;
|
||||||
redactValues: boolean;
|
redactValues: boolean;
|
||||||
lastCaptureAtMs: number;
|
lastCaptureAtMs: number;
|
||||||
@@ -67,6 +71,7 @@ export function createObservationState(options?: {
|
|||||||
return {
|
return {
|
||||||
stateRegistry: new Map(),
|
stateRegistry: new Map(),
|
||||||
lastSettled: null,
|
lastSettled: null,
|
||||||
|
documentTokenOwners: new Map(),
|
||||||
maxPageTokens: options?.maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS,
|
maxPageTokens: options?.maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS,
|
||||||
redactValues: options?.redactValues ?? false,
|
redactValues: options?.redactValues ?? false,
|
||||||
lastCaptureAtMs: 0,
|
lastCaptureAtMs: 0,
|
||||||
@@ -125,9 +130,12 @@ export async function captureAndRegisterObservation(
|
|||||||
});
|
});
|
||||||
|
|
||||||
obs.lastCaptureAtMs = Date.now();
|
obs.lastCaptureAtMs = Date.now();
|
||||||
|
obs.documentTokenOwners = rendered.documentTokenOwners;
|
||||||
const settled: LastSettledObservation = {
|
const settled: LastSettledObservation = {
|
||||||
stateId,
|
stateId,
|
||||||
captured: rendered.captured,
|
captured: rendered.captured,
|
||||||
|
matchNodes: rendered.matchNodes,
|
||||||
|
documentTokenOwners: rendered.documentTokenOwners,
|
||||||
refs: rendered.refs,
|
refs: rendered.refs,
|
||||||
url,
|
url,
|
||||||
title: resolvedTitle,
|
title: resolvedTitle,
|
||||||
@@ -164,7 +172,7 @@ export function applyTargetMatching(
|
|||||||
"geometry" in draft && draft.geometry
|
"geometry" in draft && draft.geometry
|
||||||
? matchTarget({
|
? matchTarget({
|
||||||
geometry: draft.geometry,
|
geometry: draft.geometry,
|
||||||
captured: obs.lastSettled.captured,
|
captured: obs.lastSettled.matchNodes,
|
||||||
refs: obs.lastSettled.refs,
|
refs: obs.lastSettled.refs,
|
||||||
fallback: draft.captureTarget,
|
fallback: draft.captureTarget,
|
||||||
})
|
})
|
||||||
@@ -636,6 +644,7 @@ export function buildTraceV3(input: {
|
|||||||
startUrl?: string;
|
startUrl?: string;
|
||||||
stoppedBy: StopReason;
|
stoppedBy: StopReason;
|
||||||
bskVersion: string;
|
bskVersion: string;
|
||||||
|
frameCapture?: Trace["frame_capture"];
|
||||||
}): Trace {
|
}): Trace {
|
||||||
const { steps, stepIdByDraftId } = reduceTraceSteps(input.steps, input.obs.stateRegistry);
|
const { steps, stepIdByDraftId } = reduceTraceSteps(input.steps, input.obs.stateRegistry);
|
||||||
const published = selectPublishedStates(input.obs.stateRegistry, steps);
|
const published = selectPublishedStates(input.obs.stateRegistry, steps);
|
||||||
@@ -657,6 +666,7 @@ export function buildTraceV3(input: {
|
|||||||
stopped_by: input.stoppedBy,
|
stopped_by: input.stoppedBy,
|
||||||
entry: { start_url: startUrl },
|
entry: { start_url: startUrl },
|
||||||
recorder: { bsk: input.bskVersion, vom: 1 },
|
recorder: { bsk: input.bskVersion, vom: 1 },
|
||||||
|
...(input.frameCapture ? { frame_capture: input.frameCapture } : {}),
|
||||||
states,
|
states,
|
||||||
steps,
|
steps,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -0,0 +1,86 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { RECORD_DOCUMENT_TOKEN_ATTR } from "@/lib/record-document-token";
|
||||||
|
import type { CapturedNode } from "@/tools/vom/capture";
|
||||||
|
import { collectDocumentTokenOwners } from "../capture-vom-observation";
|
||||||
|
|
||||||
|
function htmlNode(
|
||||||
|
backendNodeId: number,
|
||||||
|
token: string,
|
||||||
|
ownerFrameBackendNodeId: number | null = null,
|
||||||
|
): CapturedNode {
|
||||||
|
return {
|
||||||
|
backendNodeId,
|
||||||
|
parentBackendNodeId: null,
|
||||||
|
tag: "html",
|
||||||
|
attrs: { [RECORD_DOCUMENT_TOKEN_ATTR]: token },
|
||||||
|
rect: { x: 0, y: 0, w: 1280, h: 720 },
|
||||||
|
paintOrder: 0,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
...(ownerFrameBackendNodeId === null ? {} : { ownerFrameBackendNodeId }),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("collectDocumentTokenOwners", () => {
|
||||||
|
it("maps document tokens to their immediate iframe owner backend node", () => {
|
||||||
|
const topToken = "top-doc-token";
|
||||||
|
const childToken = "child-doc-token";
|
||||||
|
const owners = collectDocumentTokenOwners({
|
||||||
|
viewport: { width: 1280, height: 720 },
|
||||||
|
nodes: [htmlNode(1, topToken, null)],
|
||||||
|
iframeNodes: new Map([
|
||||||
|
[
|
||||||
|
13,
|
||||||
|
[
|
||||||
|
htmlNode(101, childToken, 13),
|
||||||
|
{
|
||||||
|
backendNodeId: 102,
|
||||||
|
parentBackendNodeId: 101,
|
||||||
|
tag: "input",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 10, y: 20, w: 200, h: 30 },
|
||||||
|
paintOrder: 1,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
ownerFrameBackendNodeId: 13,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
]),
|
||||||
|
excludedBackendNodeIds: new Set(),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(owners.get(topToken)).toBeNull();
|
||||||
|
expect(owners.get(childToken)).toBe(13);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps nested iframe documents to their direct owner, not the outer iframe", () => {
|
||||||
|
const nestedToken = "nested-doc-token";
|
||||||
|
const owners = collectDocumentTokenOwners({
|
||||||
|
viewport: { width: 1280, height: 720 },
|
||||||
|
nodes: [htmlNode(1, "top-token", null)],
|
||||||
|
iframeNodes: new Map([
|
||||||
|
[
|
||||||
|
13,
|
||||||
|
[
|
||||||
|
{
|
||||||
|
backendNodeId: 23,
|
||||||
|
parentBackendNodeId: 101,
|
||||||
|
tag: "iframe",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 0, y: 0, w: 400, h: 300 },
|
||||||
|
paintOrder: 1,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
ownerFrameBackendNodeId: 13,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
[23, [htmlNode(201, nestedToken, 23)]],
|
||||||
|
]),
|
||||||
|
excludedBackendNodeIds: new Set(),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(owners.get(nestedToken)).toBe(23);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -18,11 +18,13 @@ import {
|
|||||||
handleRecordStart,
|
handleRecordStart,
|
||||||
handleRecordStop,
|
handleRecordStop,
|
||||||
resetBrowserObservationForTests,
|
resetBrowserObservationForTests,
|
||||||
|
type RecordDeps,
|
||||||
} from "../record";
|
} from "../record";
|
||||||
|
|
||||||
const AGENT_WINDOW_ID = 100;
|
const AGENT_WINDOW_ID = 100;
|
||||||
const TAB_ID = 4;
|
const TAB_ID = 4;
|
||||||
const START_URL = "https://example.com/";
|
const START_URL = "https://example.com/";
|
||||||
|
const TOP_DOCUMENT_ID = "doc-top";
|
||||||
|
|
||||||
type RuntimeListener = (
|
type RuntimeListener = (
|
||||||
message: unknown,
|
message: unknown,
|
||||||
@@ -63,9 +65,18 @@ function installChrome() {
|
|||||||
webNavigation: {
|
webNavigation: {
|
||||||
onCompleted: webNavigationOnCompleted,
|
onCompleted: webNavigationOnCompleted,
|
||||||
onCommitted: webNavigationOnCommitted,
|
onCommitted: webNavigationOnCommitted,
|
||||||
|
getAllFrames: vi.fn(async ({ tabId }: { tabId: number }) => [
|
||||||
|
{
|
||||||
|
frameId: 0,
|
||||||
|
documentId: TOP_DOCUMENT_ID,
|
||||||
|
url: START_URL,
|
||||||
|
parentFrameId: -1,
|
||||||
|
tabId,
|
||||||
|
},
|
||||||
|
]),
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
return { runtimeOnMessage, webNavigationOnCompleted, webNavigationOnCommitted };
|
return { runtimeOnMessage, webNavigationOnCompleted, webNavigationOnCommitted, TOP_DOCUMENT_ID };
|
||||||
}
|
}
|
||||||
|
|
||||||
function fakeManager() {
|
function fakeManager() {
|
||||||
@@ -206,6 +217,57 @@ function makeTabsApi() {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function makeRecordDeps(
|
||||||
|
tabsApi: ReturnType<typeof makeTabsApi> | ReturnType<typeof makeMultiTabsApi>,
|
||||||
|
sendToTabMock: (tabId: number, msg: unknown) => Promise<unknown>,
|
||||||
|
cdp?: ReturnType<typeof makeFakeCdp>,
|
||||||
|
): RecordDeps {
|
||||||
|
return {
|
||||||
|
tabsApi,
|
||||||
|
sendToTab: sendToTabMock as RecordDeps["sendToTab"],
|
||||||
|
sendToDocument: (async (tabId, _documentId, msg) =>
|
||||||
|
sendToTabMock(tabId, msg)) as RecordDeps["sendToDocument"],
|
||||||
|
getAllFrames: (async () => [
|
||||||
|
{
|
||||||
|
frameId: 0,
|
||||||
|
documentId: TOP_DOCUMENT_ID,
|
||||||
|
url: START_URL,
|
||||||
|
parentFrameId: -1,
|
||||||
|
},
|
||||||
|
]) as RecordDeps["getAllFrames"],
|
||||||
|
...(cdp ? { cdp } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function makeMultiTabRecordDeps(
|
||||||
|
tabsApi: ReturnType<typeof makeMultiTabsApi>,
|
||||||
|
sendToTabMock: (tabId: number, msg: unknown) => Promise<unknown>,
|
||||||
|
tabIds: number[],
|
||||||
|
cdp?: ReturnType<typeof makeFakeCdp>,
|
||||||
|
): RecordDeps {
|
||||||
|
const framesByTab = new Map(
|
||||||
|
tabIds.map((tabId) => [
|
||||||
|
tabId,
|
||||||
|
[
|
||||||
|
{
|
||||||
|
frameId: 0,
|
||||||
|
documentId: tabId === TAB_ID ? TOP_DOCUMENT_ID : `doc-tab-${tabId}`,
|
||||||
|
url: tabId === TAB_ID ? START_URL : `https://example.com/tab-${tabId}`,
|
||||||
|
parentFrameId: -1,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
tabsApi,
|
||||||
|
sendToTab: sendToTabMock as RecordDeps["sendToTab"],
|
||||||
|
sendToDocument: (async (tabId, _documentId, msg) =>
|
||||||
|
sendToTabMock(tabId, msg)) as RecordDeps["sendToDocument"],
|
||||||
|
getAllFrames: (async (tabId) => framesByTab.get(tabId) ?? []) as RecordDeps["getAllFrames"],
|
||||||
|
...(cdp ? { cdp } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
function makeMultiTabsApi(tabs: chrome.tabs.Tab[]) {
|
function makeMultiTabsApi(tabs: chrome.tabs.Tab[]) {
|
||||||
const byId = new Map(tabs.flatMap((tab) => (typeof tab.id === "number" ? [[tab.id, tab]] : [])));
|
const byId = new Map(tabs.flatMap((tab) => (typeof tab.id === "number" ? [[tab.id, tab]] : [])));
|
||||||
return {
|
return {
|
||||||
@@ -1171,25 +1233,25 @@ describe("recorded user steps reach the exported trace", () => {
|
|||||||
}
|
}
|
||||||
return { ok: true };
|
return { ok: true };
|
||||||
});
|
});
|
||||||
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
|
const deps = makeMultiTabRecordDeps(tabsApi, sendToTab, [TAB_ID, 5], makeFakeCdp());
|
||||||
|
|
||||||
await handleRecordStart(manager, { session_id: "abcd", url: START_URL }, deps);
|
await handleRecordStart(manager, { session_id: "abcd", url: START_URL }, deps);
|
||||||
attachRecordQueryListener(deps);
|
attachRecordQueryListener(deps);
|
||||||
await new Promise<void>((resolve) => {
|
await new Promise<void>((resolve) => {
|
||||||
chromeApi.runtimeOnMessage.emit(
|
chromeApi.runtimeOnMessage.emit(
|
||||||
{ type: RECORD_QUERY },
|
{ type: RECORD_QUERY },
|
||||||
{ tab: secondTab } as chrome.runtime.MessageSender,
|
{ tab: secondTab, documentId: "doc-tab-5" } as chrome.runtime.MessageSender,
|
||||||
() => resolve(),
|
() => resolve(),
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
|
|
||||||
const firstStop = await handleRecordStop(manager, { session_id: "abcd" }, deps);
|
const firstStop = await handleRecordStop(manager, { session_id: "abcd" }, deps);
|
||||||
expect(firstStop).toMatchObject({ code: "protocol_error" });
|
expect((firstStop as RecordStopResult).trace.version).toBe(3);
|
||||||
|
expect((firstStop as RecordStopResult).trace.frame_capture?.status).toBe("partial");
|
||||||
expect(stopTabs).toEqual([TAB_ID, 5]);
|
expect(stopTabs).toEqual([TAB_ID, 5]);
|
||||||
|
|
||||||
const retry = await handleRecordStop(manager, { session_id: "abcd" }, deps);
|
const retry = await handleRecordStop(manager, { session_id: "abcd" }, deps);
|
||||||
expect((retry as RecordStopResult).trace.version).toBe(3);
|
expect(retry).toMatchObject({ code: "not_found" });
|
||||||
expect(stopTabs).toEqual([TAB_ID, 5, 5]);
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not let a closed armed tab block finish", async () => {
|
it("does not let a closed armed tab block finish", async () => {
|
||||||
@@ -1215,14 +1277,14 @@ describe("recorded user steps reach the exported trace", () => {
|
|||||||
if ((msg as { type?: string }).type === RECORD_STOP) stopTabs.push(tabId);
|
if ((msg as { type?: string }).type === RECORD_STOP) stopTabs.push(tabId);
|
||||||
return { ok: true };
|
return { ok: true };
|
||||||
});
|
});
|
||||||
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
|
const deps = makeMultiTabRecordDeps(tabsApi, sendToTab, [TAB_ID, 5], makeFakeCdp());
|
||||||
|
|
||||||
await handleRecordStart(manager, { session_id: "abcd", url: START_URL }, deps);
|
await handleRecordStart(manager, { session_id: "abcd", url: START_URL }, deps);
|
||||||
attachRecordQueryListener(deps);
|
attachRecordQueryListener(deps);
|
||||||
await new Promise<void>((resolve) => {
|
await new Promise<void>((resolve) => {
|
||||||
chromeApi.runtimeOnMessage.emit(
|
chromeApi.runtimeOnMessage.emit(
|
||||||
{ type: RECORD_QUERY },
|
{ type: RECORD_QUERY },
|
||||||
{ tab: secondTab } as chrome.runtime.MessageSender,
|
{ tab: secondTab, documentId: "doc-tab-5" } as chrome.runtime.MessageSender,
|
||||||
() => resolve(),
|
() => resolve(),
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
@@ -1231,7 +1293,7 @@ describe("recorded user steps reach the exported trace", () => {
|
|||||||
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
|
const stopped = await handleRecordStop(manager, { session_id: "abcd" }, deps);
|
||||||
|
|
||||||
expect((stopped as RecordStopResult).trace.version).toBe(3);
|
expect((stopped as RecordStopResult).trace.version).toBe(3);
|
||||||
expect(stopTabs).toEqual([TAB_ID]);
|
expect(stopTabs).toContain(TAB_ID);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("waits for an acknowledged in-flight rearm before stopping armed tabs", async () => {
|
it("waits for an acknowledged in-flight rearm before stopping armed tabs", async () => {
|
||||||
@@ -1270,13 +1332,13 @@ describe("recorded user steps reach the exported trace", () => {
|
|||||||
if (type === RECORD_STOP) stopTabs.push(tabId);
|
if (type === RECORD_STOP) stopTabs.push(tabId);
|
||||||
return { ok: true };
|
return { ok: true };
|
||||||
});
|
});
|
||||||
const deps = { tabsApi, sendToTab, cdp: makeFakeCdp() };
|
const deps = makeMultiTabRecordDeps(tabsApi, sendToTab, [TAB_ID, 5], makeFakeCdp());
|
||||||
|
|
||||||
await handleRecordStart(manager, { session_id: "abcd", url: START_URL }, deps);
|
await handleRecordStart(manager, { session_id: "abcd", url: START_URL }, deps);
|
||||||
attachRecordQueryListener(deps);
|
attachRecordQueryListener(deps);
|
||||||
chromeApi.runtimeOnMessage.emit(
|
chromeApi.runtimeOnMessage.emit(
|
||||||
{ type: RECORD_QUERY },
|
{ type: RECORD_QUERY },
|
||||||
{ tab: secondTab } as chrome.runtime.MessageSender,
|
{ tab: secondTab, documentId: "doc-tab-5" } as chrome.runtime.MessageSender,
|
||||||
() => {},
|
() => {},
|
||||||
);
|
);
|
||||||
await secondStartBegan;
|
await secondStartBegan;
|
||||||
|
|||||||
@@ -1,4 +1,5 @@
|
|||||||
import { type RenderedRef, renderVom, type VomOptions } from "@browser-skill/vom";
|
import { type RenderedRef, renderVom, type VomOptions } from "@browser-skill/vom";
|
||||||
|
import { readRecordDocumentTokenFromAttrs } from "@/lib/record-document-token";
|
||||||
import type { CdpAxNode } from "./observation";
|
import type { CdpAxNode } from "./observation";
|
||||||
import { buildVomScene } from "./observation";
|
import { buildVomScene } from "./observation";
|
||||||
import type { CdpRunner } from "./shared";
|
import type { CdpRunner } from "./shared";
|
||||||
@@ -20,9 +21,28 @@ export interface CaptureVomObservationResult {
|
|||||||
refs: RenderedRef[];
|
refs: RenderedRef[];
|
||||||
truncated: boolean;
|
truncated: boolean;
|
||||||
captured: CapturedNode[];
|
captured: CapturedNode[];
|
||||||
|
/** All nodes including iframe sub-documents, for geometric target matching. */
|
||||||
|
matchNodes: CapturedNode[];
|
||||||
|
/** Maps ephemeral record document tokens to immediate iframe owner backend ids. */
|
||||||
|
documentTokenOwners: Map<string, number | null>;
|
||||||
surfaceProbes?: CapturedSurfaceProbe[];
|
surfaceProbes?: CapturedSurfaceProbe[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function flattenCapturedNodes(vm: CapturedViewModel): CapturedNode[] {
|
||||||
|
return [...vm.nodes, ...[...vm.iframeNodes.values()].flat()];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function collectDocumentTokenOwners(vm: CapturedViewModel): Map<string, number | null> {
|
||||||
|
const owners = new Map<string, number | null>();
|
||||||
|
for (const node of flattenCapturedNodes(vm)) {
|
||||||
|
if (node.tag !== "html") continue;
|
||||||
|
const token = readRecordDocumentTokenFromAttrs(node.attrs);
|
||||||
|
if (!token) continue;
|
||||||
|
owners.set(token, node.ownerFrameBackendNodeId ?? null);
|
||||||
|
}
|
||||||
|
return owners;
|
||||||
|
}
|
||||||
|
|
||||||
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
|
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
|
||||||
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
|
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
|
||||||
}
|
}
|
||||||
@@ -91,11 +111,14 @@ export async function captureVomObservation(
|
|||||||
redactValues: opts.redactValues,
|
redactValues: opts.redactValues,
|
||||||
activeRegionPolicy: opts.activeRegionPolicy ?? true,
|
activeRegionPolicy: opts.activeRegionPolicy ?? true,
|
||||||
});
|
});
|
||||||
|
const matchNodes = flattenCapturedNodes(captured);
|
||||||
return {
|
return {
|
||||||
text: rendered.text,
|
text: rendered.text,
|
||||||
refs: rendered.refs,
|
refs: rendered.refs,
|
||||||
truncated: rendered.truncated,
|
truncated: rendered.truncated,
|
||||||
captured: captured.nodes,
|
captured: captured.nodes,
|
||||||
|
matchNodes,
|
||||||
|
documentTokenOwners: collectDocumentTokenOwners(captured),
|
||||||
surfaceProbes: captured.surfaceProbes,
|
surfaceProbes: captured.surfaceProbes,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,16 +7,28 @@ import {
|
|||||||
isRecordQueryMessage,
|
isRecordQueryMessage,
|
||||||
isRecordStepMessage,
|
isRecordStepMessage,
|
||||||
RECORD_CANCEL,
|
RECORD_CANCEL,
|
||||||
|
RECORD_CAPTURE_STATUS,
|
||||||
RECORD_START,
|
RECORD_START,
|
||||||
RECORD_STEP,
|
RECORD_STEP,
|
||||||
RECORD_STOP,
|
RECORD_STOP,
|
||||||
type RecordCancelMessage,
|
type RecordCancelMessage,
|
||||||
|
type RecordCaptureStatusMessage,
|
||||||
type RecordFinishMessage,
|
type RecordFinishMessage,
|
||||||
type RecordQueryResponse,
|
type RecordQueryResponse,
|
||||||
type RecordStartAck,
|
type RecordStartAck,
|
||||||
type RecordStartMessage,
|
type RecordStartMessage,
|
||||||
|
type RecordStepPayload,
|
||||||
type RecordStopMessage,
|
type RecordStopMessage,
|
||||||
} from "@/lib/record-bridge";
|
} from "@/lib/record-bridge";
|
||||||
|
import {
|
||||||
|
armAllDocumentsForTab,
|
||||||
|
armDocumentCapture,
|
||||||
|
cancelDocumentCapture,
|
||||||
|
type ArmedDocument,
|
||||||
|
listInjectableFrames,
|
||||||
|
type RecordFrameSendDeps,
|
||||||
|
stopDocumentCapture,
|
||||||
|
} from "@/lib/record-frame-arming";
|
||||||
import { DEFAULT_MAX_PAGE_TOKENS } from "@/lib/record-constants";
|
import { DEFAULT_MAX_PAGE_TOKENS } from "@/lib/record-constants";
|
||||||
import {
|
import {
|
||||||
applyTargetMatching,
|
applyTargetMatching,
|
||||||
@@ -39,6 +51,9 @@ import type { SessionManager } from "@/session-manager/manager";
|
|||||||
import { EXTENSION_VERSION } from "@/transport/handshake";
|
import { EXTENSION_VERSION } from "@/transport/handshake";
|
||||||
import type {
|
import type {
|
||||||
DraftTraceStep,
|
DraftTraceStep,
|
||||||
|
FrameCaptureFailure,
|
||||||
|
FrameCaptureInfo,
|
||||||
|
FrameCaptureStatus,
|
||||||
RecordAwaitParams,
|
RecordAwaitParams,
|
||||||
RecordAwaitResult,
|
RecordAwaitResult,
|
||||||
RecordStartParams,
|
RecordStartParams,
|
||||||
@@ -80,8 +95,12 @@ interface ActiveRecording {
|
|||||||
stoppedBy: StopReason;
|
stoppedBy: StopReason;
|
||||||
maxPageTokens: number;
|
maxPageTokens: number;
|
||||||
redactValues: boolean;
|
redactValues: boolean;
|
||||||
/** Tabs whose content capture acknowledged START and still need STOP. */
|
/** Documents whose capture acknowledged START and still need STOP. */
|
||||||
armedTabIds: Set<number>;
|
armedDocuments: Map<string, ArmedDocument>;
|
||||||
|
frameCaptureFailures: FrameCaptureFailure[];
|
||||||
|
frameCaptureStatus: FrameCaptureStatus;
|
||||||
|
/** When false, ignore subframe rearm requests during finish. */
|
||||||
|
acceptingNewFrames: boolean;
|
||||||
/** Rearms already sending START when finish begins. */
|
/** Rearms already sending START when finish begins. */
|
||||||
rearmCallbacks: Set<Promise<void>>;
|
rearmCallbacks: Set<Promise<void>>;
|
||||||
/** Navigation callbacks tracked from event receipt through action enqueue. */
|
/** Navigation callbacks tracked from event receipt through action enqueue. */
|
||||||
@@ -203,6 +222,15 @@ async function sendRecordStartWithAck(
|
|||||||
}
|
}
|
||||||
|
|
||||||
function buildTrace(recording: ActiveRecording): Trace {
|
function buildTrace(recording: ActiveRecording): Trace {
|
||||||
|
const frameCapture: FrameCaptureInfo | undefined =
|
||||||
|
recording.frameCaptureFailures.length > 0 || recording.frameCaptureStatus === "partial"
|
||||||
|
? {
|
||||||
|
status: recording.frameCaptureStatus,
|
||||||
|
...(recording.frameCaptureFailures.length > 0
|
||||||
|
? { failures: recording.frameCaptureFailures }
|
||||||
|
: {}),
|
||||||
|
}
|
||||||
|
: undefined;
|
||||||
return buildTraceV3({
|
return buildTraceV3({
|
||||||
obs: recording.observation,
|
obs: recording.observation,
|
||||||
steps: recording.steps,
|
steps: recording.steps,
|
||||||
@@ -211,9 +239,51 @@ function buildTrace(recording: ActiveRecording): Trace {
|
|||||||
startUrl: recording.startUrl,
|
startUrl: recording.startUrl,
|
||||||
stoppedBy: recording.stoppedBy,
|
stoppedBy: recording.stoppedBy,
|
||||||
bskVersion: BSK_TRACE_VERSION,
|
bskVersion: BSK_TRACE_VERSION,
|
||||||
|
...(frameCapture ? { frameCapture } : {}),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function recordFrameFailure(recording: ActiveRecording, failure: FrameCaptureFailure): void {
|
||||||
|
recording.frameCaptureFailures.push(failure);
|
||||||
|
recording.frameCaptureStatus = "partial";
|
||||||
|
}
|
||||||
|
|
||||||
|
function enrichRecordedStep(
|
||||||
|
recording: ActiveRecording,
|
||||||
|
step: RecordStepPayload,
|
||||||
|
): RecordStepPayload {
|
||||||
|
const enriched: RecordStepPayload = { ...step };
|
||||||
|
if (enriched.geometry && enriched.documentToken) {
|
||||||
|
const owner = recording.observation.documentTokenOwners.get(enriched.documentToken);
|
||||||
|
if (owner !== undefined) {
|
||||||
|
enriched.geometry = { ...enriched.geometry, ownerFrameBackendNodeId: owner };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!enriched.page_url && recording.currentUrl) {
|
||||||
|
enriched.page_url = recording.currentUrl;
|
||||||
|
}
|
||||||
|
return enriched;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function publishCaptureStatus(
|
||||||
|
recording: ActiveRecording,
|
||||||
|
deps: RecordDeps,
|
||||||
|
): Promise<void> {
|
||||||
|
if (!deps.broadcastCaptureStatus) return;
|
||||||
|
const msg: RecordCaptureStatusMessage = {
|
||||||
|
type: RECORD_CAPTURE_STATUS,
|
||||||
|
status: recording.frameCaptureStatus,
|
||||||
|
...(recording.frameCaptureFailures.length > 0
|
||||||
|
? { failures: recording.frameCaptureFailures }
|
||||||
|
: {}),
|
||||||
|
};
|
||||||
|
try {
|
||||||
|
await deps.broadcastCaptureStatus(recording.tabId, msg);
|
||||||
|
} catch {
|
||||||
|
// Overlay is best-effort.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function processRecordedStep(
|
async function processRecordedStep(
|
||||||
recording: ActiveRecording,
|
recording: ActiveRecording,
|
||||||
deps: RecordDeps,
|
deps: RecordDeps,
|
||||||
@@ -249,17 +319,67 @@ export interface RecordDeps {
|
|||||||
tabId: number,
|
tabId: number,
|
||||||
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
|
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
|
||||||
): Promise<unknown>;
|
): Promise<unknown>;
|
||||||
|
sendToDocument?(
|
||||||
|
tabId: number,
|
||||||
|
documentId: string,
|
||||||
|
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
|
||||||
|
): Promise<unknown>;
|
||||||
|
getAllFrames?(tabId: number): Promise<
|
||||||
|
Array<{
|
||||||
|
frameId: number;
|
||||||
|
documentId?: string;
|
||||||
|
url?: string;
|
||||||
|
parentFrameId: number;
|
||||||
|
}>
|
||||||
|
>;
|
||||||
|
broadcastCaptureStatus?(
|
||||||
|
tabId: number,
|
||||||
|
msg: RecordCaptureStatusMessage,
|
||||||
|
): Promise<void>;
|
||||||
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||||
cdp?: CdpRunner;
|
cdp?: CdpRunner;
|
||||||
signal?: AbortSignal;
|
signal?: AbortSignal;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function frameSendDeps(deps: RecordDeps): RecordFrameSendDeps {
|
||||||
|
return {
|
||||||
|
sendToDocument: (tabId, documentId, msg) => {
|
||||||
|
if (deps.sendToDocument) return deps.sendToDocument(tabId, documentId, msg);
|
||||||
|
return deps.sendToTab(tabId, msg);
|
||||||
|
},
|
||||||
|
getAllFrames: async (tabId) => {
|
||||||
|
if (deps.getAllFrames) return deps.getAllFrames(tabId);
|
||||||
|
if (!chrome.webNavigation?.getAllFrames) return [];
|
||||||
|
const frames = await chrome.webNavigation.getAllFrames({ tabId });
|
||||||
|
return (frames ?? []).map((frame) => ({
|
||||||
|
frameId: frame.frameId,
|
||||||
|
documentId: frame.documentId,
|
||||||
|
url: frame.url,
|
||||||
|
parentFrameId: frame.parentFrameId,
|
||||||
|
}));
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
let defaultDeps: RecordDeps | null = null;
|
let defaultDeps: RecordDeps | null = null;
|
||||||
function getDefaultDeps(): RecordDeps {
|
function getDefaultDeps(): RecordDeps {
|
||||||
if (!defaultDeps) {
|
if (!defaultDeps) {
|
||||||
defaultDeps = {
|
defaultDeps = {
|
||||||
tabsApi: chromeTabsApi,
|
tabsApi: chromeTabsApi,
|
||||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||||
|
sendToDocument: (tabId, documentId, msg) =>
|
||||||
|
chrome.tabs.sendMessage(tabId, msg, { documentId }),
|
||||||
|
getAllFrames: async (tabId) => {
|
||||||
|
if (!chrome.webNavigation?.getAllFrames) return [];
|
||||||
|
const frames = await chrome.webNavigation.getAllFrames({ tabId });
|
||||||
|
return (frames ?? []).map((frame) => ({
|
||||||
|
frameId: frame.frameId,
|
||||||
|
documentId: frame.documentId,
|
||||||
|
url: frame.url,
|
||||||
|
parentFrameId: frame.parentFrameId,
|
||||||
|
}));
|
||||||
|
},
|
||||||
|
broadcastCaptureStatus: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg, { frameId: 0 }),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
return defaultDeps;
|
return defaultDeps;
|
||||||
@@ -320,7 +440,7 @@ export function releaseBrowserObservationListenersIfIdle(): void {
|
|||||||
export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||||
const listener = (
|
const listener = (
|
||||||
message: unknown,
|
message: unknown,
|
||||||
_sender: chrome.runtime.MessageSender,
|
sender: chrome.runtime.MessageSender,
|
||||||
_sendResponse: () => void,
|
_sendResponse: () => void,
|
||||||
) => {
|
) => {
|
||||||
if (!isRecordStepMessage(message)) return false;
|
if (!isRecordStepMessage(message)) return false;
|
||||||
@@ -328,9 +448,6 @@ export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): (
|
|||||||
if (recording.requestId !== message.requestId) continue;
|
if (recording.requestId !== message.requestId) continue;
|
||||||
enqueueRecordingAction(recording, async () => {
|
enqueueRecordingAction(recording, async () => {
|
||||||
const cdp = recording.cdp ?? deps.cdp;
|
const cdp = recording.cdp ?? deps.cdp;
|
||||||
// Flush before appending so a coalesced OAuth navigate is ordered
|
|
||||||
// ahead of the click that landed on the destination page. The flush
|
|
||||||
// itself waits for that navigate's observation before returning.
|
|
||||||
await flushPendingRedirectLanding(
|
await flushPendingRedirectLanding(
|
||||||
recording.observation,
|
recording.observation,
|
||||||
recording.steps,
|
recording.steps,
|
||||||
@@ -339,7 +456,8 @@ export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): (
|
|||||||
deps.tabsApi,
|
deps.tabsApi,
|
||||||
{ cancelled: () => recording.settled },
|
{ cancelled: () => recording.settled },
|
||||||
);
|
);
|
||||||
const draftIndex = appendRecordedPayload(recording, message.step);
|
const enriched = enrichRecordedStep(recording, message.step);
|
||||||
|
const draftIndex = appendRecordedPayload(recording, enriched);
|
||||||
if (draftIndex !== null) await processRecordedStep(recording, deps, draftIndex);
|
if (draftIndex !== null) await processRecordedStep(recording, deps, draftIndex);
|
||||||
});
|
});
|
||||||
return false;
|
return false;
|
||||||
@@ -368,8 +486,10 @@ export function attachRecordFinishListener(deps: RecordDeps = getDefaultDeps()):
|
|||||||
|
|
||||||
function findRecordingByTabId(tabId: number): ActiveRecording | null {
|
function findRecordingByTabId(tabId: number): ActiveRecording | null {
|
||||||
for (const recording of recordings.values()) {
|
for (const recording of recordings.values()) {
|
||||||
if (!recording.settled && (recording.tabId === tabId || recording.armedTabIds.has(tabId))) {
|
if (recording.settled) continue;
|
||||||
return recording;
|
if (recording.tabId === tabId) return recording;
|
||||||
|
for (const armed of recording.armedDocuments.values()) {
|
||||||
|
if (armed.tabId === tabId) return recording;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
@@ -418,47 +538,64 @@ async function clearRearmTimersForRecording(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function stopRecordingOnAllAgentTabs(
|
async function stopRecordingOnAllDocuments(
|
||||||
recording: ActiveRecording,
|
recording: ActiveRecording,
|
||||||
deps: RecordDeps,
|
deps: RecordDeps,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
|
recording.acceptingNewFrames = false;
|
||||||
const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
|
const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
|
||||||
let failed = false;
|
const frameDeps = frameSendDeps(deps);
|
||||||
for (const tabId of [...recording.armedTabIds]) {
|
const armed = [...recording.armedDocuments.values()];
|
||||||
try {
|
for (const doc of armed) {
|
||||||
await deps.tabsApi.get(tabId);
|
const result = await stopDocumentCapture(doc, stopMsg, frameDeps);
|
||||||
} catch {
|
if ("ok" in result) {
|
||||||
// A closed tab no longer has content capture to flush.
|
recording.armedDocuments.delete(doc.documentId);
|
||||||
recording.armedTabIds.delete(tabId);
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
try {
|
recordFrameFailure(recording, result);
|
||||||
const response = await deps.sendToTab(tabId, stopMsg);
|
|
||||||
if (!isRecordStartAck(response)) {
|
|
||||||
throw new Error("content script did not confirm recorded steps");
|
|
||||||
}
|
|
||||||
recording.armedTabIds.delete(tabId);
|
|
||||||
} catch {
|
|
||||||
failed = true;
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if (deps.bypassOverlay) {
|
|
||||||
try {
|
|
||||||
await deps.bypassOverlay(tabId, false);
|
|
||||||
} catch {
|
|
||||||
// Best-effort cleanup.
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (failed || recording.armedTabIds.size > 0) {
|
if (recording.armedDocuments.size > 0) {
|
||||||
throw new Error("failed to flush recorded steps");
|
for (const doc of [...recording.armedDocuments.values()]) {
|
||||||
|
recordFrameFailure(
|
||||||
|
recording,
|
||||||
|
{
|
||||||
|
reason: "flush_failed",
|
||||||
|
frame_id: doc.frameId,
|
||||||
|
document_id: doc.documentId,
|
||||||
|
url: doc.url,
|
||||||
|
detail: "document still armed after STOP",
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
recording.armedDocuments.clear();
|
||||||
|
}
|
||||||
|
await publishCaptureStatus(recording, deps);
|
||||||
|
if (deps.bypassOverlay) {
|
||||||
|
try {
|
||||||
|
await deps.bypassOverlay(recording.tabId, false);
|
||||||
|
} catch {
|
||||||
|
// Best-effort cleanup.
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function rearmRecording(
|
async function cancelRecordingOnAllDocuments(
|
||||||
|
recording: ActiveRecording,
|
||||||
|
deps: RecordDeps,
|
||||||
|
): Promise<void> {
|
||||||
|
const cancelMsg: RecordCancelMessage = { type: RECORD_CANCEL, requestId: recording.requestId };
|
||||||
|
const frameDeps = frameSendDeps(deps);
|
||||||
|
for (const doc of [...recording.armedDocuments.values()]) {
|
||||||
|
await cancelDocumentCapture(doc, cancelMsg, frameDeps);
|
||||||
|
}
|
||||||
|
recording.armedDocuments.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function rearmRecordingDocuments(
|
||||||
recording: ActiveRecording,
|
recording: ActiveRecording,
|
||||||
targetTabId: number,
|
targetTabId: number,
|
||||||
deps: RecordDeps,
|
deps: RecordDeps,
|
||||||
|
targetDocumentId?: string,
|
||||||
): Promise<boolean> {
|
): Promise<boolean> {
|
||||||
let resolveTracked!: () => void;
|
let resolveTracked!: () => void;
|
||||||
const tracked = new Promise<void>((resolve) => {
|
const tracked = new Promise<void>((resolve) => {
|
||||||
@@ -466,29 +603,65 @@ async function rearmRecording(
|
|||||||
});
|
});
|
||||||
recording.rearmCallbacks.add(tracked);
|
recording.rearmCallbacks.add(tracked);
|
||||||
|
|
||||||
// Do NOT toggle automation-bypass here: each retry used to increment the
|
const frameDeps = frameSendDeps(deps);
|
||||||
// content-script counter, and a single stop decrement left the ControlOverlay
|
|
||||||
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
|
|
||||||
// already hides the control chrome while activeRecord is set.
|
|
||||||
try {
|
try {
|
||||||
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
|
const attemptLimit = targetDocumentId ? 1 : RECORD_REARM_MAX_ATTEMPTS;
|
||||||
if (recording.settled || recording.finishing) return false;
|
for (let attempt = 0; attempt < attemptLimit; attempt += 1) {
|
||||||
const startMsg: RecordStartMessage = {
|
if (recording.settled || recording.finishing || !recording.acceptingNewFrames) return false;
|
||||||
type: RECORD_START,
|
const { injectable, failures } = await listInjectableFrames(targetTabId, frameDeps);
|
||||||
requestId: recording.requestId,
|
for (const failure of failures) recordFrameFailure(recording, failure);
|
||||||
startedAtMs: recording.startedAtMs,
|
const targets = targetDocumentId
|
||||||
};
|
? injectable.filter((frame) => frame.documentId === targetDocumentId)
|
||||||
try {
|
: injectable.filter((frame) => !recording.armedDocuments.has(frame.documentId ?? ""));
|
||||||
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
|
if (targets.length === 0) {
|
||||||
recording.armedTabIds.add(targetTabId);
|
if (targetDocumentId) {
|
||||||
recording.tabId = targetTabId;
|
recordFrameFailure(recording, {
|
||||||
return true;
|
reason: "rearm_failed",
|
||||||
} catch {
|
frame_id: 0,
|
||||||
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
|
document_id: targetDocumentId,
|
||||||
await sleep(RECORD_REARM_RETRY_DELAY_MS);
|
detail: "document not injectable",
|
||||||
|
});
|
||||||
|
await publishCaptureStatus(recording, deps);
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
recording.tabId = targetTabId;
|
||||||
|
return injectable.some((frame) => recording.armedDocuments.has(frame.documentId ?? ""));
|
||||||
|
}
|
||||||
|
|
||||||
|
let armedAny = false;
|
||||||
|
for (const frame of targets) {
|
||||||
|
const startMsg: RecordStartMessage = {
|
||||||
|
type: RECORD_START,
|
||||||
|
requestId: recording.requestId,
|
||||||
|
startedAtMs: recording.startedAtMs,
|
||||||
|
showOverlay: frame.frameId === 0,
|
||||||
|
frameMode: frame.frameId === 0 ? "top" : "child",
|
||||||
|
};
|
||||||
|
const result = await armDocumentCapture(targetTabId, frame, startMsg, frameDeps);
|
||||||
|
if (result.armed) {
|
||||||
|
recording.armedDocuments.set(result.armed.documentId, result.armed);
|
||||||
|
armedAny = true;
|
||||||
|
}
|
||||||
|
if (result.failure) recordFrameFailure(recording, result.failure);
|
||||||
|
}
|
||||||
|
if (armedAny) {
|
||||||
|
recording.tabId = targetTabId;
|
||||||
|
await publishCaptureStatus(recording, deps);
|
||||||
|
if (deps.cdp && recording.acceptingNewFrames) {
|
||||||
|
void captureAndRegisterObservation(
|
||||||
|
recording.observation,
|
||||||
|
deps.cdp,
|
||||||
|
targetTabId,
|
||||||
|
deps.tabsApi,
|
||||||
|
).catch(() => {});
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
|
||||||
|
await sleep(RECORD_REARM_RETRY_DELAY_MS);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
await publishCaptureStatus(recording, deps);
|
||||||
return false;
|
return false;
|
||||||
} finally {
|
} finally {
|
||||||
recording.rearmCallbacks.delete(tracked);
|
recording.rearmCallbacks.delete(tracked);
|
||||||
@@ -505,7 +678,7 @@ function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
|
|||||||
rearmTimers.delete(tabId);
|
rearmTimers.delete(tabId);
|
||||||
void (async () => {
|
void (async () => {
|
||||||
const current = await findRecordingForTab(tabId, deps);
|
const current = await findRecordingForTab(tabId, deps);
|
||||||
if (current) await rearmRecording(current, tabId, deps);
|
if (current) await rearmRecordingDocuments(current, tabId, deps);
|
||||||
})();
|
})();
|
||||||
}, RECORD_REARM_DEBOUNCE_MS),
|
}, RECORD_REARM_DEBOUNCE_MS),
|
||||||
);
|
);
|
||||||
@@ -626,12 +799,24 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
|
|||||||
})();
|
})();
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const onSubFrameComplete = (tabId: number, documentId?: string) => {
|
||||||
|
if (!documentId) return;
|
||||||
|
void (async () => {
|
||||||
|
const recording = await findRecordingForTab(tabId, deps);
|
||||||
|
if (!recording || !recording.acceptingNewFrames) return;
|
||||||
|
await rearmRecordingDocuments(recording, tabId, deps, documentId);
|
||||||
|
})();
|
||||||
|
};
|
||||||
|
|
||||||
if (chrome.webNavigation?.onCompleted) {
|
if (chrome.webNavigation?.onCompleted) {
|
||||||
const completedListener = (
|
const completedListener = (
|
||||||
details: chrome.webNavigation.WebNavigationFramedCallbackDetails,
|
details: chrome.webNavigation.WebNavigationFramedCallbackDetails,
|
||||||
) => {
|
) => {
|
||||||
if (details.frameId !== 0) return;
|
if (details.frameId === 0) {
|
||||||
onMainFrameComplete(details.tabId, details.url);
|
onMainFrameComplete(details.tabId, details.url);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
onSubFrameComplete(details.tabId, details.documentId);
|
||||||
};
|
};
|
||||||
const committedListener = (
|
const committedListener = (
|
||||||
details: chrome.webNavigation.WebNavigationTransitionCallbackDetails,
|
details: chrome.webNavigation.WebNavigationTransitionCallbackDetails,
|
||||||
@@ -719,11 +904,15 @@ export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()):
|
|||||||
sendResponse({ active: false });
|
sendResponse({ active: false });
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
await rearmRecording(recording, tabId, deps);
|
await rearmRecordingDocuments(recording, tabId, deps, sender.documentId);
|
||||||
sendResponse({
|
sendResponse({
|
||||||
active: true,
|
active: true,
|
||||||
requestId: recording.requestId,
|
requestId: recording.requestId,
|
||||||
startedAtMs: recording.startedAtMs,
|
startedAtMs: recording.startedAtMs,
|
||||||
|
captureStatus: recording.frameCaptureStatus,
|
||||||
|
...(recording.frameCaptureFailures.length > 0
|
||||||
|
? { captureFailures: recording.frameCaptureFailures }
|
||||||
|
: {}),
|
||||||
});
|
});
|
||||||
})();
|
})();
|
||||||
return true;
|
return true;
|
||||||
@@ -760,10 +949,7 @@ async function finishRecording(
|
|||||||
await clearRearmTimersForRecording(recording, deps);
|
await clearRearmTimersForRecording(recording, deps);
|
||||||
await Promise.all([...recording.rearmCallbacks]);
|
await Promise.all([...recording.rearmCallbacks]);
|
||||||
try {
|
try {
|
||||||
// Disposing content capture may commit a final dirty fill. Flush content
|
await stopRecordingOnAllDocuments(recording, deps);
|
||||||
// first so all resulting RECORD_STEP messages enter actionQueue before it
|
|
||||||
// and the observation queues are drained.
|
|
||||||
await stopRecordingOnAllAgentTabs(recording, deps);
|
|
||||||
} catch {
|
} catch {
|
||||||
recording.finishing = false;
|
recording.finishing = false;
|
||||||
return null;
|
return null;
|
||||||
@@ -845,7 +1031,10 @@ export async function handleRecordStart(
|
|||||||
stoppedBy: "user_finish",
|
stoppedBy: "user_finish",
|
||||||
maxPageTokens: maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS,
|
maxPageTokens: maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS,
|
||||||
redactValues,
|
redactValues,
|
||||||
armedTabIds: new Set(),
|
armedDocuments: new Map(),
|
||||||
|
frameCaptureFailures: [],
|
||||||
|
frameCaptureStatus: "complete",
|
||||||
|
acceptingNewFrames: true,
|
||||||
rearmCallbacks: new Set(),
|
rearmCallbacks: new Set(),
|
||||||
navigationCallbacks: new Set(),
|
navigationCallbacks: new Set(),
|
||||||
acceptingNavigation: true,
|
acceptingNavigation: true,
|
||||||
@@ -857,17 +1046,11 @@ export async function handleRecordStart(
|
|||||||
ensureBrowserObservationListeners(deps);
|
ensureBrowserObservationListeners(deps);
|
||||||
|
|
||||||
const abortPending = async (notifyContent: boolean) => {
|
const abortPending = async (notifyContent: boolean) => {
|
||||||
|
const pending = recordings.get(params.session_id);
|
||||||
recordings.delete(params.session_id);
|
recordings.delete(params.session_id);
|
||||||
releaseBrowserObservationListenersIfIdle();
|
releaseBrowserObservationListenersIfIdle();
|
||||||
if (notifyContent) {
|
if (notifyContent && pending) {
|
||||||
try {
|
await cancelRecordingOnAllDocuments(pending, deps);
|
||||||
await deps.sendToTab(target.tabId, {
|
|
||||||
type: RECORD_CANCEL,
|
|
||||||
requestId,
|
|
||||||
});
|
|
||||||
} catch {
|
|
||||||
// Content may never have received RECORD_START.
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (deps.bypassOverlay) {
|
if (deps.bypassOverlay) {
|
||||||
try {
|
try {
|
||||||
@@ -970,11 +1153,28 @@ export async function handleRecordStart(
|
|||||||
if (cancelled) return cancelled;
|
if (cancelled) return cancelled;
|
||||||
}
|
}
|
||||||
|
|
||||||
const startMsg: RecordStartMessage = { type: RECORD_START, requestId, startedAtMs };
|
const startMsg: RecordStartMessage = {
|
||||||
|
type: RECORD_START,
|
||||||
|
requestId,
|
||||||
|
startedAtMs,
|
||||||
|
showOverlay: true,
|
||||||
|
frameMode: "top",
|
||||||
|
};
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
|
const frameDeps = frameSendDeps(deps);
|
||||||
active.armedTabIds.add(target.tabId);
|
const { armed, failures } = await armAllDocumentsForTab(target.tabId, startMsg, frameDeps);
|
||||||
|
for (const failure of failures) recordFrameFailure(active, failure);
|
||||||
|
for (const doc of armed) active.armedDocuments.set(doc.documentId, doc);
|
||||||
|
if (active.armedDocuments.size === 0) {
|
||||||
|
await abortPending(true);
|
||||||
|
return {
|
||||||
|
code: "protocol_error",
|
||||||
|
message:
|
||||||
|
"failed to start recording in any frame — reload the BrowserSkill extension, then retry",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
await publishCaptureStatus(active, deps);
|
||||||
} catch {
|
} catch {
|
||||||
await abortPending(true);
|
await abortPending(true);
|
||||||
return {
|
return {
|
||||||
|
|||||||
@@ -681,6 +681,28 @@ export interface RecorderInfo {
|
|||||||
vom: number;
|
vom: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type FrameCaptureStatus = "complete" | "partial";
|
||||||
|
|
||||||
|
export type FrameCaptureFailureReason =
|
||||||
|
| "not_injectable"
|
||||||
|
| "arm_failed"
|
||||||
|
| "rearm_failed"
|
||||||
|
| "drain_failed"
|
||||||
|
| "flush_failed";
|
||||||
|
|
||||||
|
export interface FrameCaptureFailure {
|
||||||
|
reason: FrameCaptureFailureReason;
|
||||||
|
frame_id: number;
|
||||||
|
document_id?: string;
|
||||||
|
url?: string;
|
||||||
|
detail?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FrameCaptureInfo {
|
||||||
|
status: FrameCaptureStatus;
|
||||||
|
failures?: FrameCaptureFailure[];
|
||||||
|
}
|
||||||
|
|
||||||
export type StopReason = "user_finish" | "cli_stop";
|
export type StopReason = "user_finish" | "cli_stop";
|
||||||
|
|
||||||
export interface TraceState {
|
export interface TraceState {
|
||||||
@@ -864,6 +886,8 @@ export interface Trace {
|
|||||||
stopped_by: StopReason;
|
stopped_by: StopReason;
|
||||||
entry: TraceEntry;
|
entry: TraceEntry;
|
||||||
recorder: RecorderInfo;
|
recorder: RecorderInfo;
|
||||||
|
/** Present when one or more frames could not be armed, rearmed, or flushed. */
|
||||||
|
frame_capture?: FrameCaptureInfo;
|
||||||
states: TraceState[];
|
states: TraceState[];
|
||||||
steps: Step[];
|
steps: Step[];
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -285,6 +285,7 @@ bsk record stop [--output trace] # terminal fallback if the browser panel is u
|
|||||||
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
|
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
|
||||||
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
||||||
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
|
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
|
||||||
|
- Recording captures interactions in the **top frame and injectable child frames** (same-origin and cross-origin where the extension can attach). Embedded frames that cannot be injected are omitted; the trace may include `frame_capture.status: "partial"` with failure details.
|
||||||
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
||||||
|
|
||||||
## Error handling
|
## Error handling
|
||||||
|
|||||||
@@ -8,8 +8,8 @@ use std::time::Duration;
|
|||||||
|
|
||||||
use anyhow::Context;
|
use anyhow::Context;
|
||||||
use bsk_protocol::tools::{
|
use bsk_protocol::tools::{
|
||||||
RecordAwaitParams, RecordAwaitResult, RecordStartParams, RecordStartResult, RecordStopParams,
|
FrameCaptureStatus, RecordAwaitParams, RecordAwaitResult, RecordStartParams, RecordStartResult,
|
||||||
RecordStopResult, TRACE_VERSION, Trace, TraceState,
|
RecordStopParams, RecordStopResult, TRACE_VERSION, Trace, TraceState,
|
||||||
};
|
};
|
||||||
use bsk_protocol::{ErrorCode, Method};
|
use bsk_protocol::{ErrorCode, Method};
|
||||||
use clap::{Args, Subcommand};
|
use clap::{Args, Subcommand};
|
||||||
@@ -550,6 +550,13 @@ fn render_finish(
|
|||||||
trace_path.display(),
|
trace_path.display(),
|
||||||
pages_dir.display()
|
pages_dir.display()
|
||||||
);
|
);
|
||||||
|
if let Some(frame_capture) = &trace.frame_capture {
|
||||||
|
if matches!(frame_capture.status, FrameCaptureStatus::Partial) {
|
||||||
|
eprintln!(
|
||||||
|
"warning: trace is partial — some embedded frames were not recorded (see frame_capture.failures in trace.json)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -584,6 +591,7 @@ mod tests {
|
|||||||
bsk: "0.1.10".into(),
|
bsk: "0.1.10".into(),
|
||||||
vom: VOM_FORMAT_VERSION,
|
vom: VOM_FORMAT_VERSION,
|
||||||
},
|
},
|
||||||
|
frame_capture: None,
|
||||||
states: vec![TraceState {
|
states: vec![TraceState {
|
||||||
id: state_id.into(),
|
id: state_id.into(),
|
||||||
url: "https://example.com/".into(),
|
url: "https://example.com/".into(),
|
||||||
|
|||||||
@@ -96,6 +96,7 @@ fn sample_trace() -> Trace {
|
|||||||
bsk: "0.1.10".into(),
|
bsk: "0.1.10".into(),
|
||||||
vom: VOM_FORMAT_VERSION,
|
vom: VOM_FORMAT_VERSION,
|
||||||
},
|
},
|
||||||
|
frame_capture: None,
|
||||||
states: vec![TraceState {
|
states: vec![TraceState {
|
||||||
id: "s1".into(),
|
id: "s1".into(),
|
||||||
url: "https://example.com/".into(),
|
url: "https://example.com/".into(),
|
||||||
|
|||||||
@@ -16,6 +16,16 @@
|
|||||||
"entry": {
|
"entry": {
|
||||||
"$ref": "#/definitions/TraceEntry"
|
"$ref": "#/definitions/TraceEntry"
|
||||||
},
|
},
|
||||||
|
"frame_capture": {
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"$ref": "#/definitions/FrameCaptureInfo"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
"purpose": {
|
"purpose": {
|
||||||
"type": [
|
"type": [
|
||||||
"string",
|
"string",
|
||||||
@@ -64,6 +74,77 @@
|
|||||||
"blur"
|
"blur"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
"FrameCaptureFailure": {
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"frame_id",
|
||||||
|
"reason"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"detail": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"document_id": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"frame_id": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "int64"
|
||||||
|
},
|
||||||
|
"reason": {
|
||||||
|
"$ref": "#/definitions/FrameCaptureFailureReason"
|
||||||
|
},
|
||||||
|
"url": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"FrameCaptureFailureReason": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"not_injectable",
|
||||||
|
"arm_failed",
|
||||||
|
"rearm_failed",
|
||||||
|
"drain_failed",
|
||||||
|
"flush_failed"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"FrameCaptureInfo": {
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"status"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"failures": {
|
||||||
|
"type": [
|
||||||
|
"array",
|
||||||
|
"null"
|
||||||
|
],
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/FrameCaptureFailure"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"status": {
|
||||||
|
"$ref": "#/definitions/FrameCaptureStatus"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"FrameCaptureStatus": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"complete",
|
||||||
|
"partial"
|
||||||
|
]
|
||||||
|
},
|
||||||
"KeyModifier": {
|
"KeyModifier": {
|
||||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||||
"type": "string",
|
"type": "string",
|
||||||
|
|||||||
@@ -167,6 +167,46 @@ pub enum Step {
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Frame capture metadata
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum FrameCaptureStatus {
|
||||||
|
Complete,
|
||||||
|
Partial,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum FrameCaptureFailureReason {
|
||||||
|
NotInjectable,
|
||||||
|
ArmFailed,
|
||||||
|
RearmFailed,
|
||||||
|
DrainFailed,
|
||||||
|
FlushFailed,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct FrameCaptureFailure {
|
||||||
|
pub reason: FrameCaptureFailureReason,
|
||||||
|
pub frame_id: i64,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub document_id: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub url: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub detail: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct FrameCaptureInfo {
|
||||||
|
pub status: FrameCaptureStatus,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub failures: Option<Vec<FrameCaptureFailure>>,
|
||||||
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Trace root
|
// Trace root
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -183,6 +223,8 @@ pub struct Trace {
|
|||||||
pub stopped_by: StopReason,
|
pub stopped_by: StopReason,
|
||||||
pub entry: TraceEntry,
|
pub entry: TraceEntry,
|
||||||
pub recorder: RecorderInfo,
|
pub recorder: RecorderInfo,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub frame_capture: Option<FrameCaptureInfo>,
|
||||||
pub states: Vec<TraceState>,
|
pub states: Vec<TraceState>,
|
||||||
pub steps: Vec<Step>,
|
pub steps: Vec<Step>,
|
||||||
}
|
}
|
||||||
@@ -277,6 +319,7 @@ mod tests {
|
|||||||
bsk: "0.1.10".into(),
|
bsk: "0.1.10".into(),
|
||||||
vom: VOM_FORMAT_VERSION,
|
vom: VOM_FORMAT_VERSION,
|
||||||
},
|
},
|
||||||
|
frame_capture: None,
|
||||||
states: vec![
|
states: vec![
|
||||||
TraceState {
|
TraceState {
|
||||||
id: "s1".into(),
|
id: "s1".into(),
|
||||||
@@ -479,4 +522,39 @@ mod tests {
|
|||||||
assert_eq!(trace.states.len(), 2);
|
assert_eq!(trace.states.len(), 2);
|
||||||
assert_eq!(trace.steps.len(), 2);
|
assert_eq!(trace.steps.len(), 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn trace_deserializes_optional_frame_capture_metadata() {
|
||||||
|
let v = json!({
|
||||||
|
"version": 3,
|
||||||
|
"recorded_at": "2026-07-21T08:00:00Z",
|
||||||
|
"stopped_by": "cli_stop",
|
||||||
|
"entry": { "start_url": "https://example.com/editor" },
|
||||||
|
"recorder": { "bsk": "0.1.10", "vom": 1 },
|
||||||
|
"frame_capture": {
|
||||||
|
"status": "partial",
|
||||||
|
"failures": [
|
||||||
|
{
|
||||||
|
"reason": "not_injectable",
|
||||||
|
"frame_id": 2,
|
||||||
|
"url": "chrome://settings",
|
||||||
|
"detail": "restricted frame url"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"states": [],
|
||||||
|
"steps": []
|
||||||
|
});
|
||||||
|
let trace: Trace = serde_json::from_value(v).unwrap();
|
||||||
|
let frame_capture = trace.frame_capture.expect("frame_capture");
|
||||||
|
assert_eq!(frame_capture.status, FrameCaptureStatus::Partial);
|
||||||
|
let failures = frame_capture.failures.expect("failures");
|
||||||
|
assert_eq!(failures.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
failures[0].reason,
|
||||||
|
FrameCaptureFailureReason::NotInjectable
|
||||||
|
);
|
||||||
|
assert_eq!(failures[0].frame_id, 2);
|
||||||
|
assert_eq!(failures[0].url.as_deref(), Some("chrome://settings"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -90,6 +90,7 @@
|
|||||||
},
|
},
|
||||||
"recordOverlay": {
|
"recordOverlay": {
|
||||||
"recording": "Recording your actions",
|
"recording": "Recording your actions",
|
||||||
"finish": "Finish"
|
"finish": "Finish",
|
||||||
|
"partialCapture": "Some embedded frames may not be recorded."
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -90,6 +90,7 @@
|
|||||||
},
|
},
|
||||||
"recordOverlay": {
|
"recordOverlay": {
|
||||||
"recording": "正在录制你的操作",
|
"recording": "正在录制你的操作",
|
||||||
"finish": "结束"
|
"finish": "结束",
|
||||||
|
"partialCapture": "部分嵌入页面可能未被录制。"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -285,6 +285,7 @@ bsk record stop [--output trace] # terminal fallback if the browser panel is u
|
|||||||
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
|
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
|
||||||
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
||||||
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
|
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
|
||||||
|
- Recording captures interactions in the **top frame and injectable child frames** (same-origin and cross-origin where the extension can attach). Embedded frames that cannot be injected are omitted; the trace may include `frame_capture.status: "partial"` with failure details.
|
||||||
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
||||||
|
|
||||||
## Error handling
|
## Error handling
|
||||||
|
|||||||
Reference in New Issue
Block a user