mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f7e08f461b | ||
|
|
61a0e0d218 | ||
|
|
67b7227abf | ||
|
|
02d1d69379 | ||
|
|
025751ef03 | ||
|
|
13a61b2688 | ||
|
|
1508839bf9 |
@@ -17,3 +17,7 @@ dist/
|
|||||||
# Environment
|
# Environment
|
||||||
.env.local
|
.env.local
|
||||||
.env.*.local
|
.env.*.local
|
||||||
|
.pnpm-store/
|
||||||
|
|
||||||
|
# Local recording output
|
||||||
|
/trace/
|
||||||
@@ -1,4 +1,5 @@
|
|||||||
import { useTranslation } from "@browser-skill/i18n/react";
|
import { useTranslation } from "@browser-skill/i18n/react";
|
||||||
|
import type { FrameCaptureFailure, FrameCaptureStatus } from "@/transport/types";
|
||||||
import {
|
import {
|
||||||
type PointerEvent as ReactPointerEvent,
|
type PointerEvent as ReactPointerEvent,
|
||||||
useCallback,
|
useCallback,
|
||||||
@@ -21,6 +22,8 @@ export interface RecordRequestData {
|
|||||||
|
|
||||||
type Props = {
|
type Props = {
|
||||||
request: RecordRequestData | null;
|
request: RecordRequestData | null;
|
||||||
|
captureStatus?: FrameCaptureStatus;
|
||||||
|
captureFailures?: FrameCaptureFailure[];
|
||||||
};
|
};
|
||||||
|
|
||||||
const VIEWPORT_MARGIN = 16;
|
const VIEWPORT_MARGIN = 16;
|
||||||
@@ -52,7 +55,11 @@ function clampDragPos(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export function RecordOverlay({ request }: Props) {
|
export function RecordOverlay({
|
||||||
|
request,
|
||||||
|
captureStatus = "complete",
|
||||||
|
captureFailures = [],
|
||||||
|
}: Props) {
|
||||||
const { t } = useTranslation("extension");
|
const { t } = useTranslation("extension");
|
||||||
const [show, setShow] = useState(false);
|
const [show, setShow] = useState(false);
|
||||||
const [elapsedSeconds, setElapsedSeconds] = useState(0);
|
const [elapsedSeconds, setElapsedSeconds] = useState(0);
|
||||||
@@ -154,6 +161,7 @@ export function RecordOverlay({ request }: Props) {
|
|||||||
|
|
||||||
const introVisible = elapsedSeconds * 1_000 < INTRO_DURATION_MS;
|
const introVisible = elapsedSeconds * 1_000 < INTRO_DURATION_MS;
|
||||||
const positioned = dragPos !== null;
|
const positioned = dragPos !== null;
|
||||||
|
const partialCapture = captureStatus === "partial" || captureFailures.length > 0;
|
||||||
const defaultTransform = show
|
const defaultTransform = show
|
||||||
? "translateX(-50%) translateY(0)"
|
? "translateX(-50%) translateY(0)"
|
||||||
: "translateX(-50%) translateY(8px)";
|
: "translateX(-50%) translateY(8px)";
|
||||||
@@ -208,8 +216,9 @@ export function RecordOverlay({ request }: Props) {
|
|||||||
data-slot="record-overlay-status"
|
data-slot="record-overlay-status"
|
||||||
style={{
|
style={{
|
||||||
display: "flex",
|
display: "flex",
|
||||||
alignItems: "center",
|
flexDirection: "column",
|
||||||
gap: 8,
|
alignItems: "flex-start",
|
||||||
|
gap: 2,
|
||||||
minWidth: 0,
|
minWidth: 0,
|
||||||
flex: 1,
|
flex: 1,
|
||||||
borderRadius: 9999,
|
borderRadius: 9999,
|
||||||
@@ -218,41 +227,57 @@ export function RecordOverlay({ request }: Props) {
|
|||||||
paddingRight: 4,
|
paddingRight: 4,
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
<span
|
<div style={{ display: "flex", alignItems: "center", gap: 8, minWidth: 0, width: "100%" }}>
|
||||||
data-slot="record-overlay-indicator"
|
<span
|
||||||
style={{
|
data-slot="record-overlay-indicator"
|
||||||
width: 9,
|
style={{
|
||||||
height: 9,
|
width: 9,
|
||||||
borderRadius: "50%",
|
height: 9,
|
||||||
backgroundColor: "#ef4444",
|
borderRadius: "50%",
|
||||||
flexShrink: 0,
|
backgroundColor: "#ef4444",
|
||||||
animation: "bsk-rec-pulse 1.4s ease-in-out infinite",
|
flexShrink: 0,
|
||||||
}}
|
animation: "bsk-rec-pulse 1.4s ease-in-out infinite",
|
||||||
/>
|
}}
|
||||||
<span
|
/>
|
||||||
data-slot="record-overlay-label"
|
<span
|
||||||
style={{
|
data-slot="record-overlay-label"
|
||||||
display: "block",
|
style={{
|
||||||
flex: "0 1 auto",
|
display: "block",
|
||||||
fontSize: 16,
|
flex: "0 1 auto",
|
||||||
fontWeight: 500,
|
fontSize: 16,
|
||||||
color: "#333",
|
fontWeight: 500,
|
||||||
whiteSpace: "nowrap",
|
color: "#333",
|
||||||
userSelect: "none",
|
whiteSpace: "nowrap",
|
||||||
overflow: "hidden",
|
userSelect: "none",
|
||||||
textOverflow: "ellipsis",
|
overflow: "hidden",
|
||||||
width: introVisible ? "auto" : 48,
|
textOverflow: "ellipsis",
|
||||||
maxWidth: introVisible ? 280 : 48,
|
width: introVisible ? "auto" : 48,
|
||||||
fontVariantNumeric: introVisible ? undefined : "tabular-nums",
|
maxWidth: introVisible ? 280 : 48,
|
||||||
textAlign: introVisible ? "left" : "center",
|
fontVariantNumeric: introVisible ? undefined : "tabular-nums",
|
||||||
margin: 0,
|
textAlign: introVisible ? "left" : "center",
|
||||||
padding: 0,
|
margin: 0,
|
||||||
minWidth: 0,
|
padding: 0,
|
||||||
transition: "max-width 180ms ease-out",
|
minWidth: 0,
|
||||||
}}
|
transition: "max-width 180ms ease-out",
|
||||||
>
|
}}
|
||||||
{introVisible ? t("recordOverlay.recording") : formatElapsed(elapsedSeconds)}
|
>
|
||||||
</span>
|
{introVisible ? t("recordOverlay.recording") : formatElapsed(elapsedSeconds)}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
{partialCapture ? (
|
||||||
|
<span
|
||||||
|
data-slot="record-overlay-partial-warning"
|
||||||
|
style={{
|
||||||
|
fontSize: 12,
|
||||||
|
lineHeight: "16px",
|
||||||
|
color: "#b45309",
|
||||||
|
whiteSpace: "normal",
|
||||||
|
maxWidth: 280,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{t("recordOverlay.partialCapture")}
|
||||||
|
</span>
|
||||||
|
) : null}
|
||||||
</div>
|
</div>
|
||||||
<button
|
<button
|
||||||
type="button"
|
type="button"
|
||||||
|
|||||||
@@ -178,4 +178,30 @@ describe("RecordOverlay", () => {
|
|||||||
expect(pill.style.bottom).toBe("auto");
|
expect(pill.style.bottom).toBe("auto");
|
||||||
expect(pill.style.transform).toBe("none");
|
expect(pill.style.transform).toBe("none");
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("shows a persistent partial-capture warning when frame capture is incomplete", () => {
|
||||||
|
const { container } = render(
|
||||||
|
createElement(
|
||||||
|
I18nextProvider,
|
||||||
|
{ i18n },
|
||||||
|
createElement(RecordOverlay, {
|
||||||
|
request: { id: "rec-1", onFinish: vi.fn() },
|
||||||
|
captureStatus: "partial",
|
||||||
|
captureFailures: [
|
||||||
|
{
|
||||||
|
reason: "not_injectable",
|
||||||
|
frame_id: 2,
|
||||||
|
url: "chrome://settings",
|
||||||
|
},
|
||||||
|
],
|
||||||
|
}),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
const warning = container.querySelector("[data-slot='record-overlay-partial-warning']");
|
||||||
|
expect(warning).toBeTruthy();
|
||||||
|
expect(warning?.textContent).toBe(
|
||||||
|
i18n.t("recordOverlay.partialCapture", { ns: "extension" }),
|
||||||
|
);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||||
import { RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
|
import { RECORD_START, RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
|
||||||
import { handleRecordContentMessage, startRecordCapture } from "../record-capture";
|
import { handleRecordContentMessage, startRecordCapture } from "../record-capture";
|
||||||
|
|
||||||
vi.stubGlobal("chrome", {
|
vi.stubGlobal("chrome", {
|
||||||
@@ -96,6 +96,65 @@ describe("handleRecordContentMessage stop/cancel", () => {
|
|||||||
expect(dispose).not.toHaveBeenCalled();
|
expect(dispose).not.toHaveBeenCalled();
|
||||||
expect(onStop).not.toHaveBeenCalled();
|
expect(onStop).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("keeps a failed STOP retryable and redelivers its recorded step", async () => {
|
||||||
|
document.body.innerHTML = `<label for="retry">Draft</label><input id="retry" />`;
|
||||||
|
const sendMessage = vi.mocked(chrome.runtime.sendMessage);
|
||||||
|
sendMessage.mockReset();
|
||||||
|
sendMessage.mockRejectedValueOnce(new Error("service worker unavailable"));
|
||||||
|
sendMessage.mockResolvedValueOnce(undefined);
|
||||||
|
|
||||||
|
let activeRequestId: string | null = null;
|
||||||
|
let capture: ReturnType<typeof startRecordCapture> | null = null;
|
||||||
|
const onStop = vi.fn();
|
||||||
|
const dispatch = (
|
||||||
|
message:
|
||||||
|
| { type: typeof RECORD_START; requestId: string; startedAtMs?: number }
|
||||||
|
| { type: typeof RECORD_STOP; requestId: string },
|
||||||
|
sendResponse?: (response: unknown) => void,
|
||||||
|
) =>
|
||||||
|
handleRecordContentMessage(
|
||||||
|
message,
|
||||||
|
{
|
||||||
|
activeRequestId,
|
||||||
|
capture,
|
||||||
|
setActiveRequestId: (id) => {
|
||||||
|
activeRequestId = id;
|
||||||
|
},
|
||||||
|
setCapture: (next) => {
|
||||||
|
capture = next;
|
||||||
|
},
|
||||||
|
onStart: vi.fn(),
|
||||||
|
onStop,
|
||||||
|
},
|
||||||
|
sendResponse,
|
||||||
|
);
|
||||||
|
|
||||||
|
dispatch({ type: RECORD_START, requestId: "rec-retry" });
|
||||||
|
const input = document.querySelector("input")!;
|
||||||
|
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||||
|
input.value = "dirty final value";
|
||||||
|
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||||
|
|
||||||
|
const firstResponse = vi.fn();
|
||||||
|
expect(dispatch({ type: RECORD_STOP, requestId: "rec-retry" }, firstResponse)).toBe(true);
|
||||||
|
await vi.waitFor(() =>
|
||||||
|
expect(firstResponse).toHaveBeenCalledWith({
|
||||||
|
ok: false,
|
||||||
|
error: "failed to deliver one or more recorded steps",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
expect(activeRequestId).toBe("rec-retry");
|
||||||
|
expect(onStop).not.toHaveBeenCalled();
|
||||||
|
|
||||||
|
const retryResponse = vi.fn();
|
||||||
|
expect(dispatch({ type: RECORD_STOP, requestId: "rec-retry" }, retryResponse)).toBe(true);
|
||||||
|
await vi.waitFor(() => expect(retryResponse).toHaveBeenCalledWith({ ok: true }));
|
||||||
|
|
||||||
|
expect(sendMessage).toHaveBeenCalledTimes(2);
|
||||||
|
expect(activeRequestId).toBeNull();
|
||||||
|
expect(onStop).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("record-capture semantic", () => {
|
describe("record-capture semantic", () => {
|
||||||
@@ -228,6 +287,7 @@ describe("record-capture semantic", () => {
|
|||||||
expect(steps[0]).toMatchObject({
|
expect(steps[0]).toMatchObject({
|
||||||
op: "hover",
|
op: "hover",
|
||||||
target: { role: "button", name: "Open user navigation menu" },
|
target: { role: "button", name: "Open user navigation menu" },
|
||||||
|
geometry: { tag: "button", rect: { x: 900, y: 8, w: 32, h: 32 } },
|
||||||
});
|
});
|
||||||
expect(steps[1]).toMatchObject({
|
expect(steps[1]).toMatchObject({
|
||||||
op: "click",
|
op: "click",
|
||||||
@@ -1554,3 +1614,79 @@ describe("record-capture semantic", () => {
|
|||||||
expect(steps.map((s) => s.op)).toEqual(["click"]);
|
expect(steps.map((s) => s.op)).toEqual(["click"]);
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("record-capture child frame", () => {
|
||||||
|
let steps: RecordStepPayload[];
|
||||||
|
|
||||||
|
beforeEach(() => {
|
||||||
|
steps = [];
|
||||||
|
document.body.innerHTML = `
|
||||||
|
<label for="child-q">Child query</label>
|
||||||
|
<input id="child-q" name="child-q" />
|
||||||
|
<button type="button" aria-label="Child search">Search</button>
|
||||||
|
<select aria-label="Child status">
|
||||||
|
<option value="on">On</option>
|
||||||
|
<option value="off">Off</option>
|
||||||
|
</select>
|
||||||
|
`;
|
||||||
|
});
|
||||||
|
|
||||||
|
it("stamps a document token on start and clears it on dispose", () => {
|
||||||
|
const capture = startRecordCapture("rec-child-token", (step) => steps.push(step), {
|
||||||
|
frameMode: "child",
|
||||||
|
});
|
||||||
|
const token = document.documentElement.getAttribute("data-bsk-record-document");
|
||||||
|
expect(token).toBeTruthy();
|
||||||
|
click(document.querySelector("button")!);
|
||||||
|
expect(steps[0]?.documentToken).toBe(token);
|
||||||
|
expect(steps[0]?.frame_url).toBeTruthy();
|
||||||
|
capture.dispose();
|
||||||
|
expect(document.documentElement.hasAttribute("data-bsk-record-document")).toBe(false);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("records click, fill, select, and press in child frame mode", () => {
|
||||||
|
const capture = startRecordCapture("rec-child-ops", (step) => steps.push(step), {
|
||||||
|
frameMode: "child",
|
||||||
|
});
|
||||||
|
const input = document.querySelector("input")!;
|
||||||
|
const select = document.querySelector("select")!;
|
||||||
|
|
||||||
|
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||||
|
input.value = "child value";
|
||||||
|
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||||
|
input.dispatchEvent(new FocusEvent("focusout", { bubbles: true }));
|
||||||
|
select.value = "off";
|
||||||
|
select.dispatchEvent(new Event("change", { bubbles: true }));
|
||||||
|
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||||
|
input.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", bubbles: true }));
|
||||||
|
click(document.querySelector("button")!);
|
||||||
|
capture.dispose();
|
||||||
|
|
||||||
|
expect(steps.map((s) => s.op)).toEqual(
|
||||||
|
expect.arrayContaining(["fill", "select", "press", "click"]),
|
||||||
|
);
|
||||||
|
expect(steps.every((s) => s.documentToken)).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("commits the last fill when dispose runs in child frame mode", () => {
|
||||||
|
const capture = startRecordCapture("rec-child-fill-stop", (step) => steps.push(step), {
|
||||||
|
frameMode: "child",
|
||||||
|
});
|
||||||
|
const input = document.querySelector("input")!;
|
||||||
|
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||||
|
input.value = "pending child fill";
|
||||||
|
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||||
|
capture.dispose();
|
||||||
|
|
||||||
|
expect(steps.some((s) => s.op === "fill" && s.value === "pending child fill")).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("does not emit navigate steps for child frame navigations", () => {
|
||||||
|
const capture = startRecordCapture("rec-child-nav", (step) => steps.push(step), {
|
||||||
|
frameMode: "child",
|
||||||
|
});
|
||||||
|
history.pushState({}, "", "/child-frame-path");
|
||||||
|
capture.dispose();
|
||||||
|
expect(steps.filter((s) => s.op === "navigate")).toHaveLength(0);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
import {
|
import {
|
||||||
|
type CaptureTargetDescriptor,
|
||||||
describeEventTarget,
|
describeEventTarget,
|
||||||
describeTarget,
|
describeTarget,
|
||||||
resolveClickableElement,
|
resolveClickableElement,
|
||||||
resolveHoverElement,
|
resolveHoverElement,
|
||||||
type TargetDescriptor,
|
|
||||||
} from "@/lib/describe-target";
|
} from "@/lib/describe-target";
|
||||||
import {
|
import {
|
||||||
evaluateHoverTrigger,
|
evaluateHoverTrigger,
|
||||||
@@ -21,12 +21,18 @@ import {
|
|||||||
RECORD_STEP,
|
RECORD_STEP,
|
||||||
RECORD_STOP,
|
RECORD_STOP,
|
||||||
type RecordCancelMessage,
|
type RecordCancelMessage,
|
||||||
|
type RecordFrameMode,
|
||||||
type RecordStartAck,
|
type RecordStartAck,
|
||||||
type RecordStartMessage,
|
type RecordStartMessage,
|
||||||
type RecordStepPayload,
|
type RecordStepPayload,
|
||||||
type RecordStopAck,
|
type RecordStopAck,
|
||||||
type RecordStopMessage,
|
type RecordStopMessage,
|
||||||
} from "@/lib/record-bridge";
|
} from "@/lib/record-bridge";
|
||||||
|
import {
|
||||||
|
clearRecordDocumentToken,
|
||||||
|
createRecordDocumentToken,
|
||||||
|
stampRecordDocumentToken,
|
||||||
|
} from "@/lib/record-document-token";
|
||||||
import { shouldRecordPress } from "@/lib/trace-reducer";
|
import { shouldRecordPress } from "@/lib/trace-reducer";
|
||||||
import {
|
import {
|
||||||
closestHoverSurfaceCandidate,
|
closestHoverSurfaceCandidate,
|
||||||
@@ -39,17 +45,26 @@ import {
|
|||||||
|
|
||||||
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
|
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
|
||||||
const failedStepDeliveries = new Set<string>();
|
const failedStepDeliveries = new Set<string>();
|
||||||
|
const failedStepPayloads = new Map<string, RecordStepPayload[]>();
|
||||||
|
const pendingStopFlushes = new Map<string, Promise<RecordStopAck>>();
|
||||||
const knownRecordRequests = new Set<string>();
|
const knownRecordRequests = new Set<string>();
|
||||||
|
|
||||||
function sendRecordStep(requestId: string, step: RecordStepPayload): void {
|
function deliverRecordStep(requestId: string, step: RecordStepPayload): Promise<boolean> {
|
||||||
const payload = { type: RECORD_STEP, requestId, step };
|
const payload = { type: RECORD_STEP, requestId, step };
|
||||||
const pending = Promise.resolve(chrome.runtime.sendMessage(payload)).then(
|
return Promise.resolve(chrome.runtime.sendMessage(payload)).then(
|
||||||
() => true,
|
() => true,
|
||||||
() => {
|
() => {
|
||||||
failedStepDeliveries.add(requestId);
|
failedStepDeliveries.add(requestId);
|
||||||
|
const failed = failedStepPayloads.get(requestId) ?? [];
|
||||||
|
failed.push(step);
|
||||||
|
failedStepPayloads.set(requestId, failed);
|
||||||
return false;
|
return false;
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function sendRecordStep(requestId: string, step: RecordStepPayload): void {
|
||||||
|
const pending = deliverRecordStep(requestId, step);
|
||||||
const sends = pendingStepSends.get(requestId) ?? new Set<Promise<boolean>>();
|
const sends = pendingStepSends.get(requestId) ?? new Set<Promise<boolean>>();
|
||||||
sends.add(pending);
|
sends.add(pending);
|
||||||
pendingStepSends.set(requestId, sends);
|
pendingStepSends.set(requestId, sends);
|
||||||
@@ -65,14 +80,15 @@ export interface RecordCaptureController {
|
|||||||
|
|
||||||
interface FillSession {
|
interface FillSession {
|
||||||
element: FillableElement;
|
element: FillableElement;
|
||||||
target: TargetDescriptor;
|
target: CaptureTargetDescriptor;
|
||||||
baselineValue: string;
|
baselineValue: string;
|
||||||
lastValue: string;
|
lastValue: string;
|
||||||
|
pendingCommit?: "enter" | "suggestion" | "blur";
|
||||||
}
|
}
|
||||||
|
|
||||||
interface HoverCandidate {
|
interface HoverCandidate {
|
||||||
element: Element;
|
element: Element;
|
||||||
target: TargetDescriptor;
|
target: CaptureTargetDescriptor;
|
||||||
recordedAt: number;
|
recordedAt: number;
|
||||||
score: number;
|
score: number;
|
||||||
eligible: boolean;
|
eligible: boolean;
|
||||||
@@ -157,6 +173,28 @@ function nearbyFillableFromSearchChrome(target: Element): FillableElement | null
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function captureGeometry(el: Element): RecordStepPayload["geometry"] {
|
||||||
|
const rect = el.getBoundingClientRect();
|
||||||
|
const style = window.getComputedStyle(el);
|
||||||
|
return {
|
||||||
|
rect: { x: rect.x, y: rect.y, w: rect.width, h: rect.height },
|
||||||
|
scrollX: window.scrollX,
|
||||||
|
scrollY: window.scrollY,
|
||||||
|
position: style.position,
|
||||||
|
tag: el.tagName.toLowerCase(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function geometryForEventTarget(
|
||||||
|
target: EventTarget | null,
|
||||||
|
): RecordStepPayload["geometry"] | undefined {
|
||||||
|
if (!(target instanceof Element)) return undefined;
|
||||||
|
const clickable = target.closest(
|
||||||
|
'a,button,input,select,textarea,[role="button"],[role="link"],[role="menuitem"],[contenteditable="true"]',
|
||||||
|
);
|
||||||
|
return captureGeometry(clickable ?? target);
|
||||||
|
}
|
||||||
|
|
||||||
function fillableValue(el: FillableElement): string {
|
function fillableValue(el: FillableElement): string {
|
||||||
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
|
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
|
||||||
return el.value;
|
return el.value;
|
||||||
@@ -232,7 +270,7 @@ function collectHoverTriggerLabelText(root: Element): string {
|
|||||||
return normalizeLabelText(text);
|
return normalizeLabelText(text);
|
||||||
}
|
}
|
||||||
|
|
||||||
function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDescriptor {
|
function compactHoverTargetName(el: Element, desc: CaptureTargetDescriptor): CaptureTargetDescriptor {
|
||||||
if (!desc.name) return desc;
|
if (!desc.name) return desc;
|
||||||
const fullText = normalizeLabelText(el.textContent ?? "");
|
const fullText = normalizeLabelText(el.textContent ?? "");
|
||||||
const compactName = collectHoverTriggerLabelText(el);
|
const compactName = collectHoverTriggerLabelText(el);
|
||||||
@@ -250,7 +288,7 @@ function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDesc
|
|||||||
return desc;
|
return desc;
|
||||||
}
|
}
|
||||||
|
|
||||||
function isWeakHoverTarget(target: TargetDescriptor): boolean {
|
function isWeakHoverTarget(target: CaptureTargetDescriptor): boolean {
|
||||||
return !target.role && !target.name && target.tag === "div";
|
return !target.role && !target.name && target.tag === "div";
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -261,9 +299,9 @@ function looksLikeAvatarElement(el: Element): boolean {
|
|||||||
|
|
||||||
function normalizeHoverTarget(
|
function normalizeHoverTarget(
|
||||||
el: Element,
|
el: Element,
|
||||||
desc: TargetDescriptor,
|
desc: CaptureTargetDescriptor,
|
||||||
decision: HoverTriggerDecision,
|
decision: HoverTriggerDecision,
|
||||||
): TargetDescriptor {
|
): CaptureTargetDescriptor {
|
||||||
if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
|
if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
|
||||||
return { ...desc, name: "image" };
|
return { ...desc, name: "image" };
|
||||||
}
|
}
|
||||||
@@ -277,7 +315,7 @@ function normalizeHoverTarget(
|
|||||||
return desc;
|
return desc;
|
||||||
}
|
}
|
||||||
|
|
||||||
function hoverTriggerSignals(el: Element, desc: TargetDescriptor) {
|
function hoverTriggerSignals(el: Element, desc: CaptureTargetDescriptor) {
|
||||||
if (!(el instanceof HTMLElement)) return null;
|
if (!(el instanceof HTMLElement)) return null;
|
||||||
const style = hoverTriggerStyle(el);
|
const style = hoverTriggerStyle(el);
|
||||||
return {
|
return {
|
||||||
@@ -370,12 +408,37 @@ function scheduleInputCompletionCommit(
|
|||||||
setTimeout(syncAndCommit, 0);
|
setTimeout(syncAndCommit, 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface RecordCaptureOptions {
|
||||||
|
frameMode?: RecordFrameMode;
|
||||||
|
}
|
||||||
|
|
||||||
export function startRecordCapture(
|
export function startRecordCapture(
|
||||||
_requestId: string,
|
_requestId: string,
|
||||||
sendStep: (step: RecordStepPayload) => void,
|
sendStep: (step: RecordStepPayload) => void,
|
||||||
|
options: RecordCaptureOptions = {},
|
||||||
): RecordCaptureController {
|
): RecordCaptureController {
|
||||||
|
const frameMode = options.frameMode ?? "top";
|
||||||
|
const emitNavigate = frameMode === "top";
|
||||||
|
const documentToken = createRecordDocumentToken();
|
||||||
|
stampRecordDocumentToken(documentToken);
|
||||||
|
const capturedAtMs = () => Date.now();
|
||||||
|
|
||||||
|
const resolveTopPageUrl = (): string => {
|
||||||
|
try {
|
||||||
|
return window.top?.location.href ?? location.href;
|
||||||
|
} catch {
|
||||||
|
return location.href;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
const emitStep = (step: RecordStepPayload) => {
|
const emitStep = (step: RecordStepPayload) => {
|
||||||
sendStep({ page_url: location.href, ...step });
|
sendStep({
|
||||||
|
page_url: resolveTopPageUrl(),
|
||||||
|
frame_url: location.href,
|
||||||
|
documentToken,
|
||||||
|
capturedAtMs: capturedAtMs(),
|
||||||
|
...step,
|
||||||
|
});
|
||||||
};
|
};
|
||||||
const hoverSurfaceStateMap = (states: HoverSurfaceState[]): Map<Element, string> =>
|
const hoverSurfaceStateMap = (states: HoverSurfaceState[]): Map<Element, string> =>
|
||||||
new Map(states.map((state) => [state.element, state.signature]));
|
new Map(states.map((state) => [state.element, state.signature]));
|
||||||
@@ -411,14 +474,17 @@ export function startRecordCapture(
|
|||||||
emitStep({
|
emitStep({
|
||||||
op: "fill",
|
op: "fill",
|
||||||
target: session.target,
|
target: session.target,
|
||||||
|
geometry: captureGeometry(session.element),
|
||||||
value,
|
value,
|
||||||
|
commit: session.pendingCommit ?? "blur",
|
||||||
...(isPassword ? { redacted: true } : {}),
|
...(isPassword ? { redacted: true } : {}),
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
const commitFillSession = () => {
|
const commitFillSession = (commit: "enter" | "suggestion" | "blur" = "blur") => {
|
||||||
if (!fillSession || composing) return;
|
if (!fillSession || composing) return;
|
||||||
const session = fillSession;
|
const session = fillSession;
|
||||||
|
session.pendingCommit = commit;
|
||||||
fillSession = null;
|
fillSession = null;
|
||||||
emitFill(session);
|
emitFill(session);
|
||||||
committedValues.set(session.element, session.lastValue);
|
committedValues.set(session.element, session.lastValue);
|
||||||
@@ -443,6 +509,7 @@ export function startRecordCapture(
|
|||||||
};
|
};
|
||||||
|
|
||||||
const emitNavigateIfChanged = (causedByAction?: boolean) => {
|
const emitNavigateIfChanged = (causedByAction?: boolean) => {
|
||||||
|
if (!emitNavigate) return;
|
||||||
if (location.href === lastUrl) return;
|
if (location.href === lastUrl) return;
|
||||||
commitFillSession();
|
commitFillSession();
|
||||||
lastUrl = location.href;
|
lastUrl = location.href;
|
||||||
@@ -624,6 +691,7 @@ export function startRecordCapture(
|
|||||||
emitStep({
|
emitStep({
|
||||||
op: "hover",
|
op: "hover",
|
||||||
target: hover.target,
|
target: hover.target,
|
||||||
|
geometry: captureGeometry(hover.element),
|
||||||
});
|
});
|
||||||
emittedHoverElements.add(hover.element);
|
emittedHoverElements.add(hover.element);
|
||||||
};
|
};
|
||||||
@@ -678,6 +746,7 @@ export function startRecordCapture(
|
|||||||
emitStep({
|
emitStep({
|
||||||
op: "click",
|
op: "click",
|
||||||
target,
|
target,
|
||||||
|
geometry: geometryForEventTarget(eventTarget(event)),
|
||||||
expects_navigation: true,
|
expects_navigation: true,
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
@@ -779,7 +848,7 @@ export function startRecordCapture(
|
|||||||
scheduleInputCompletionCommit(
|
scheduleInputCompletionCommit(
|
||||||
sessionElement,
|
sessionElement,
|
||||||
syncFillSessionValue,
|
syncFillSessionValue,
|
||||||
commitFillSession,
|
() => commitFillSession("suggestion"),
|
||||||
(el) => fillSession?.element === el,
|
(el) => fillSession?.element === el,
|
||||||
);
|
);
|
||||||
return;
|
return;
|
||||||
@@ -839,6 +908,7 @@ export function startRecordCapture(
|
|||||||
emitStep({
|
emitStep({
|
||||||
op: "select",
|
op: "select",
|
||||||
target: desc,
|
target: desc,
|
||||||
|
geometry: captureGeometry(target),
|
||||||
values,
|
values,
|
||||||
labels,
|
labels,
|
||||||
expects_navigation: true,
|
expects_navigation: true,
|
||||||
@@ -879,7 +949,7 @@ export function startRecordCapture(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
if (fillable) {
|
if (fillable) {
|
||||||
commitFillSession();
|
commitFillSession(event.key === "Enter" ? "enter" : "blur");
|
||||||
}
|
}
|
||||||
const desc = describeEventTarget(target);
|
const desc = describeEventTarget(target);
|
||||||
if (!desc && !event.key) return;
|
if (!desc && !event.key) return;
|
||||||
@@ -887,7 +957,7 @@ export function startRecordCapture(
|
|||||||
emitStep({
|
emitStep({
|
||||||
op: "press",
|
op: "press",
|
||||||
key: event.key,
|
key: event.key,
|
||||||
...(desc ? { target: desc } : {}),
|
...(desc ? { target: desc, geometry: geometryForEventTarget(target) } : {}),
|
||||||
...(modifiers.length ? { modifiers } : {}),
|
...(modifiers.length ? { modifiers } : {}),
|
||||||
expects_navigation: event.key === "Enter",
|
expects_navigation: event.key === "Enter",
|
||||||
});
|
});
|
||||||
@@ -906,9 +976,12 @@ export function startRecordCapture(
|
|||||||
const urlObserver = new MutationObserver(() => emitNavigateIfChanged());
|
const urlObserver = new MutationObserver(() => emitNavigateIfChanged());
|
||||||
urlObserver.observe(document, { subtree: true, childList: true });
|
urlObserver.observe(document, { subtree: true, childList: true });
|
||||||
const onUrlEvent = () => emitNavigateIfChanged();
|
const onUrlEvent = () => emitNavigateIfChanged();
|
||||||
|
// Wrap: passing commitFillSession directly would forward the DOM event as
|
||||||
|
// the `commit` argument and stamp it onto the recorded fill step.
|
||||||
|
const onPageHide = () => commitFillSession();
|
||||||
window.addEventListener("hashchange", onUrlEvent);
|
window.addEventListener("hashchange", onUrlEvent);
|
||||||
window.addEventListener("popstate", onUrlEvent);
|
window.addEventListener("popstate", onUrlEvent);
|
||||||
window.addEventListener("pagehide", commitFillSession);
|
window.addEventListener("pagehide", onPageHide);
|
||||||
|
|
||||||
const originalPushState = history.pushState;
|
const originalPushState = history.pushState;
|
||||||
const originalReplaceState = history.replaceState;
|
const originalReplaceState = history.replaceState;
|
||||||
@@ -924,6 +997,7 @@ export function startRecordCapture(
|
|||||||
return {
|
return {
|
||||||
dispose() {
|
dispose() {
|
||||||
commitFillSession();
|
commitFillSession();
|
||||||
|
clearRecordDocumentToken();
|
||||||
document.removeEventListener("click", onClick, true);
|
document.removeEventListener("click", onClick, true);
|
||||||
document.removeEventListener("mouseover", onMouseOver, true);
|
document.removeEventListener("mouseover", onMouseOver, true);
|
||||||
document.removeEventListener("focusin", onFocusIn, true);
|
document.removeEventListener("focusin", onFocusIn, true);
|
||||||
@@ -936,7 +1010,7 @@ export function startRecordCapture(
|
|||||||
urlObserver.disconnect();
|
urlObserver.disconnect();
|
||||||
window.removeEventListener("hashchange", onUrlEvent);
|
window.removeEventListener("hashchange", onUrlEvent);
|
||||||
window.removeEventListener("popstate", onUrlEvent);
|
window.removeEventListener("popstate", onUrlEvent);
|
||||||
window.removeEventListener("pagehide", commitFillSession);
|
window.removeEventListener("pagehide", onPageHide);
|
||||||
history.pushState = originalPushState;
|
history.pushState = originalPushState;
|
||||||
history.replaceState = originalReplaceState;
|
history.replaceState = originalReplaceState;
|
||||||
},
|
},
|
||||||
@@ -956,8 +1030,8 @@ export function handleRecordContentMessage(
|
|||||||
capture: RecordCaptureController | null;
|
capture: RecordCaptureController | null;
|
||||||
setActiveRequestId(id: string | null): void;
|
setActiveRequestId(id: string | null): void;
|
||||||
setCapture(capture: RecordCaptureController | null): void;
|
setCapture(capture: RecordCaptureController | null): void;
|
||||||
onStart(requestId: string, startedAtMs?: number): void;
|
onStart?(requestId: string, startedAtMs?: number): void;
|
||||||
onStop(): void;
|
onStop?(): void;
|
||||||
},
|
},
|
||||||
sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
|
sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
|
||||||
): boolean {
|
): boolean {
|
||||||
@@ -965,15 +1039,23 @@ export function handleRecordContentMessage(
|
|||||||
if (!knownRecordRequests.has(message.requestId)) {
|
if (!knownRecordRequests.has(message.requestId)) {
|
||||||
knownRecordRequests.add(message.requestId);
|
knownRecordRequests.add(message.requestId);
|
||||||
failedStepDeliveries.delete(message.requestId);
|
failedStepDeliveries.delete(message.requestId);
|
||||||
|
failedStepPayloads.delete(message.requestId);
|
||||||
}
|
}
|
||||||
state.capture?.dispose();
|
state.capture?.dispose();
|
||||||
|
const frameMode = message.frameMode ?? "top";
|
||||||
state.setCapture(
|
state.setCapture(
|
||||||
startRecordCapture(message.requestId, (step) => {
|
startRecordCapture(
|
||||||
sendRecordStep(message.requestId, step);
|
message.requestId,
|
||||||
}),
|
(step) => {
|
||||||
|
sendRecordStep(message.requestId, step);
|
||||||
|
},
|
||||||
|
{ frameMode },
|
||||||
|
),
|
||||||
);
|
);
|
||||||
state.setActiveRequestId(message.requestId);
|
state.setActiveRequestId(message.requestId);
|
||||||
state.onStart(message.requestId, message.startedAtMs);
|
if (message.showOverlay !== false && frameMode === "top") {
|
||||||
|
state.onStart?.(message.requestId, message.startedAtMs);
|
||||||
|
}
|
||||||
sendResponse?.({ ok: true });
|
sendResponse?.({ ok: true });
|
||||||
return sendResponse !== undefined;
|
return sendResponse !== undefined;
|
||||||
}
|
}
|
||||||
@@ -988,31 +1070,55 @@ export function handleRecordContentMessage(
|
|||||||
state.capture?.dispose();
|
state.capture?.dispose();
|
||||||
state.setCapture(null);
|
state.setCapture(null);
|
||||||
const finishStop = () => {
|
const finishStop = () => {
|
||||||
state.onStop();
|
state.onStop?.();
|
||||||
state.setActiveRequestId(null);
|
state.setActiveRequestId(null);
|
||||||
};
|
};
|
||||||
if (isRecordStopMessage(message) && sendResponse) {
|
if (isRecordStopMessage(message) && sendResponse) {
|
||||||
|
const existingFlush = pendingStopFlushes.get(message.requestId);
|
||||||
|
if (existingFlush) {
|
||||||
|
void existingFlush.then(sendResponse);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
const failedBeforeStop = [...(failedStepPayloads.get(message.requestId) ?? [])];
|
||||||
|
if (failedBeforeStop.length > 0) {
|
||||||
|
failedStepPayloads.delete(message.requestId);
|
||||||
|
failedStepDeliveries.delete(message.requestId);
|
||||||
|
}
|
||||||
const pending = [...(pendingStepSends.get(message.requestId) ?? [])];
|
const pending = [...(pendingStepSends.get(message.requestId) ?? [])];
|
||||||
void Promise.all(pending).then((delivered) => {
|
const flush = Promise.all(pending).then(async (delivered): Promise<RecordStopAck> => {
|
||||||
finishStop();
|
const retried = await Promise.all(
|
||||||
const succeeded = delivered.every(Boolean) && !failedStepDeliveries.has(message.requestId);
|
failedBeforeStop.map((step) => deliverRecordStep(message.requestId, step)),
|
||||||
|
);
|
||||||
|
const succeeded =
|
||||||
|
delivered.every(Boolean) &&
|
||||||
|
retried.every(Boolean) &&
|
||||||
|
!failedStepDeliveries.has(message.requestId);
|
||||||
if (succeeded) {
|
if (succeeded) {
|
||||||
failedStepDeliveries.delete(message.requestId);
|
failedStepDeliveries.delete(message.requestId);
|
||||||
|
failedStepPayloads.delete(message.requestId);
|
||||||
knownRecordRequests.delete(message.requestId);
|
knownRecordRequests.delete(message.requestId);
|
||||||
|
finishStop();
|
||||||
|
}
|
||||||
|
return succeeded
|
||||||
|
? { ok: true }
|
||||||
|
: {
|
||||||
|
ok: false,
|
||||||
|
error: "failed to deliver one or more recorded steps",
|
||||||
|
};
|
||||||
|
});
|
||||||
|
pendingStopFlushes.set(message.requestId, flush);
|
||||||
|
void flush.then(sendResponse).finally(() => {
|
||||||
|
if (pendingStopFlushes.get(message.requestId) === flush) {
|
||||||
|
pendingStopFlushes.delete(message.requestId);
|
||||||
}
|
}
|
||||||
sendResponse(
|
|
||||||
succeeded
|
|
||||||
? { ok: true }
|
|
||||||
: {
|
|
||||||
ok: false,
|
|
||||||
error: "failed to deliver one or more recorded steps",
|
|
||||||
},
|
|
||||||
);
|
|
||||||
});
|
});
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
if (isRecordCancelMessage(message)) {
|
if (isRecordCancelMessage(message)) {
|
||||||
failedStepDeliveries.delete(message.requestId);
|
failedStepDeliveries.delete(message.requestId);
|
||||||
|
failedStepPayloads.delete(message.requestId);
|
||||||
|
pendingStopFlushes.delete(message.requestId);
|
||||||
knownRecordRequests.delete(message.requestId);
|
knownRecordRequests.delete(message.requestId);
|
||||||
}
|
}
|
||||||
finishStop();
|
finishStop();
|
||||||
|
|||||||
@@ -33,6 +33,9 @@ import {
|
|||||||
defaultBorrowChromeWindows,
|
defaultBorrowChromeWindows,
|
||||||
requestBorrowConfirmation,
|
requestBorrowConfirmation,
|
||||||
} from "@/tools/borrow-confirmation";
|
} from "@/tools/borrow-confirmation";
|
||||||
|
import {
|
||||||
|
type RecordCaptureStatusMessage,
|
||||||
|
} from "@/lib/record-bridge";
|
||||||
import { ToolDispatcher } from "@/tools/dispatcher";
|
import { ToolDispatcher } from "@/tools/dispatcher";
|
||||||
import {
|
import {
|
||||||
attachRecordFinishListener,
|
attachRecordFinishListener,
|
||||||
@@ -172,8 +175,27 @@ export default defineBackground(() => {
|
|||||||
dispatcher.start();
|
dispatcher.start();
|
||||||
const recordDeps = {
|
const recordDeps = {
|
||||||
tabsApi: chrome.tabs,
|
tabsApi: chrome.tabs,
|
||||||
|
cdp,
|
||||||
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
|
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
|
||||||
chrome.tabs.sendMessage(tabId, msg),
|
chrome.tabs.sendMessage(tabId, msg),
|
||||||
|
sendToDocument: (
|
||||||
|
tabId: number,
|
||||||
|
documentId: string,
|
||||||
|
msg: Parameters<typeof chrome.tabs.sendMessage>[1],
|
||||||
|
) => chrome.tabs.sendMessage(tabId, msg, { documentId }),
|
||||||
|
getAllFrames: async (tabId: number) => {
|
||||||
|
if (!chrome.webNavigation?.getAllFrames) return [];
|
||||||
|
const frames = await chrome.webNavigation.getAllFrames({ tabId });
|
||||||
|
return (frames ?? []).map((frame) => ({
|
||||||
|
frameId: frame.frameId,
|
||||||
|
documentId: frame.documentId,
|
||||||
|
url: frame.url,
|
||||||
|
parentFrameId: frame.parentFrameId,
|
||||||
|
}));
|
||||||
|
},
|
||||||
|
broadcastCaptureStatus: async (tabId: number, msg: RecordCaptureStatusMessage) => {
|
||||||
|
await chrome.tabs.sendMessage(tabId, msg, { frameId: 0 });
|
||||||
|
},
|
||||||
bypassOverlay: async (tabId: number, enabled: boolean) => {
|
bypassOverlay: async (tabId: number, enabled: boolean) => {
|
||||||
try {
|
try {
|
||||||
await chrome.tabs.sendMessage(tabId, {
|
await chrome.tabs.sendMessage(tabId, {
|
||||||
@@ -188,7 +210,7 @@ export default defineBackground(() => {
|
|||||||
// Message listeners stay up (cheap; fire only for record message types).
|
// Message listeners stay up (cheap; fire only for record message types).
|
||||||
// Tab / webNavigation observation attaches lazily while a recording is
|
// Tab / webNavigation observation attaches lazily while a recording is
|
||||||
// active — see ensureBrowserObservationListeners in tools/record.ts.
|
// active — see ensureBrowserObservationListeners in tools/record.ts.
|
||||||
attachRecordStepListener();
|
attachRecordStepListener(recordDeps);
|
||||||
attachRecordFinishListener(recordDeps);
|
attachRecordFinishListener(recordDeps);
|
||||||
attachRecordQueryListener(recordDeps);
|
attachRecordQueryListener(recordDeps);
|
||||||
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
|
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
|
||||||
|
|||||||
@@ -44,12 +44,15 @@ import {
|
|||||||
} from "@/lib/overlay-bridge";
|
} from "@/lib/overlay-bridge";
|
||||||
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
|
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
|
||||||
import {
|
import {
|
||||||
|
isRecordCaptureStatusMessage,
|
||||||
RECORD_FINISH,
|
RECORD_FINISH,
|
||||||
RECORD_QUERY,
|
RECORD_QUERY,
|
||||||
|
type RecordCaptureStatusMessage,
|
||||||
type RecordQueryResponse,
|
type RecordQueryResponse,
|
||||||
type RecordStartAck,
|
type RecordStartAck,
|
||||||
type RecordStopAck,
|
type RecordStopAck,
|
||||||
} from "@/lib/record-bridge";
|
} from "@/lib/record-bridge";
|
||||||
|
import type { FrameCaptureFailure, FrameCaptureStatus } from "@/transport/types";
|
||||||
import type {
|
import type {
|
||||||
BorrowCancelMessage,
|
BorrowCancelMessage,
|
||||||
BorrowRequestMessage,
|
BorrowRequestMessage,
|
||||||
@@ -70,6 +73,8 @@ export default defineContentScript({
|
|||||||
const overlays = new OverlayController();
|
const overlays = new OverlayController();
|
||||||
let recordCapture: RecordCaptureController | null = null;
|
let recordCapture: RecordCaptureController | null = null;
|
||||||
let activeRecordRequestId: string | null = null;
|
let activeRecordRequestId: string | null = null;
|
||||||
|
let recordCaptureStatus: FrameCaptureStatus = "complete";
|
||||||
|
let recordCaptureFailures: FrameCaptureFailure[] = [];
|
||||||
let reactRoot: ReactDOM.Root | null = null;
|
let reactRoot: ReactDOM.Root | null = null;
|
||||||
let overlayHost: HTMLElement | null = null;
|
let overlayHost: HTMLElement | null = null;
|
||||||
let overlayContainer: HTMLElement | null = null;
|
let overlayContainer: HTMLElement | null = null;
|
||||||
@@ -188,7 +193,11 @@ export default defineContentScript({
|
|||||||
requests: overlayState.borrowRequests,
|
requests: overlayState.borrowRequests,
|
||||||
}),
|
}),
|
||||||
React.createElement(HelpRequestOverlay, { request: overlayState.activeHelp }),
|
React.createElement(HelpRequestOverlay, { request: overlayState.activeHelp }),
|
||||||
React.createElement(RecordOverlay, { request: overlayState.activeRecord }),
|
React.createElement(RecordOverlay, {
|
||||||
|
request: overlayState.activeRecord,
|
||||||
|
captureStatus: recordCaptureStatus,
|
||||||
|
captureFailures: recordCaptureFailures,
|
||||||
|
}),
|
||||||
React.createElement(ControlOverlay, {
|
React.createElement(ControlOverlay, {
|
||||||
visible: controlOverlayVisible,
|
visible: controlOverlayVisible,
|
||||||
interrupting: overlayState.interrupting,
|
interrupting: overlayState.interrupting,
|
||||||
@@ -260,7 +269,8 @@ export default defineContentScript({
|
|||||||
| CaptureSuppressMessage
|
| CaptureSuppressMessage
|
||||||
| OverlayAgentOverlayResetMessage
|
| OverlayAgentOverlayResetMessage
|
||||||
| OverlayAgentStateMessage
|
| OverlayAgentStateMessage
|
||||||
| OverlayAutomationBypassMessage,
|
| OverlayAutomationBypassMessage
|
||||||
|
| RecordCaptureStatusMessage,
|
||||||
_sender: chrome.runtime.MessageSender,
|
_sender: chrome.runtime.MessageSender,
|
||||||
sendResponse: (response: BorrowResponseMessage | HelpAckMessage | CaptureSuppressAck) => void,
|
sendResponse: (response: BorrowResponseMessage | HelpAckMessage | CaptureSuppressAck) => void,
|
||||||
) => {
|
) => {
|
||||||
@@ -268,6 +278,13 @@ export default defineContentScript({
|
|||||||
return captureSuppress.handleMessage(message, sendResponse);
|
return captureSuppress.handleMessage(message, sendResponse);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (isRecordCaptureStatusMessage(message)) {
|
||||||
|
recordCaptureStatus = message.status;
|
||||||
|
recordCaptureFailures = message.failures ?? [];
|
||||||
|
renderAll();
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
if (isRecordContentMessage(message)) {
|
if (isRecordContentMessage(message)) {
|
||||||
const needsAsync = handleRecordContentMessage(
|
const needsAsync = handleRecordContentMessage(
|
||||||
message,
|
message,
|
||||||
@@ -295,6 +312,8 @@ export default defineContentScript({
|
|||||||
},
|
},
|
||||||
onStop: () => {
|
onStop: () => {
|
||||||
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
|
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
|
||||||
|
recordCaptureStatus = "complete";
|
||||||
|
recordCaptureFailures = [];
|
||||||
renderAll();
|
renderAll();
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
@@ -449,6 +468,8 @@ export default defineContentScript({
|
|||||||
) {
|
) {
|
||||||
const requestId = recordQuery.requestId;
|
const requestId = recordQuery.requestId;
|
||||||
const startedAtMs = recordQuery.startedAtMs;
|
const startedAtMs = recordQuery.startedAtMs;
|
||||||
|
recordCaptureStatus = recordQuery.captureStatus ?? "complete";
|
||||||
|
recordCaptureFailures = recordQuery.captureFailures ?? [];
|
||||||
overlays.setAgentRecordRequest({
|
overlays.setAgentRecordRequest({
|
||||||
id: requestId,
|
id: requestId,
|
||||||
...(typeof startedAtMs === "number" ? { startedAtMs } : {}),
|
...(typeof startedAtMs === "number" ? { startedAtMs } : {}),
|
||||||
|
|||||||
@@ -100,6 +100,24 @@ describe("App", () => {
|
|||||||
expect(screen.queryByText("录制你的操作,供 Agent 参考")).toBeNull();
|
expect(screen.queryByText("录制你的操作,供 Agent 参考")).toBeNull();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("renders localized copy for a version failure instead of protocol details", () => {
|
||||||
|
mockUseConnectionState.mockReturnValue({
|
||||||
|
snapshot: {
|
||||||
|
...baseSnapshot,
|
||||||
|
lastError:
|
||||||
|
"[handshake] daemon rejected handshake: version_too_old — peer protocol 1.0 is below local min_compatible_protocol 1.1",
|
||||||
|
},
|
||||||
|
statusState: "disconnected",
|
||||||
|
setLabel,
|
||||||
|
setConnectionEnabled,
|
||||||
|
});
|
||||||
|
|
||||||
|
render(<App />);
|
||||||
|
|
||||||
|
const error = screen.getByText(/bsk 与 BrowserSkill 扩展版本不兼容/);
|
||||||
|
expect(error.textContent).not.toContain("min_compatible_protocol");
|
||||||
|
});
|
||||||
|
|
||||||
it("shows single-line compact metadata and copies the instance id", async () => {
|
it("shows single-line compact metadata and copies the instance id", async () => {
|
||||||
mockUseConnectionState.mockReturnValue({
|
mockUseConnectionState.mockReturnValue({
|
||||||
snapshot: {
|
snapshot: {
|
||||||
@@ -200,6 +218,13 @@ describe("App", () => {
|
|||||||
expect(copied).toContain("--browser 03c3e47f");
|
expect(copied).toContain("--browser 03c3e47f");
|
||||||
expect(copied).toContain('--purpose "发布 wiki 文档"');
|
expect(copied).toContain('--purpose "发布 wiki 文档"');
|
||||||
expect(copied).not.toMatch(/bsk record start[^\n]*--url/);
|
expect(copied).not.toMatch(/bsk record start[^\n]*--url/);
|
||||||
|
expect(copied).toContain("./trace");
|
||||||
|
expect(copied).toContain("trace.json");
|
||||||
|
expect(copied).toContain("pages/");
|
||||||
|
expect(copied).toContain("总结我做了什么");
|
||||||
|
expect(copied).not.toContain("不要重跑");
|
||||||
|
expect(copied).not.toContain("pages + steps");
|
||||||
|
expect(copied).not.toContain("@eN");
|
||||||
// Button label stays static; a transient toast confirms the copy.
|
// Button label stays static; a transient toast confirms the copy.
|
||||||
expect(copyButton.textContent).toContain("复制录制指令");
|
expect(copyButton.textContent).toContain("复制录制指令");
|
||||||
await waitFor(() => expect(screen.getByRole("status").textContent).toContain("已复制"));
|
await waitFor(() => expect(screen.getByRole("status").textContent).toContain("已复制"));
|
||||||
|
|||||||
@@ -83,6 +83,9 @@ export function App() {
|
|||||||
const daemonProtocol = snapshot.handshake?.protocol_version ?? "—";
|
const daemonProtocol = snapshot.handshake?.protocol_version ?? "—";
|
||||||
const extensionVersion = snapshot.extensionVersion || "—";
|
const extensionVersion = snapshot.extensionVersion || "—";
|
||||||
const instanceId = snapshot.instanceId || "—";
|
const instanceId = snapshot.instanceId || "—";
|
||||||
|
const errorMessage = snapshot.lastError?.includes("version_too_old")
|
||||||
|
? t("popup.versionIncompatibleError")
|
||||||
|
: snapshot.lastError;
|
||||||
|
|
||||||
const copyInstanceId = async () => {
|
const copyInstanceId = async () => {
|
||||||
if (!snapshot.instanceId || !navigator.clipboard?.writeText) return;
|
if (!snapshot.instanceId || !navigator.clipboard?.writeText) return;
|
||||||
@@ -252,12 +255,12 @@ export function App() {
|
|||||||
</div>
|
</div>
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
{snapshot.lastError && (
|
{errorMessage && (
|
||||||
<div
|
<div
|
||||||
className="rounded-lg border border-destructive/25 bg-destructive/10 px-3 py-2 text-xs leading-snug text-destructive"
|
className="rounded-lg border border-destructive/25 bg-destructive/10 px-3 py-2 text-xs leading-snug text-destructive"
|
||||||
data-slot="popup-error"
|
data-slot="popup-error"
|
||||||
>
|
>
|
||||||
{snapshot.lastError}
|
{errorMessage}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,77 @@
|
|||||||
|
import {
|
||||||
|
handleRecordContentMessage,
|
||||||
|
isRecordContentMessage,
|
||||||
|
type RecordCaptureController,
|
||||||
|
} from "@/content/record-capture";
|
||||||
|
import {
|
||||||
|
RECORD_QUERY,
|
||||||
|
type RecordQueryResponse,
|
||||||
|
} from "@/lib/record-bridge";
|
||||||
|
|
||||||
|
/** Capture-only content script for child frames (no React overlay). */
|
||||||
|
export default defineContentScript({
|
||||||
|
matches: ["<all_urls>"],
|
||||||
|
runAt: "document_end",
|
||||||
|
allFrames: true,
|
||||||
|
matchAboutBlank: true,
|
||||||
|
matchOriginAsFallback: true,
|
||||||
|
|
||||||
|
main(ctx) {
|
||||||
|
if (window.top === window) return;
|
||||||
|
|
||||||
|
let recordCapture: RecordCaptureController | null = null;
|
||||||
|
let activeRecordRequestId: string | null = null;
|
||||||
|
|
||||||
|
const onMessage = (
|
||||||
|
message: unknown,
|
||||||
|
_sender: chrome.runtime.MessageSender,
|
||||||
|
sendResponse: (response: unknown) => void,
|
||||||
|
) => {
|
||||||
|
if (isRecordContentMessage(message)) {
|
||||||
|
const needsAsync = handleRecordContentMessage(
|
||||||
|
message,
|
||||||
|
{
|
||||||
|
activeRequestId: activeRecordRequestId,
|
||||||
|
capture: recordCapture,
|
||||||
|
setActiveRequestId: (id) => {
|
||||||
|
activeRecordRequestId = id;
|
||||||
|
},
|
||||||
|
setCapture: (capture) => {
|
||||||
|
recordCapture = capture;
|
||||||
|
},
|
||||||
|
},
|
||||||
|
sendResponse as never,
|
||||||
|
);
|
||||||
|
return needsAsync;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
|
||||||
|
async function queryActiveRecord(): Promise<void> {
|
||||||
|
try {
|
||||||
|
const recordQuery = (await chrome.runtime.sendMessage({
|
||||||
|
type: RECORD_QUERY,
|
||||||
|
})) as RecordQueryResponse | undefined;
|
||||||
|
if (
|
||||||
|
recordQuery?.active &&
|
||||||
|
typeof recordQuery.requestId === "string" &&
|
||||||
|
activeRecordRequestId === null
|
||||||
|
) {
|
||||||
|
// Background rearm sends RECORD_START; nothing to mount locally.
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
console.debug("[bsk record-frame] record query failed", err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
chrome.runtime.onMessage.addListener(onMessage);
|
||||||
|
void queryActiveRecord();
|
||||||
|
|
||||||
|
ctx.onInvalidated(() => {
|
||||||
|
chrome.runtime.onMessage.removeListener(onMessage);
|
||||||
|
recordCapture?.dispose();
|
||||||
|
recordCapture = null;
|
||||||
|
activeRecordRequestId = null;
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
@@ -27,19 +27,19 @@ function handshake(
|
|||||||
|
|
||||||
describe("computeConnectedState (protocol-based compat)", () => {
|
describe("computeConnectedState (protocol-based compat)", () => {
|
||||||
it("returns connected when protocol strings match", () => {
|
it("returns connected when protocol strings match", () => {
|
||||||
expect(computeConnectedState(handshake("1.0", "1.0"), MIN_COMPATIBLE_PROTOCOL)).toEqual({
|
expect(computeConnectedState(handshake("1.1", "1.1"), MIN_COMPATIBLE_PROTOCOL)).toEqual({
|
||||||
kind: "connected",
|
kind: "connected",
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns version_skew when daemon protocol minor is newer", () => {
|
it("returns version_skew when daemon protocol minor is newer", () => {
|
||||||
expect(computeConnectedState(handshake("1.1", "1.0"))).toEqual({
|
expect(computeConnectedState(handshake("1.2", "1.1"))).toEqual({
|
||||||
kind: "version_skew",
|
kind: "version_skew",
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("returns version_skew when daemon protocol string differs but floor is satisfied", () => {
|
it("returns version_skew when daemon protocol string differs but floor is satisfied", () => {
|
||||||
expect(computeConnectedState(handshake("1", "1.0"))).toEqual({
|
expect(computeConnectedState(handshake("1.1.0", "1.1"))).toEqual({
|
||||||
kind: "version_skew",
|
kind: "version_skew",
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -48,16 +48,19 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
|||||||
const result = computeConnectedState(handshake("2.0", "1.0"));
|
const result = computeConnectedState(handshake("2.0", "1.0"));
|
||||||
expect(result.kind).toBe("rejected");
|
expect(result.kind).toBe("rejected");
|
||||||
if (result.kind === "rejected") {
|
if (result.kind === "rejected") {
|
||||||
expect(result.reason).toContain("protocol-major mismatch");
|
expect(result.reason).toContain("bsk CLI");
|
||||||
|
expect(result.reason).toContain("BrowserSkill extension");
|
||||||
|
expect(result.reason).not.toContain("protocol");
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
it("rejects when extension is below daemon min_compatible_protocol", () => {
|
it("rejects when extension is below daemon min_compatible_protocol", () => {
|
||||||
const result = computeConnectedState(handshake("1.0", "1.5"));
|
const result = computeConnectedState(handshake("1.1", "1.5"));
|
||||||
expect(result.kind).toBe("rejected");
|
expect(result.kind).toBe("rejected");
|
||||||
if (result.kind === "rejected") {
|
if (result.kind === "rejected") {
|
||||||
expect(result.reason).toContain("min_compatible_protocol");
|
expect(result.reason).toContain("BrowserSkill extension is out of date");
|
||||||
expect(result.reason).toContain("1.5");
|
expect(result.reason).toContain("Update and reload the extension");
|
||||||
|
expect(result.reason).not.toContain("min_compatible_protocol");
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -65,7 +68,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
|||||||
const result = computeConnectedState({
|
const result = computeConnectedState({
|
||||||
server: "browser-skill-daemon",
|
server: "browser-skill-daemon",
|
||||||
version: "0.1.0",
|
version: "0.1.0",
|
||||||
protocol_version: "1.0",
|
protocol_version: "1.1",
|
||||||
min_compatible_peer: "0.1.0",
|
min_compatible_peer: "0.1.0",
|
||||||
});
|
});
|
||||||
expect(result).toEqual({ kind: "connected" });
|
expect(result).toEqual({ kind: "connected" });
|
||||||
@@ -75,16 +78,23 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
|||||||
const result = computeConnectedState(handshake("1.0", "1.0"), "1.1");
|
const result = computeConnectedState(handshake("1.0", "1.0"), "1.1");
|
||||||
expect(result.kind).toBe("rejected");
|
expect(result.kind).toBe("rejected");
|
||||||
if (result.kind === "rejected") {
|
if (result.kind === "rejected") {
|
||||||
expect(result.reason).toContain("below extension min_compatible_protocol");
|
expect(result.reason).toContain("bsk CLI is out of date");
|
||||||
|
expect(result.reason).toContain("bsk daemon restart");
|
||||||
|
expect(result.reason).not.toContain("min_compatible_protocol");
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("rejects a pre-Trace-v3 daemon using the configured extension floor", () => {
|
||||||
|
const result = computeConnectedState(handshake("1.0", "1.0"));
|
||||||
|
expect(result.kind).toBe("rejected");
|
||||||
|
});
|
||||||
|
|
||||||
it("rejects malformed daemon min_compatible_protocol with a daemon-floor reason", () => {
|
it("rejects malformed daemon min_compatible_protocol with a daemon-floor reason", () => {
|
||||||
const result = computeConnectedState(handshake("1.0", "not-a-protocol"));
|
const result = computeConnectedState(handshake("1.1", "not-a-protocol"));
|
||||||
expect(result.kind).toBe("rejected");
|
expect(result.kind).toBe("rejected");
|
||||||
if (result.kind === "rejected") {
|
if (result.kind === "rejected") {
|
||||||
expect(result.reason).toContain("daemon min_compatible_protocol");
|
expect(result.reason).toContain("could not verify version compatibility");
|
||||||
expect(result.reason).toContain("not-a-protocol");
|
expect(result.reason).not.toContain("min_compatible_protocol");
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -242,11 +252,11 @@ describe("ConnectionController connectionEnabled", () => {
|
|||||||
const second = transport.send.mock.calls[1]?.[0] as { id: string };
|
const second = transport.send.mock.calls[1]?.[0] as { id: string };
|
||||||
expect(second.id).not.toBe(first.id);
|
expect(second.id).not.toBe(first.id);
|
||||||
|
|
||||||
transport.emitMessage({ id: first.id, result: handshake("1.0", "1.0") });
|
transport.emitMessage({ id: first.id, result: handshake("1.1", "1.1") });
|
||||||
await Promise.resolve();
|
await Promise.resolve();
|
||||||
expect(controller.snapshot().state).not.toBe("connected");
|
expect(controller.snapshot().state).not.toBe("connected");
|
||||||
|
|
||||||
transport.emitMessage({ id: second.id, result: handshake("1.0", "1.0") });
|
transport.emitMessage({ id: second.id, result: handshake("1.1", "1.1") });
|
||||||
await vi.waitFor(() => expect(controller.snapshot().state).toBe("connected"));
|
await vi.waitFor(() => expect(controller.snapshot().state).toBe("connected"));
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -0,0 +1,172 @@
|
|||||||
|
import type { RenderedRef } from "@browser-skill/vom";
|
||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import type { CapturedNode } from "@/tools/vom/capture";
|
||||||
|
import { matchTarget } from "../match-target";
|
||||||
|
|
||||||
|
function node(backendNodeId: number, overrides: Partial<CapturedNode> = {}): CapturedNode {
|
||||||
|
return {
|
||||||
|
backendNodeId,
|
||||||
|
parentBackendNodeId: null,
|
||||||
|
tag: "button",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
paintOrder: 1,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("matchTarget", () => {
|
||||||
|
const refs: RenderedRef[] = [
|
||||||
|
{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 3 },
|
||||||
|
];
|
||||||
|
|
||||||
|
it("matches a unique node by viewport coordinates", () => {
|
||||||
|
const target = matchTarget({
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
scrollX: 0,
|
||||||
|
scrollY: 0,
|
||||||
|
position: "static",
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
captured: [node(42)],
|
||||||
|
refs,
|
||||||
|
fallback: { tag: "button", name: "发布" },
|
||||||
|
});
|
||||||
|
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("matches a non-fixed node after the top frame scrolls", () => {
|
||||||
|
const target = matchTarget({
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
scrollX: 40,
|
||||||
|
scrollY: 300,
|
||||||
|
position: "static",
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
captured: [
|
||||||
|
node(42, {
|
||||||
|
documentRect: { x: 50, y: 320, w: 100, h: 30 },
|
||||||
|
localRect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
refs,
|
||||||
|
fallback: { tag: "button", name: "发布" },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("matches a static node by document coordinates when scroll changed after observation", () => {
|
||||||
|
const target = matchTarget({
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 10, y: 50, w: 100, h: 30 },
|
||||||
|
scrollX: 0,
|
||||||
|
scrollY: 250,
|
||||||
|
position: "static",
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
captured: [
|
||||||
|
node(42, {
|
||||||
|
documentRect: { x: 10, y: 300, w: 100, h: 30 },
|
||||||
|
localRect: { x: 10, y: 200, w: 100, h: 30 },
|
||||||
|
rect: { x: 10, y: 200, w: 100, h: 30 },
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
refs,
|
||||||
|
fallback: { tag: "button", name: "发布" },
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("returns unmatched when multiple candidates match", () => {
|
||||||
|
const target = matchTarget({
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||||
|
scrollX: 0,
|
||||||
|
scrollY: 0,
|
||||||
|
position: "static",
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
captured: [node(42), node(43)],
|
||||||
|
refs,
|
||||||
|
});
|
||||||
|
expect(target.unmatched).toBe(true);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("uses viewport coordinates for fixed elements", () => {
|
||||||
|
const target = matchTarget({
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 5, y: 5, w: 80, h: 24 },
|
||||||
|
scrollX: 100,
|
||||||
|
scrollY: 200,
|
||||||
|
position: "fixed",
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
captured: [node(42, { position: "fixed", rect: { x: 5, y: 5, w: 80, h: 24 } })],
|
||||||
|
refs,
|
||||||
|
});
|
||||||
|
expect(target.ref).toBe("e1");
|
||||||
|
});
|
||||||
|
|
||||||
|
it("isolates geometry matching to the owner iframe backend node", () => {
|
||||||
|
const sharedRect = { x: 10, y: 20, w: 100, h: 30 };
|
||||||
|
const iframeRefs: RenderedRef[] = [
|
||||||
|
{ ref: "e1", backendNodeId: 42, role: "button", name: "Top", line: 1 },
|
||||||
|
{ ref: "e2", backendNodeId: 99, role: "button", name: "Iframe", line: 2 },
|
||||||
|
];
|
||||||
|
const target = matchTarget({
|
||||||
|
geometry: {
|
||||||
|
rect: sharedRect,
|
||||||
|
scrollX: 0,
|
||||||
|
scrollY: 0,
|
||||||
|
position: "static",
|
||||||
|
tag: "button",
|
||||||
|
ownerFrameBackendNodeId: 13,
|
||||||
|
},
|
||||||
|
captured: [
|
||||||
|
node(42, { rect: sharedRect, documentRect: sharedRect }),
|
||||||
|
node(99, {
|
||||||
|
rect: sharedRect,
|
||||||
|
documentRect: sharedRect,
|
||||||
|
ownerFrameBackendNodeId: 13,
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
refs: iframeRefs,
|
||||||
|
fallback: { tag: "button", name: "Iframe" },
|
||||||
|
});
|
||||||
|
expect(target).toEqual({ ref: "e2", role: "button", name: "Iframe" });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("matches iframe static elements using frame scroll and document coordinates", () => {
|
||||||
|
const iframeRefs: RenderedRef[] = [
|
||||||
|
{ ref: "e2", backendNodeId: 99, role: "button", name: "Iframe", line: 2 },
|
||||||
|
];
|
||||||
|
const target = matchTarget({
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 10, y: 50, w: 100, h: 30 },
|
||||||
|
scrollX: 0,
|
||||||
|
scrollY: 120,
|
||||||
|
position: "static",
|
||||||
|
tag: "button",
|
||||||
|
ownerFrameBackendNodeId: 13,
|
||||||
|
},
|
||||||
|
captured: [
|
||||||
|
node(99, {
|
||||||
|
ownerFrameBackendNodeId: 13,
|
||||||
|
documentRect: { x: 10, y: 170, w: 100, h: 30 },
|
||||||
|
localRect: { x: 10, y: 50, w: 100, h: 30 },
|
||||||
|
rect: { x: 10, y: 50, w: 100, h: 30 },
|
||||||
|
}),
|
||||||
|
],
|
||||||
|
refs: iframeRefs,
|
||||||
|
fallback: { tag: "button", name: "Iframe" },
|
||||||
|
});
|
||||||
|
expect(target).toEqual({ ref: "e2", role: "button", name: "Iframe" });
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -0,0 +1,116 @@
|
|||||||
|
import { beforeEach, describe, expect, it } from "vitest";
|
||||||
|
import type { CapturedNode } from "@/tools/vom/capture";
|
||||||
|
import type { DraftTraceStep } from "@/transport/types";
|
||||||
|
import { resetStateIdCounterForTests } from "../record-constants";
|
||||||
|
import {
|
||||||
|
applyTargetMatching,
|
||||||
|
buildTraceV3,
|
||||||
|
createObservationState,
|
||||||
|
rememberStepOnPage,
|
||||||
|
} from "../record-observation";
|
||||||
|
import { registerObservation } from "../trace-reducer";
|
||||||
|
|
||||||
|
const URL = "https://example.com/login";
|
||||||
|
|
||||||
|
function capturedInput(): CapturedNode {
|
||||||
|
return {
|
||||||
|
backendNodeId: 42,
|
||||||
|
parentBackendNodeId: null,
|
||||||
|
tag: "input",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 20, y: 40, w: 200, h: 30 },
|
||||||
|
localRect: { x: 20, y: 40, w: 200, h: 30 },
|
||||||
|
paintOrder: 1,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function finalizedFillBody(value: string, redactValues: boolean): string {
|
||||||
|
const obs = createObservationState({ redactValues });
|
||||||
|
const rawVomText = '@vom 1\ntextbox "Password" value="••••••" [ref=e1]';
|
||||||
|
const stateId = registerObservation(obs.stateRegistry, { url: URL, rawVomText });
|
||||||
|
obs.lastSettled = {
|
||||||
|
stateId,
|
||||||
|
captured: [capturedInput()],
|
||||||
|
matchNodes: [capturedInput()],
|
||||||
|
documentTokenOwners: new Map(),
|
||||||
|
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
|
||||||
|
url: URL,
|
||||||
|
vomText: rawVomText,
|
||||||
|
};
|
||||||
|
|
||||||
|
const draft: DraftTraceStep = {
|
||||||
|
op: "fill",
|
||||||
|
target: { unmatched: true },
|
||||||
|
value,
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 20, y: 40, w: 200, h: 30 },
|
||||||
|
scrollX: 0,
|
||||||
|
scrollY: 0,
|
||||||
|
position: "static",
|
||||||
|
tag: "input",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
applyTargetMatching(obs, draft, 1);
|
||||||
|
draft.postStateId = stateId;
|
||||||
|
rememberStepOnPage(obs, draft, 1);
|
||||||
|
|
||||||
|
return (
|
||||||
|
buildTraceV3({
|
||||||
|
obs,
|
||||||
|
steps: [draft],
|
||||||
|
startedAt: "2026-08-12T00:00:00.000Z",
|
||||||
|
stoppedBy: "user_finish",
|
||||||
|
bskVersion: "test",
|
||||||
|
}).states[0]?.body ?? ""
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("record observation annotations", () => {
|
||||||
|
beforeEach(() => {
|
||||||
|
resetStateIdCounterForTests();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("omits a fill literal from finalized state bodies when values are redacted", () => {
|
||||||
|
const secret = "hunter2-private";
|
||||||
|
const body = finalizedFillBody(secret, true);
|
||||||
|
|
||||||
|
expect(body).toContain("step 1: fill");
|
||||||
|
expect(body).not.toContain(secret);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps fill details in ordinary recording annotations", () => {
|
||||||
|
const value = "ordinary text";
|
||||||
|
const body = finalizedFillBody(value, false);
|
||||||
|
|
||||||
|
expect(body).toContain(`step 1: fill: ${JSON.stringify(value)}`);
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
describe("applyTargetMatching without a settled observation", () => {
|
||||||
|
it("keeps the capture role/name instead of a bare unmatched target", () => {
|
||||||
|
const obs = createObservationState();
|
||||||
|
const draft: DraftTraceStep = {
|
||||||
|
op: "hover",
|
||||||
|
target: { unmatched: true },
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "新建" },
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||||
|
scrollX: 0,
|
||||||
|
scrollY: 0,
|
||||||
|
position: "static",
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
|
applyTargetMatching(obs, draft, 1);
|
||||||
|
|
||||||
|
expect(draft.preStateId).toBeUndefined();
|
||||||
|
expect(draft.target).toEqual({
|
||||||
|
role: "button",
|
||||||
|
name: "新建",
|
||||||
|
unmatched: true,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -12,12 +12,42 @@ describe("recording-step-buffer", () => {
|
|||||||
expect(buffer.steps).toEqual([
|
expect(buffer.steps).toEqual([
|
||||||
{
|
{
|
||||||
op: "click",
|
op: "click",
|
||||||
target: { tag: "button", role: "button", name: "发布" },
|
target: { unmatched: true },
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
expect(buffer.pendingNavigation).toBe(true);
|
expect(buffer.pendingNavigation).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("keeps the hovered element description as a capture fallback", () => {
|
||||||
|
const buffer = { steps: [], pendingNavigation: false };
|
||||||
|
appendRecordedPayload(buffer, {
|
||||||
|
op: "hover",
|
||||||
|
target: { tag: "button", role: "button", name: "新建" },
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||||
|
scrollX: 0,
|
||||||
|
scrollY: 0,
|
||||||
|
position: "static",
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
expect(buffer.steps).toEqual([
|
||||||
|
{
|
||||||
|
op: "hover",
|
||||||
|
target: { unmatched: true },
|
||||||
|
captureTarget: { tag: "button", role: "button", name: "新建" },
|
||||||
|
geometry: {
|
||||||
|
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||||
|
scrollX: 0,
|
||||||
|
scrollY: 0,
|
||||||
|
position: "static",
|
||||||
|
tag: "button",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
});
|
||||||
|
|
||||||
it("annotates navigated_to on action-caused navigation instead of wait_for_navigation", () => {
|
it("annotates navigated_to on action-caused navigation instead of wait_for_navigation", () => {
|
||||||
const buffer = {
|
const buffer = {
|
||||||
steps: [
|
steps: [
|
||||||
@@ -71,10 +101,41 @@ describe("recording-step-buffer", () => {
|
|||||||
currentUrl: "https://example.com/a",
|
currentUrl: "https://example.com/a",
|
||||||
pendingNavigation: false,
|
pendingNavigation: false,
|
||||||
};
|
};
|
||||||
observeRecordedNavigation(buffer, "https://example.com/b", false);
|
const result = observeRecordedNavigation(buffer, "https://example.com/b", false);
|
||||||
|
expect(result).toEqual({ kind: "appended", index: 0 });
|
||||||
expect(buffer.steps[0]).toMatchObject({
|
expect(buffer.steps[0]).toMatchObject({
|
||||||
op: "navigate",
|
op: "navigate",
|
||||||
url: "https://example.com/b",
|
url: "https://example.com/b",
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("asks the recorder to coalesce redirect hops instead of emitting each one", () => {
|
||||||
|
const buffer = {
|
||||||
|
steps: [],
|
||||||
|
currentUrl: "https://passport.example/login",
|
||||||
|
pendingNavigation: false,
|
||||||
|
};
|
||||||
|
const hop1 = observeRecordedNavigation(
|
||||||
|
buffer,
|
||||||
|
"https://passport.example/callback",
|
||||||
|
false,
|
||||||
|
"link",
|
||||||
|
["server_redirect"],
|
||||||
|
);
|
||||||
|
expect(hop1).toEqual({
|
||||||
|
kind: "coalesce_redirect",
|
||||||
|
url: "https://passport.example/callback",
|
||||||
|
});
|
||||||
|
expect(buffer.steps).toEqual([]);
|
||||||
|
|
||||||
|
const hop2 = observeRecordedNavigation(buffer, "https://app.example/dashboard", false, "link", [
|
||||||
|
"client_redirect",
|
||||||
|
]);
|
||||||
|
expect(hop2).toEqual({
|
||||||
|
kind: "coalesce_redirect",
|
||||||
|
url: "https://app.example/dashboard",
|
||||||
|
});
|
||||||
|
expect(buffer.steps).toEqual([]);
|
||||||
|
expect(buffer.currentUrl).toBe("https://app.example/dashboard");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,6 +1,11 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
import type { DraftTraceStep } from "@/transport/types";
|
import { resetStateIdCounterForTests } from "@/lib/record-constants";
|
||||||
import { reduceTraceSteps, resolveTraceStartUrl, shouldRecordPress } from "../trace-reducer";
|
import {
|
||||||
|
reduceTraceSteps,
|
||||||
|
registerObservation,
|
||||||
|
resolveTraceStartUrl,
|
||||||
|
shouldRecordPress,
|
||||||
|
} from "../trace-reducer";
|
||||||
|
|
||||||
describe("shouldRecordPress", () => {
|
describe("shouldRecordPress", () => {
|
||||||
it("keeps Enter and Escape", () => {
|
it("keeps Enter and Escape", () => {
|
||||||
@@ -17,86 +22,85 @@ describe("shouldRecordPress", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("reduceTraceSteps", () => {
|
describe("reduceTraceSteps v3", () => {
|
||||||
it("builds steps with pages dictionary and page id references", () => {
|
it("builds steps with state/result references and deduped states", () => {
|
||||||
const drafts: DraftTraceStep[] = [
|
resetStateIdCounterForTests();
|
||||||
{
|
const registry = new Map();
|
||||||
op: "navigate",
|
const body = "@vom 1\nL1 page";
|
||||||
url: "https://example.com/search?q=hello&utm_source=x",
|
const s1 = registerObservation(registry, {
|
||||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
url: "https://example.com/search",
|
||||||
},
|
rawVomText: body,
|
||||||
{
|
});
|
||||||
op: "fill",
|
const s2 = registerObservation(registry, {
|
||||||
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
url: "https://example.com/results/42",
|
||||||
value: "browser skill",
|
rawVomText: "@vom 1\nL1 results",
|
||||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
op: "press",
|
|
||||||
key: "Enter",
|
|
||||||
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
|
||||||
navigated_to: "https://example.com/results/42",
|
|
||||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
op: "click",
|
|
||||||
target: { tag: "button", role: "button", name: "发布" },
|
|
||||||
navigated_to: "https://example.com/p/99",
|
|
||||||
page_url: "https://example.com/results/42",
|
|
||||||
},
|
|
||||||
{
|
|
||||||
op: "press",
|
|
||||||
key: "a",
|
|
||||||
page_url: "https://example.com/p/99",
|
|
||||||
},
|
|
||||||
];
|
|
||||||
|
|
||||||
const { pages, steps } = reduceTraceSteps(
|
|
||||||
drafts,
|
|
||||||
"https://example.com/search?q=hello&utm_source=x",
|
|
||||||
);
|
|
||||||
expect(JSON.stringify(steps)).not.toContain("parameters");
|
|
||||||
expect(JSON.stringify(steps)).not.toContain("intent");
|
|
||||||
expect(JSON.stringify(steps)).not.toContain("summary");
|
|
||||||
expect(steps.map((s) => s.op)).toEqual(["navigate", "fill", "press", "click"]);
|
|
||||||
expect(pages.map((p) => p.url)).toEqual([
|
|
||||||
"https://example.com/search?q=hello&utm_source=x",
|
|
||||||
"https://example.com/results/42",
|
|
||||||
"https://example.com/p/99",
|
|
||||||
]);
|
|
||||||
|
|
||||||
expect(steps[0]).toMatchObject({
|
|
||||||
id: 1,
|
|
||||||
op: "navigate",
|
|
||||||
page: "p1",
|
|
||||||
to: "https://example.com/search?q=hello&utm_source=x",
|
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const { states, steps } = reduceTraceSteps(
|
||||||
|
[
|
||||||
|
{
|
||||||
|
op: "navigate",
|
||||||
|
url: "https://example.com/search",
|
||||||
|
preStateId: s1,
|
||||||
|
postStateId: s1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
op: "fill",
|
||||||
|
target: { ref: "e1", role: "textbox", name: "搜索" },
|
||||||
|
value: "browser skill",
|
||||||
|
commit: "enter",
|
||||||
|
preStateId: s1,
|
||||||
|
postStateId: s1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
op: "press",
|
||||||
|
key: "Enter",
|
||||||
|
target: { ref: "e1", role: "textbox", name: "搜索" },
|
||||||
|
preStateId: s1,
|
||||||
|
postStateId: s2,
|
||||||
|
navigated_to: "https://example.com/results/42",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
op: "click",
|
||||||
|
target: { ref: "e2", role: "button", name: "发布" },
|
||||||
|
preStateId: s2,
|
||||||
|
postStateId: s2,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
registry,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(states).toHaveLength(2);
|
||||||
|
expect(steps.map((s) => s.op)).toEqual(["navigate", "fill", "press", "click"]);
|
||||||
expect(steps[1]).toMatchObject({
|
expect(steps[1]).toMatchObject({
|
||||||
op: "fill",
|
op: "fill",
|
||||||
page: "p1",
|
state: s1,
|
||||||
value: "browser skill",
|
result: { state: s1 },
|
||||||
|
commit: "enter",
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(steps[2]).toMatchObject({
|
expect(steps[2]).toMatchObject({
|
||||||
op: "press",
|
op: "press",
|
||||||
key: "Enter",
|
state: s1,
|
||||||
page: "p1",
|
result: { state: s2 },
|
||||||
effect: { navigated_to: "p2" },
|
|
||||||
});
|
|
||||||
|
|
||||||
expect(steps[3]).toMatchObject({
|
|
||||||
op: "click",
|
|
||||||
page: "p2",
|
|
||||||
effect: { navigated_to: "p3" },
|
|
||||||
});
|
});
|
||||||
|
expect(JSON.stringify(steps)).not.toContain("page");
|
||||||
|
expect(JSON.stringify(steps)).not.toContain("effect");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("collapses consecutive navigations", () => {
|
it("collapses consecutive navigations", () => {
|
||||||
const { steps } = reduceTraceSteps([
|
resetStateIdCounterForTests();
|
||||||
{ op: "navigate", url: "https://a.example/redirect1" },
|
const registry = new Map();
|
||||||
{ op: "navigate", url: "https://a.example/final" },
|
const s1 = registerObservation(registry, {
|
||||||
]);
|
url: "https://a.example/redirect1",
|
||||||
|
rawVomText: "a",
|
||||||
|
});
|
||||||
|
const { steps } = reduceTraceSteps(
|
||||||
|
[
|
||||||
|
{ op: "navigate", url: "https://a.example/redirect1", preStateId: s1, postStateId: s1 },
|
||||||
|
{ op: "navigate", url: "https://a.example/final", preStateId: s1, postStateId: s1 },
|
||||||
|
],
|
||||||
|
registry,
|
||||||
|
);
|
||||||
expect(steps).toHaveLength(1);
|
expect(steps).toHaveLength(1);
|
||||||
expect(steps[0]).toMatchObject({
|
expect(steps[0]).toMatchObject({
|
||||||
op: "navigate",
|
op: "navigate",
|
||||||
@@ -104,52 +108,124 @@ describe("reduceTraceSteps", () => {
|
|||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("maps select navigated_to onto effect.navigated_to (page id)", () => {
|
it("keeps the origin state and cause of the hop that started a redirect chain", () => {
|
||||||
const { pages, steps } = reduceTraceSteps(
|
resetStateIdCounterForTests();
|
||||||
|
const registry = new Map();
|
||||||
|
const start = registerObservation(registry, {
|
||||||
|
url: "https://example.com/",
|
||||||
|
rawVomText: "start",
|
||||||
|
});
|
||||||
|
const hop = registerObservation(registry, {
|
||||||
|
url: "https://iwiki.woa.com/",
|
||||||
|
rawVomText: "hop",
|
||||||
|
});
|
||||||
|
const landing = registerObservation(registry, {
|
||||||
|
url: "https://iwiki.woa.com/dashboard",
|
||||||
|
rawVomText: "landing",
|
||||||
|
});
|
||||||
|
|
||||||
|
const { steps, stepIdByDraftId } = reduceTraceSteps(
|
||||||
[
|
[
|
||||||
{
|
{
|
||||||
op: "select",
|
op: "navigate",
|
||||||
target: { tag: "select", role: "combobox", name: "分类" },
|
url: "https://iwiki.woa.com/",
|
||||||
values: ["tech"],
|
preStateId: start,
|
||||||
labels: ["技术"],
|
postStateId: hop,
|
||||||
navigated_to: "https://example.com/list?cat=tech",
|
transitionType: "typed",
|
||||||
page_url: "https://example.com/list",
|
},
|
||||||
|
{
|
||||||
|
op: "navigate",
|
||||||
|
url: "https://iwiki.woa.com/dashboard",
|
||||||
|
preStateId: hop,
|
||||||
|
postStateId: landing,
|
||||||
|
transitionType: "link",
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
"https://example.com/list",
|
registry,
|
||||||
);
|
);
|
||||||
expect(pages.map((p) => p.url)).toEqual([
|
|
||||||
"https://example.com/list",
|
expect(steps).toHaveLength(1);
|
||||||
"https://example.com/list?cat=tech",
|
|
||||||
]);
|
|
||||||
expect(steps[0]).toMatchObject({
|
expect(steps[0]).toMatchObject({
|
||||||
op: "select",
|
op: "navigate",
|
||||||
page: "p1",
|
id: 1,
|
||||||
selection: [{ value: "tech", label: "技术" }],
|
state: start,
|
||||||
effect: { navigated_to: "p2" },
|
result: { state: landing },
|
||||||
|
to: "https://iwiki.woa.com/dashboard",
|
||||||
|
cause: "user_typed",
|
||||||
});
|
});
|
||||||
|
// Both drafts fold into step 1, so anything tagged with either draft id
|
||||||
|
// must resolve to it.
|
||||||
|
expect(stepIdByDraftId.get(1)).toBe(1);
|
||||||
|
expect(stepIdByDraftId.get(2)).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps draft ids to published step ids across dropped drafts", () => {
|
||||||
|
resetStateIdCounterForTests();
|
||||||
|
const registry = new Map();
|
||||||
|
const s1 = registerObservation(registry, { url: "https://a.example/", rawVomText: "a" });
|
||||||
|
|
||||||
|
const { steps, stepIdByDraftId } = reduceTraceSteps(
|
||||||
|
[
|
||||||
|
// Dropped: bare typing press.
|
||||||
|
{ op: "press", key: "a", preStateId: s1, postStateId: s1 },
|
||||||
|
{ op: "click", target: { ref: "e2" }, preStateId: s1, postStateId: s1 },
|
||||||
|
],
|
||||||
|
registry,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(steps.map((step) => step.op)).toEqual(["click"]);
|
||||||
|
expect(stepIdByDraftId.get(1)).toBeUndefined();
|
||||||
|
expect(stepIdByDraftId.get(2)).toBe(1);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("keeps committed empty fills", () => {
|
||||||
|
resetStateIdCounterForTests();
|
||||||
|
const registry = new Map();
|
||||||
|
const s1 = registerObservation(registry, { url: "https://a.example/", rawVomText: "a" });
|
||||||
|
|
||||||
|
const { steps } = reduceTraceSteps(
|
||||||
|
[
|
||||||
|
{ op: "fill", target: { ref: "e1", role: "textbox", name: "Search" }, value: "", preStateId: s1, postStateId: s1 },
|
||||||
|
{ op: "click", target: { ref: "e2", role: "button", name: "Apply" }, preStateId: s1, postStateId: s1 },
|
||||||
|
],
|
||||||
|
registry,
|
||||||
|
);
|
||||||
|
|
||||||
|
expect(steps.map((step) => step.op)).toEqual(["fill", "click"]);
|
||||||
|
expect(steps[0]).toMatchObject({ op: "fill", value: "" });
|
||||||
});
|
});
|
||||||
|
|
||||||
it("keeps hover steps before menu clicks", () => {
|
it("keeps hover steps before menu clicks", () => {
|
||||||
|
resetStateIdCounterForTests();
|
||||||
|
const registry = new Map();
|
||||||
|
const state = registerObservation(registry, {
|
||||||
|
url: "https://example.com/app",
|
||||||
|
rawVomText: '@e1 button "Account"\n@e2 link "Profile"',
|
||||||
|
});
|
||||||
const { steps } = reduceTraceSteps(
|
const { steps } = reduceTraceSteps(
|
||||||
[
|
[
|
||||||
{
|
{
|
||||||
op: "hover",
|
op: "hover",
|
||||||
target: { tag: "span", role: "button", name: "Account" },
|
target: { ref: "e1", role: "button", name: "Account" },
|
||||||
page_url: "https://example.com/app",
|
page_url: "https://example.com/app",
|
||||||
|
preStateId: state,
|
||||||
|
postStateId: state,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
op: "click",
|
op: "click",
|
||||||
target: { tag: "a", role: "link", name: "Profile" },
|
target: { ref: "e2", role: "link", name: "Profile" },
|
||||||
page_url: "https://example.com/app",
|
page_url: "https://example.com/app",
|
||||||
|
preStateId: state,
|
||||||
|
postStateId: state,
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
"https://example.com/app",
|
registry,
|
||||||
);
|
);
|
||||||
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||||
expect(steps[0]).toMatchObject({
|
expect(steps[0]).toMatchObject({
|
||||||
op: "hover",
|
op: "hover",
|
||||||
page: "p1",
|
state,
|
||||||
|
result: { state },
|
||||||
target: { name: "Account" },
|
target: { name: "Account" },
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
@@ -157,7 +233,7 @@ describe("reduceTraceSteps", () => {
|
|||||||
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
||||||
expect(
|
expect(
|
||||||
resolveTraceStartUrl(
|
resolveTraceStartUrl(
|
||||||
[{ op: "navigate", url: "https://example.com/other" }],
|
[{ op: "navigate", url: "https://example.com/other", preStateId: "s1", postStateId: "s1" }],
|
||||||
"https://example.com/start",
|
"https://example.com/start",
|
||||||
),
|
),
|
||||||
).toBe("https://example.com/start");
|
).toBe("https://example.com/start");
|
||||||
|
|||||||
@@ -313,6 +313,15 @@ export type HandshakeVerdict =
|
|||||||
| { kind: "version_skew" }
|
| { kind: "version_skew" }
|
||||||
| { kind: "rejected"; reason: string };
|
| { kind: "rejected"; reason: string };
|
||||||
|
|
||||||
|
const UPDATE_BOTH_VERSION_MESSAGE =
|
||||||
|
"The installed bsk CLI and BrowserSkill extension are incompatible. Update both, reload the extension, then run `bsk daemon restart`.";
|
||||||
|
const UPDATE_BSK_VERSION_MESSAGE =
|
||||||
|
"The installed bsk CLI is out of date for this BrowserSkill extension. Update bsk, then run `bsk daemon restart`.";
|
||||||
|
const UPDATE_EXTENSION_VERSION_MESSAGE =
|
||||||
|
"This BrowserSkill extension is out of date for the installed bsk CLI. Update and reload the extension, then try again.";
|
||||||
|
const INVALID_VERSION_MESSAGE =
|
||||||
|
"BrowserSkill could not verify version compatibility. Update bsk and the BrowserSkill extension, reload the extension, then run `bsk daemon restart`.";
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Symmetric extension-side **protocol** compat check.
|
* Symmetric extension-side **protocol** compat check.
|
||||||
*
|
*
|
||||||
@@ -339,19 +348,19 @@ export function computeConnectedState(
|
|||||||
if (daemonMajor === null || ourMajor === null || daemonMajor !== ourMajor) {
|
if (daemonMajor === null || ourMajor === null || daemonMajor !== ourMajor) {
|
||||||
return {
|
return {
|
||||||
kind: "rejected",
|
kind: "rejected",
|
||||||
reason: `protocol-major mismatch (daemon protocol v${handshake.protocol_version}, extension protocol v${PROTOCOL_VERSION})`,
|
reason: UPDATE_BOTH_VERSION_MESSAGE,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
if (compareProtocol(handshake.protocol_version, localMinCompatibleProtocol) === null) {
|
if (compareProtocol(handshake.protocol_version, localMinCompatibleProtocol) === null) {
|
||||||
return {
|
return {
|
||||||
kind: "rejected",
|
kind: "rejected",
|
||||||
reason: `daemon protocol v${handshake.protocol_version} is unparseable`,
|
reason: INVALID_VERSION_MESSAGE,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
if (compareProtocol(handshake.protocol_version, localMinCompatibleProtocol)! < 0) {
|
if (compareProtocol(handshake.protocol_version, localMinCompatibleProtocol)! < 0) {
|
||||||
return {
|
return {
|
||||||
kind: "rejected",
|
kind: "rejected",
|
||||||
reason: `daemon protocol v${handshake.protocol_version} below extension min_compatible_protocol ${localMinCompatibleProtocol}`,
|
reason: UPDATE_BSK_VERSION_MESSAGE,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
const peerFloor = handshake.min_compatible_protocol;
|
const peerFloor = handshake.min_compatible_protocol;
|
||||||
@@ -359,13 +368,13 @@ export function computeConnectedState(
|
|||||||
if (compareProtocol(PROTOCOL_VERSION, peerFloor) === null) {
|
if (compareProtocol(PROTOCOL_VERSION, peerFloor) === null) {
|
||||||
return {
|
return {
|
||||||
kind: "rejected",
|
kind: "rejected",
|
||||||
reason: `daemon min_compatible_protocol ${peerFloor} is unparseable`,
|
reason: INVALID_VERSION_MESSAGE,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
if (compareProtocol(PROTOCOL_VERSION, peerFloor)! < 0) {
|
if (compareProtocol(PROTOCOL_VERSION, peerFloor)! < 0) {
|
||||||
return {
|
return {
|
||||||
kind: "rejected",
|
kind: "rejected",
|
||||||
reason: `extension protocol v${PROTOCOL_VERSION} below daemon min_compatible_protocol ${peerFloor}`,
|
reason: UPDATE_EXTENSION_VERSION_MESSAGE,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,7 +6,8 @@
|
|||||||
* `{ "tag": "div" }` / “点击div” is useless and must not be recorded.
|
* `{ "tag": "div" }` / “点击div” is useless and must not be recorded.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
export interface TargetDescriptor {
|
/** Content-script capture descriptor before VOM geometric matching. */
|
||||||
|
export interface CaptureTargetDescriptor {
|
||||||
role?: string;
|
role?: string;
|
||||||
name?: string;
|
name?: string;
|
||||||
tag: string;
|
tag: string;
|
||||||
@@ -247,7 +248,7 @@ export function resolveHoverElement(target: Element): Element | null {
|
|||||||
* (visible name), or at least a form `name_attr` for checkbox/radio.
|
* (visible name), or at least a form `name_attr` for checkbox/radio.
|
||||||
* Recording “点击div” with no name fails this bar.
|
* Recording “点击div” with no name fails this bar.
|
||||||
*/
|
*/
|
||||||
export function isMeaningfulClickTarget(target: TargetDescriptor): boolean {
|
export function isMeaningfulClickTarget(target: CaptureTargetDescriptor): boolean {
|
||||||
const name = target.name?.trim();
|
const name = target.name?.trim();
|
||||||
if (name && isActionableLabel(name)) return true;
|
if (name && isActionableLabel(name)) return true;
|
||||||
if (
|
if (
|
||||||
@@ -323,7 +324,7 @@ function nearbyLabelText(el: Element): string | undefined {
|
|||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function describeTarget(el: Element): TargetDescriptor {
|
export function describeTarget(el: Element): CaptureTargetDescriptor {
|
||||||
const tag = el.tagName.toLowerCase();
|
const tag = el.tagName.toLowerCase();
|
||||||
const role = inferRole(el);
|
const role = inferRole(el);
|
||||||
const name = accessibleName(el);
|
const name = accessibleName(el);
|
||||||
@@ -350,7 +351,7 @@ export function describeTarget(el: Element): TargetDescriptor {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export function describeEventTarget(target: EventTarget | null): TargetDescriptor | null {
|
export function describeEventTarget(target: EventTarget | null): CaptureTargetDescriptor | null {
|
||||||
if (!(target instanceof Element)) return null;
|
if (!(target instanceof Element)) return null;
|
||||||
const clickable = resolveClickableElement(target);
|
const clickable = resolveClickableElement(target);
|
||||||
if (!clickable) return null;
|
if (!clickable) return null;
|
||||||
|
|||||||
@@ -0,0 +1,62 @@
|
|||||||
|
import type { Step } from "@/transport/types";
|
||||||
|
import { OBSERVATION_FILE_VERSION } from "./record-constants";
|
||||||
|
|
||||||
|
export interface ObservationAnnotation {
|
||||||
|
stepId: number;
|
||||||
|
op: Step["op"];
|
||||||
|
line: number;
|
||||||
|
stateId: string;
|
||||||
|
detail?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatAnnotation({ stepId, op, detail }: ObservationAnnotation): string {
|
||||||
|
const suffix = detail ? `: ${detail}` : "";
|
||||||
|
return ` ⟵ step ${stepId}: ${op}${suffix}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Serialize a page observation file (front matter + VOM body + step annotations). */
|
||||||
|
export function formatObservationFile(input: {
|
||||||
|
stateId: string;
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
stepsHere: number[];
|
||||||
|
body: string;
|
||||||
|
annotations?: ObservationAnnotation[];
|
||||||
|
}): string {
|
||||||
|
const lines: string[] = [
|
||||||
|
`# bsk-observation ${OBSERVATION_FILE_VERSION}`,
|
||||||
|
`state: ${input.stateId}`,
|
||||||
|
`url: ${input.url}`,
|
||||||
|
];
|
||||||
|
if (input.title) lines.push(`title: ${input.title}`);
|
||||||
|
if (input.stepsHere.length > 0) {
|
||||||
|
lines.push(`steps_here: [${input.stepsHere.join(", ")}]`);
|
||||||
|
}
|
||||||
|
lines.push("---");
|
||||||
|
|
||||||
|
const bodyLines = input.body.split("\n");
|
||||||
|
const annotationMap = new Map<number, ObservationAnnotation[]>();
|
||||||
|
for (const ann of input.annotations ?? []) {
|
||||||
|
const bucket = annotationMap.get(ann.line) ?? [];
|
||||||
|
bucket.push(ann);
|
||||||
|
annotationMap.set(ann.line, bucket);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (let i = 0; i < bodyLines.length; i += 1) {
|
||||||
|
let line = bodyLines[i] ?? "";
|
||||||
|
const anns = annotationMap.get(i);
|
||||||
|
if (anns) {
|
||||||
|
for (const ann of anns) {
|
||||||
|
line += formatAnnotation(ann);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
lines.push(line);
|
||||||
|
}
|
||||||
|
|
||||||
|
return `${lines.join("\n")}\n`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Hash input: VOM body **before** annotations are inserted. */
|
||||||
|
export function observationBodyForHash(vomText: string): string {
|
||||||
|
return vomText;
|
||||||
|
}
|
||||||
@@ -0,0 +1,102 @@
|
|||||||
|
import type { RenderedRef } from "@browser-skill/vom";
|
||||||
|
import type { CapturedNode } from "@/tools/vom/capture";
|
||||||
|
import type { CaptureGeometry, TargetDescriptor } from "@/transport/types";
|
||||||
|
import type { CaptureTargetDescriptor } from "./describe-target";
|
||||||
|
import { GEOM_MATCH_TOLERANCE_PX } from "./record-constants";
|
||||||
|
|
||||||
|
export interface MatchTargetInput {
|
||||||
|
geometry: CaptureGeometry;
|
||||||
|
captured: CapturedNode[];
|
||||||
|
refs: RenderedRef[];
|
||||||
|
fallback?: CaptureTargetDescriptor;
|
||||||
|
}
|
||||||
|
|
||||||
|
function withinTolerance(a: number, b: number): boolean {
|
||||||
|
return Math.abs(a - b) <= GEOM_MATCH_TOLERANCE_PX;
|
||||||
|
}
|
||||||
|
|
||||||
|
function rectsMatch(
|
||||||
|
a: { x: number; y: number; w: number; h: number },
|
||||||
|
b: { x: number; y: number; w: number; h: number },
|
||||||
|
): boolean {
|
||||||
|
return (
|
||||||
|
withinTolerance(a.x, b.x) &&
|
||||||
|
withinTolerance(a.y, b.y) &&
|
||||||
|
withinTolerance(a.w, b.w) &&
|
||||||
|
withinTolerance(a.h, b.h)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function nodeViewportRect(
|
||||||
|
node: CapturedNode,
|
||||||
|
): { x: number; y: number; w: number; h: number } | null {
|
||||||
|
return node.localRect ?? node.rect;
|
||||||
|
}
|
||||||
|
|
||||||
|
function matchingRects(
|
||||||
|
geometry: CaptureGeometry,
|
||||||
|
node: CapturedNode,
|
||||||
|
): {
|
||||||
|
target: { x: number; y: number; w: number; h: number };
|
||||||
|
node: { x: number; y: number; w: number; h: number } | null;
|
||||||
|
} {
|
||||||
|
const viewportPosition = geometry.position === "fixed" || geometry.position === "sticky";
|
||||||
|
if (viewportPosition || !node.documentRect) {
|
||||||
|
return { target: geometry.rect, node: nodeViewportRect(node) };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
target: {
|
||||||
|
x: geometry.rect.x + geometry.scrollX,
|
||||||
|
y: geometry.rect.y + geometry.scrollY,
|
||||||
|
w: geometry.rect.w,
|
||||||
|
h: geometry.rect.h,
|
||||||
|
},
|
||||||
|
node: node.documentRect,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function tagMatches(geometryTag: string, nodeTag: string): boolean {
|
||||||
|
return geometryTag.toLowerCase() === nodeTag.toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Best description available when the element cannot be located in the VOM. */
|
||||||
|
export function fallbackDescriptor(fallback?: CaptureTargetDescriptor): TargetDescriptor {
|
||||||
|
if (!fallback) {
|
||||||
|
return { unmatched: true };
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
...(fallback.role ? { role: fallback.role } : {}),
|
||||||
|
...(fallback.name ? { name: fallback.name } : {}),
|
||||||
|
unmatched: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Locate the interacted element in the last settled observation by geometry. */
|
||||||
|
export function matchTarget(input: MatchTargetInput): TargetDescriptor {
|
||||||
|
const ownerFrame = input.geometry.ownerFrameBackendNodeId ?? null;
|
||||||
|
|
||||||
|
const candidates = input.captured.filter((node) => {
|
||||||
|
if (ownerFrame !== (node.ownerFrameBackendNodeId ?? null)) return false;
|
||||||
|
if (!tagMatches(input.geometry.tag, node.tag)) return false;
|
||||||
|
const rects = matchingRects(input.geometry, node);
|
||||||
|
if (!rects.node) return false;
|
||||||
|
return rectsMatch(rects.target, rects.node);
|
||||||
|
});
|
||||||
|
|
||||||
|
if (candidates.length !== 1) {
|
||||||
|
return fallbackDescriptor(input.fallback);
|
||||||
|
}
|
||||||
|
|
||||||
|
const backendNodeId = candidates[0]!.backendNodeId;
|
||||||
|
const refEntry = input.refs.find((r) => r.backendNodeId === backendNodeId);
|
||||||
|
if (!refEntry) {
|
||||||
|
return fallbackDescriptor(input.fallback);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
ref: refEntry.ref,
|
||||||
|
...(refEntry.role ? { role: refEntry.role } : {}),
|
||||||
|
...(refEntry.name ? { name: refEntry.name } : {}),
|
||||||
|
...(refEntry.ctx ? { ctx: refEntry.ctx } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
// Deciding *when* a page is worth observing.
|
||||||
|
//
|
||||||
|
// A fixed delay cannot serve both a modal that toggles in one frame and a
|
||||||
|
// route change that renders for a second. Instead of guessing, ask the page:
|
||||||
|
// a document that has stopped mutating and is no longer loading is done
|
||||||
|
// reacting to whatever the user just did.
|
||||||
|
|
||||||
|
import type { CdpRunner } from "@/tools/shared";
|
||||||
|
import { SETTLE_MAX_MS, SETTLE_MIN_MS, SETTLE_POLL_MS, SETTLE_QUIET_MS } from "./record-constants";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Installed once per document. Records only the timestamp of the last DOM
|
||||||
|
* change so each poll stays O(1) no matter how large the page is.
|
||||||
|
*/
|
||||||
|
const QUIET_PROBE = `(() => {
|
||||||
|
const scope = window;
|
||||||
|
let probe = scope.__bskRecordQuiet;
|
||||||
|
if (!probe) {
|
||||||
|
probe = { changedAt: Date.now() };
|
||||||
|
const observer = new MutationObserver(() => {
|
||||||
|
probe.changedAt = Date.now();
|
||||||
|
});
|
||||||
|
observer.observe(document.documentElement, {
|
||||||
|
subtree: true,
|
||||||
|
childList: true,
|
||||||
|
attributes: true,
|
||||||
|
characterData: true,
|
||||||
|
});
|
||||||
|
scope.__bskRecordQuiet = probe;
|
||||||
|
}
|
||||||
|
return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState };
|
||||||
|
})()`;
|
||||||
|
|
||||||
|
export type SettleOutcome =
|
||||||
|
/** The page stopped changing on its own. */
|
||||||
|
| "quiet"
|
||||||
|
/** Still changing when the budget ran out; observe it as it is. */
|
||||||
|
| "timeout"
|
||||||
|
/** A newer action took over; this observation is no longer wanted. */
|
||||||
|
| "cancelled";
|
||||||
|
|
||||||
|
interface QuietProbe {
|
||||||
|
idleMs: number;
|
||||||
|
readyState: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
function sleep(ms: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readQuietProbe(cdp: CdpRunner, tabId: number): Promise<QuietProbe | null> {
|
||||||
|
try {
|
||||||
|
const reply = await cdp.send<{ result?: { value?: unknown } }>(tabId, "Runtime.evaluate", {
|
||||||
|
expression: QUIET_PROBE,
|
||||||
|
returnByValue: true,
|
||||||
|
});
|
||||||
|
const value = reply.result?.value;
|
||||||
|
if (!value || typeof value !== "object") return null;
|
||||||
|
const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown };
|
||||||
|
if (typeof idleMs !== "number" || typeof readyState !== "string") return null;
|
||||||
|
return { idleMs, readyState };
|
||||||
|
} catch {
|
||||||
|
// An unreadable page is a page mid-swap; that is a reason to keep waiting,
|
||||||
|
// not a reason to give up.
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Wait until the page has finished reacting, or until the budget runs out. */
|
||||||
|
export async function waitForPageSettled(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
options: { cancelled?: () => boolean } = {},
|
||||||
|
): Promise<SettleOutcome> {
|
||||||
|
const startedAt = Date.now();
|
||||||
|
const floor = startedAt + SETTLE_MIN_MS;
|
||||||
|
const deadline = startedAt + SETTLE_MAX_MS;
|
||||||
|
|
||||||
|
for (;;) {
|
||||||
|
if (options.cancelled?.()) return "cancelled";
|
||||||
|
await sleep(SETTLE_POLL_MS);
|
||||||
|
if (options.cancelled?.()) return "cancelled";
|
||||||
|
|
||||||
|
const probe = await readQuietProbe(cdp, tabId);
|
||||||
|
const now = Date.now();
|
||||||
|
if (now < floor) continue;
|
||||||
|
if (now >= deadline) return "timeout";
|
||||||
|
if (!probe) continue;
|
||||||
|
if (probe.readyState === "loading") continue;
|
||||||
|
if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet";
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,7 +3,8 @@
|
|||||||
* service worker and a tab's content script.
|
* service worker and a tab's content script.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import type { TargetDescriptor } from "./describe-target";
|
import type { CaptureTargetDescriptor } from "./describe-target";
|
||||||
|
import type { FrameCaptureFailure, FrameCaptureStatus } from "@/transport/types";
|
||||||
|
|
||||||
export const RECORD_START = "bsk-record-start";
|
export const RECORD_START = "bsk-record-start";
|
||||||
export const RECORD_STEP = "bsk-record-step";
|
export const RECORD_STEP = "bsk-record-step";
|
||||||
@@ -11,6 +12,7 @@ export const RECORD_STOP = "bsk-record-stop";
|
|||||||
export const RECORD_CANCEL = "bsk-record-cancel";
|
export const RECORD_CANCEL = "bsk-record-cancel";
|
||||||
export const RECORD_FINISH = "bsk-record-finish";
|
export const RECORD_FINISH = "bsk-record-finish";
|
||||||
export const RECORD_QUERY = "bsk-record-query";
|
export const RECORD_QUERY = "bsk-record-query";
|
||||||
|
export const RECORD_CAPTURE_STATUS = "bsk-record-capture-status";
|
||||||
|
|
||||||
export interface RecordStartAck {
|
export interface RecordStartAck {
|
||||||
ok: true;
|
ok: true;
|
||||||
@@ -27,8 +29,12 @@ export interface RecordQueryResponse {
|
|||||||
requestId?: string;
|
requestId?: string;
|
||||||
/** Epoch ms when the recording began; see `RecordStartMessage.startedAtMs`. */
|
/** Epoch ms when the recording began; see `RecordStartMessage.startedAtMs`. */
|
||||||
startedAtMs?: number;
|
startedAtMs?: number;
|
||||||
|
captureStatus?: FrameCaptureStatus;
|
||||||
|
captureFailures?: FrameCaptureFailure[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type RecordFrameMode = "top" | "child";
|
||||||
|
|
||||||
export interface RecordStartMessage {
|
export interface RecordStartMessage {
|
||||||
type: typeof RECORD_START;
|
type: typeof RECORD_START;
|
||||||
requestId: string;
|
requestId: string;
|
||||||
@@ -38,11 +44,15 @@ export interface RecordStartMessage {
|
|||||||
* content-script remounts instead of restarting per page.
|
* content-script remounts instead of restarting per page.
|
||||||
*/
|
*/
|
||||||
startedAtMs?: number;
|
startedAtMs?: number;
|
||||||
|
/** When true, the top-frame overlay should mount. Child frames omit this. */
|
||||||
|
showOverlay?: boolean;
|
||||||
|
/** Whether this document runs top-frame or child-frame capture semantics. */
|
||||||
|
frameMode?: RecordFrameMode;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RecordStepPayload {
|
export interface RecordStepPayload {
|
||||||
op: "click" | "hover" | "fill" | "press" | "select" | "navigate";
|
op: "click" | "hover" | "fill" | "press" | "select" | "navigate" | "scroll";
|
||||||
target?: TargetDescriptor;
|
target?: CaptureTargetDescriptor;
|
||||||
value?: string;
|
value?: string;
|
||||||
key?: string;
|
key?: string;
|
||||||
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">;
|
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">;
|
||||||
@@ -50,12 +60,31 @@ export interface RecordStepPayload {
|
|||||||
labels?: string[];
|
labels?: string[];
|
||||||
url?: string;
|
url?: string;
|
||||||
redacted?: boolean;
|
redacted?: boolean;
|
||||||
/** Page URL when the step was captured. */
|
commit?: "enter" | "suggestion" | "blur";
|
||||||
|
/** Top-level tab URL when the step was captured. */
|
||||||
page_url?: string;
|
page_url?: string;
|
||||||
|
/** URL of the frame document where the interaction occurred. */
|
||||||
|
frame_url?: string;
|
||||||
|
/** Ephemeral token stamped on the active document root; not persisted in trace steps. */
|
||||||
|
documentToken?: string;
|
||||||
|
/** Epoch ms when the step was captured in the content script. */
|
||||||
|
capturedAtMs?: number;
|
||||||
|
/** Document geometry at capture time for background geometric matching. */
|
||||||
|
geometry?: {
|
||||||
|
rect: { x: number; y: number; w: number; h: number };
|
||||||
|
scrollX: number;
|
||||||
|
scrollY: number;
|
||||||
|
position: string;
|
||||||
|
tag: string;
|
||||||
|
ownerFrameBackendNodeId?: number | null;
|
||||||
|
};
|
||||||
/** Capture-only hint; never persisted unless converted to navigated_to. */
|
/** Capture-only hint; never persisted unless converted to navigated_to. */
|
||||||
expects_navigation?: boolean;
|
expects_navigation?: boolean;
|
||||||
/** Whether an observed URL change was synchronously caused by the action. */
|
/** Whether an observed URL change was synchronously caused by the action. */
|
||||||
navigation_caused_by_action?: boolean;
|
navigation_caused_by_action?: boolean;
|
||||||
|
/** Raw webNavigation transition metadata for cause mapping. */
|
||||||
|
transitionType?: string;
|
||||||
|
transitionQualifiers?: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RecordStepMessage {
|
export interface RecordStepMessage {
|
||||||
@@ -79,6 +108,12 @@ export interface RecordFinishMessage {
|
|||||||
requestId: string;
|
requestId: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface RecordCaptureStatusMessage {
|
||||||
|
type: typeof RECORD_CAPTURE_STATUS;
|
||||||
|
status: FrameCaptureStatus;
|
||||||
|
failures?: FrameCaptureFailure[];
|
||||||
|
}
|
||||||
|
|
||||||
export function isRecordStartMessage(msg: unknown): msg is RecordStartMessage {
|
export function isRecordStartMessage(msg: unknown): msg is RecordStartMessage {
|
||||||
if (typeof msg !== "object" || msg === null) return false;
|
if (typeof msg !== "object" || msg === null) return false;
|
||||||
const m = msg as Record<string, unknown>;
|
const m = msg as Record<string, unknown>;
|
||||||
@@ -109,6 +144,16 @@ export function isRecordQueryMessage(msg: unknown): msg is RecordQueryMessage {
|
|||||||
return m.type === RECORD_QUERY;
|
return m.type === RECORD_QUERY;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export function isRecordCaptureStatusMessage(msg: unknown): msg is RecordCaptureStatusMessage {
|
||||||
|
if (typeof msg !== "object" || msg === null) return false;
|
||||||
|
const m = msg as Record<string, unknown>;
|
||||||
|
return (
|
||||||
|
m.type === RECORD_CAPTURE_STATUS &&
|
||||||
|
(m.status === "complete" || m.status === "partial") &&
|
||||||
|
(m.failures === undefined || Array.isArray(m.failures))
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage {
|
export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage {
|
||||||
if (typeof msg !== "object" || msg === null) return false;
|
if (typeof msg !== "object" || msg === null) return false;
|
||||||
const m = msg as Record<string, unknown>;
|
const m = msg as Record<string, unknown>;
|
||||||
|
|||||||
@@ -0,0 +1,97 @@
|
|||||||
|
import type { FillCommit, NavigationCause } from "@/transport/types";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Floor before a post-action observation may be taken. A page usually has not
|
||||||
|
* started reacting in the first frames after an action, and capturing then
|
||||||
|
* would record the page as it was, not as the action left it.
|
||||||
|
*/
|
||||||
|
export const SETTLE_MIN_MS = 150;
|
||||||
|
|
||||||
|
/** How long the DOM must stop changing before a page counts as settled. */
|
||||||
|
export const SETTLE_QUIET_MS = 250;
|
||||||
|
|
||||||
|
/** Upper bound on waiting for a page to settle; animations never stop. */
|
||||||
|
export const SETTLE_MAX_MS = 2_000;
|
||||||
|
|
||||||
|
/** How often to ask the page whether it has gone quiet. */
|
||||||
|
export const SETTLE_POLL_MS = 60;
|
||||||
|
|
||||||
|
/** Minimum interval between consecutive observations on the same tab. */
|
||||||
|
export const OBSERVATION_MIN_INTERVAL_MS = 200;
|
||||||
|
|
||||||
|
/** Delay before retrying a capture that lost its execution context. */
|
||||||
|
export const CAPTURE_RETRY_DELAY_MS = 250;
|
||||||
|
|
||||||
|
/** Document-coordinate matching tolerance in CSS pixels. */
|
||||||
|
export const GEOM_MATCH_TOLERANCE_PX = 2;
|
||||||
|
|
||||||
|
/** Default max tokens per page observation when CLI omits `--max-page-tokens`. */
|
||||||
|
export const DEFAULT_MAX_PAGE_TOKENS = 3000;
|
||||||
|
|
||||||
|
/** VOM observation file format version (front matter header). */
|
||||||
|
export const OBSERVATION_FILE_VERSION = 1;
|
||||||
|
|
||||||
|
/** FNV-1a 64-bit offset basis. */
|
||||||
|
const FNV_OFFSET = 0xcbf29ce484222325n;
|
||||||
|
/** FNV-1a 64-bit prime. */
|
||||||
|
const FNV_PRIME = 0x100000001b3n;
|
||||||
|
|
||||||
|
/** Content-hash for state deduplication (sync, non-cryptographic). */
|
||||||
|
export function fnv1a64(text: string): string {
|
||||||
|
let hash = FNV_OFFSET;
|
||||||
|
for (let i = 0; i < text.length; i += 1) {
|
||||||
|
hash ^= BigInt(text.charCodeAt(i));
|
||||||
|
hash = (hash * FNV_PRIME) & 0xffffffffffffffffn;
|
||||||
|
}
|
||||||
|
return hash.toString(16).padStart(16, "0");
|
||||||
|
}
|
||||||
|
|
||||||
|
let nextStateSerial = 0;
|
||||||
|
|
||||||
|
/** Monotonic state id generator (`s1`, `s2`, …). */
|
||||||
|
export function nextStateId(): string {
|
||||||
|
nextStateSerial += 1;
|
||||||
|
return `s${nextStateSerial}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Test seam: reset the state id counter. */
|
||||||
|
export function resetStateIdCounterForTests(): void {
|
||||||
|
nextStateSerial = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
|
||||||
|
|
||||||
|
const TRANSITION_TO_CAUSE: Record<string, NavigationCause> = {
|
||||||
|
typed: "user_typed",
|
||||||
|
generated: "user_typed",
|
||||||
|
keyword: "user_typed",
|
||||||
|
keyword_generated: "user_typed",
|
||||||
|
link: "link",
|
||||||
|
form_submit: "form_submit",
|
||||||
|
reload: "reload",
|
||||||
|
auto_bookmark: "browser",
|
||||||
|
start_page: "browser",
|
||||||
|
};
|
||||||
|
|
||||||
|
export interface NavigationTransitionMeta {
|
||||||
|
transitionType?: string;
|
||||||
|
transitionQualifiers?: string[];
|
||||||
|
navigationActionPending?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Map webNavigation metadata to protocol `NavigationCause`. Returns null for redirects. */
|
||||||
|
export function mapNavigationCause(meta: NavigationTransitionMeta): NavigationCause | null {
|
||||||
|
const qualifiers = meta.transitionQualifiers ?? [];
|
||||||
|
if (qualifiers.includes("forward_back")) return "history";
|
||||||
|
if (qualifiers.some((q) => REDIRECT_QUALIFIERS.has(q))) return null;
|
||||||
|
if (qualifiers.includes("from_address_bar")) return "user_typed";
|
||||||
|
|
||||||
|
const type = meta.transitionType ?? "";
|
||||||
|
const mapped = TRANSITION_TO_CAUSE[type];
|
||||||
|
if (mapped) return mapped;
|
||||||
|
|
||||||
|
if (!type && meta.navigationActionPending === false) return "script";
|
||||||
|
return "browser";
|
||||||
|
}
|
||||||
|
|
||||||
|
export const DEFAULT_FILL_COMMIT: FillCommit = "blur";
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
/** Attribute written on `<html>` while a document is actively recording. */
|
||||||
|
export const RECORD_DOCUMENT_TOKEN_ATTR = "data-bsk-record-document";
|
||||||
|
|
||||||
|
export function createRecordDocumentToken(): string {
|
||||||
|
if (typeof crypto !== "undefined" && typeof crypto.randomUUID === "function") {
|
||||||
|
return crypto.randomUUID();
|
||||||
|
}
|
||||||
|
return `doc-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function stampRecordDocumentToken(token: string): void {
|
||||||
|
document.documentElement.setAttribute(RECORD_DOCUMENT_TOKEN_ATTR, token);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function clearRecordDocumentToken(): void {
|
||||||
|
document.documentElement.removeAttribute(RECORD_DOCUMENT_TOKEN_ATTR);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function readRecordDocumentTokenFromAttrs(
|
||||||
|
attrs: Record<string, string> | undefined,
|
||||||
|
): string | undefined {
|
||||||
|
const token = attrs?.[RECORD_DOCUMENT_TOKEN_ATTR];
|
||||||
|
return token?.trim() ? token : undefined;
|
||||||
|
}
|
||||||
@@ -0,0 +1,250 @@
|
|||||||
|
import {
|
||||||
|
RECORD_CANCEL,
|
||||||
|
RECORD_START,
|
||||||
|
RECORD_STOP,
|
||||||
|
type RecordCancelMessage,
|
||||||
|
type RecordStartAck,
|
||||||
|
type RecordStartMessage,
|
||||||
|
type RecordStopAck,
|
||||||
|
type RecordStopMessage,
|
||||||
|
} from "@/lib/record-bridge";
|
||||||
|
import type { FrameCaptureFailure, FrameCaptureFailureReason } from "@/transport/types";
|
||||||
|
|
||||||
|
export interface WebNavigationFrameInfo {
|
||||||
|
frameId: number;
|
||||||
|
documentId?: string;
|
||||||
|
url?: string;
|
||||||
|
parentFrameId: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ArmedDocument {
|
||||||
|
tabId: number;
|
||||||
|
frameId: number;
|
||||||
|
documentId: string;
|
||||||
|
url?: string;
|
||||||
|
parentFrameId: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RecordFrameSendDeps {
|
||||||
|
sendToDocument(
|
||||||
|
tabId: number,
|
||||||
|
documentId: string,
|
||||||
|
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
|
||||||
|
): Promise<unknown>;
|
||||||
|
getAllFrames(tabId: number): Promise<WebNavigationFrameInfo[]>;
|
||||||
|
}
|
||||||
|
|
||||||
|
const RECORD_START_RETRIES = 3;
|
||||||
|
const RECORD_START_RETRY_DELAY_MS = 500;
|
||||||
|
|
||||||
|
function sleep(ms: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function isFrameUrlRestricted(url: string | undefined): boolean {
|
||||||
|
if (!url) return false;
|
||||||
|
const lower = url.toLowerCase();
|
||||||
|
return (
|
||||||
|
lower.startsWith("chrome://") ||
|
||||||
|
lower.startsWith("chrome-extension://") ||
|
||||||
|
lower.startsWith("edge://") ||
|
||||||
|
lower.startsWith("devtools://") ||
|
||||||
|
lower.startsWith("devtools:") ||
|
||||||
|
lower.startsWith("https://chrome.google.com/webstore")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function isRecordStartAck(response: unknown): response is RecordStartAck {
|
||||||
|
return (
|
||||||
|
typeof response === "object" &&
|
||||||
|
response !== null &&
|
||||||
|
"ok" in response &&
|
||||||
|
(response as RecordStartAck).ok === true
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function isRecordStopAckOk(response: unknown): boolean {
|
||||||
|
return (
|
||||||
|
typeof response === "object" &&
|
||||||
|
response !== null &&
|
||||||
|
"ok" in response &&
|
||||||
|
(response as RecordStopAck).ok === true
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function makeFrameCaptureFailure(
|
||||||
|
reason: FrameCaptureFailureReason,
|
||||||
|
input: { frameId: number; documentId?: string; url?: string; detail?: string },
|
||||||
|
): FrameCaptureFailure {
|
||||||
|
return {
|
||||||
|
reason,
|
||||||
|
frame_id: input.frameId,
|
||||||
|
...(input.documentId ? { document_id: input.documentId } : {}),
|
||||||
|
...(input.url ? { url: input.url } : {}),
|
||||||
|
...(input.detail ? { detail: input.detail } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function listInjectableFrames(
|
||||||
|
tabId: number,
|
||||||
|
deps: RecordFrameSendDeps,
|
||||||
|
): Promise<{ injectable: WebNavigationFrameInfo[]; failures: FrameCaptureFailure[] }> {
|
||||||
|
const failures: FrameCaptureFailure[] = [];
|
||||||
|
let frames: WebNavigationFrameInfo[] = [];
|
||||||
|
try {
|
||||||
|
frames = await deps.getAllFrames(tabId);
|
||||||
|
} catch (err) {
|
||||||
|
failures.push(
|
||||||
|
makeFrameCaptureFailure("arm_failed", {
|
||||||
|
frameId: 0,
|
||||||
|
detail: err instanceof Error ? err.message : "webNavigation.getAllFrames failed",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return { injectable: [], failures };
|
||||||
|
}
|
||||||
|
|
||||||
|
const injectable = frames.filter((frame) => {
|
||||||
|
if (!frame.documentId) {
|
||||||
|
failures.push(
|
||||||
|
makeFrameCaptureFailure("not_injectable", {
|
||||||
|
frameId: frame.frameId,
|
||||||
|
url: frame.url,
|
||||||
|
detail: "missing documentId",
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (isFrameUrlRestricted(frame.url)) {
|
||||||
|
failures.push(
|
||||||
|
makeFrameCaptureFailure("not_injectable", {
|
||||||
|
frameId: frame.frameId,
|
||||||
|
documentId: frame.documentId,
|
||||||
|
url: frame.url,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
return { injectable, failures };
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function sendRecordStartWithAckToDocument(
|
||||||
|
tabId: number,
|
||||||
|
documentId: string,
|
||||||
|
msg: RecordStartMessage,
|
||||||
|
deps: RecordFrameSendDeps,
|
||||||
|
): Promise<void> {
|
||||||
|
let lastError: unknown;
|
||||||
|
for (let attempt = 0; attempt < RECORD_START_RETRIES; attempt += 1) {
|
||||||
|
try {
|
||||||
|
const response = await deps.sendToDocument(tabId, documentId, msg);
|
||||||
|
if (isRecordStartAck(response)) return;
|
||||||
|
lastError = new Error("content script did not ack RECORD_START");
|
||||||
|
} catch (err) {
|
||||||
|
lastError = err;
|
||||||
|
}
|
||||||
|
if (attempt + 1 < RECORD_START_RETRIES) {
|
||||||
|
await sleep(RECORD_START_RETRY_DELAY_MS);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
throw lastError ?? new Error("failed to start recording in content script");
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function armDocumentCapture(
|
||||||
|
tabId: number,
|
||||||
|
frame: WebNavigationFrameInfo,
|
||||||
|
startMsg: RecordStartMessage,
|
||||||
|
deps: RecordFrameSendDeps,
|
||||||
|
): Promise<{ armed?: ArmedDocument; failure?: FrameCaptureFailure }> {
|
||||||
|
if (!frame.documentId) {
|
||||||
|
return {
|
||||||
|
failure: makeFrameCaptureFailure("not_injectable", {
|
||||||
|
frameId: frame.frameId,
|
||||||
|
url: frame.url,
|
||||||
|
detail: "missing documentId",
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (isFrameUrlRestricted(frame.url)) {
|
||||||
|
return {
|
||||||
|
failure: makeFrameCaptureFailure("not_injectable", {
|
||||||
|
frameId: frame.frameId,
|
||||||
|
documentId: frame.documentId,
|
||||||
|
url: frame.url,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await sendRecordStartWithAckToDocument(tabId, frame.documentId, startMsg, deps);
|
||||||
|
return {
|
||||||
|
armed: {
|
||||||
|
tabId,
|
||||||
|
frameId: frame.frameId,
|
||||||
|
documentId: frame.documentId,
|
||||||
|
url: frame.url,
|
||||||
|
parentFrameId: frame.parentFrameId,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
} catch (err) {
|
||||||
|
return {
|
||||||
|
failure: makeFrameCaptureFailure("arm_failed", {
|
||||||
|
frameId: frame.frameId,
|
||||||
|
documentId: frame.documentId,
|
||||||
|
url: frame.url,
|
||||||
|
detail: err instanceof Error ? err.message : "RECORD_START failed",
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function armAllDocumentsForTab(
|
||||||
|
tabId: number,
|
||||||
|
startMsg: RecordStartMessage,
|
||||||
|
deps: RecordFrameSendDeps,
|
||||||
|
): Promise<{ armed: ArmedDocument[]; failures: FrameCaptureFailure[] }> {
|
||||||
|
const { injectable, failures } = await listInjectableFrames(tabId, deps);
|
||||||
|
const armed: ArmedDocument[] = [];
|
||||||
|
for (const frame of injectable) {
|
||||||
|
const result = await armDocumentCapture(tabId, frame, startMsg, deps);
|
||||||
|
if (result.armed) armed.push(result.armed);
|
||||||
|
if (result.failure) failures.push(result.failure);
|
||||||
|
}
|
||||||
|
return { armed, failures };
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function stopDocumentCapture(
|
||||||
|
armed: ArmedDocument,
|
||||||
|
stopMsg: RecordStopMessage,
|
||||||
|
deps: RecordFrameSendDeps,
|
||||||
|
): Promise<{ ok: true } | FrameCaptureFailure> {
|
||||||
|
try {
|
||||||
|
const response = await deps.sendToDocument(armed.tabId, armed.documentId, stopMsg);
|
||||||
|
if (isRecordStopAckOk(response)) return { ok: true };
|
||||||
|
return makeFrameCaptureFailure("flush_failed", {
|
||||||
|
frameId: armed.frameId,
|
||||||
|
documentId: armed.documentId,
|
||||||
|
url: armed.url,
|
||||||
|
detail: "content script did not confirm recorded steps",
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
return makeFrameCaptureFailure("flush_failed", {
|
||||||
|
frameId: armed.frameId,
|
||||||
|
documentId: armed.documentId,
|
||||||
|
url: armed.url,
|
||||||
|
detail: err instanceof Error ? err.message : "RECORD_STOP failed",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function cancelDocumentCapture(
|
||||||
|
armed: ArmedDocument,
|
||||||
|
cancelMsg: RecordCancelMessage,
|
||||||
|
deps: RecordFrameSendDeps,
|
||||||
|
): Promise<void> {
|
||||||
|
try {
|
||||||
|
await deps.sendToDocument(armed.tabId, armed.documentId, cancelMsg);
|
||||||
|
} catch {
|
||||||
|
// Best-effort cleanup.
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,673 @@
|
|||||||
|
import type { RenderedRef } from "@browser-skill/vom";
|
||||||
|
import { formatObservationFile, type ObservationAnnotation } from "@/lib/format-observation-file";
|
||||||
|
import { fallbackDescriptor, matchTarget } from "@/lib/match-target";
|
||||||
|
import { waitForPageSettled } from "@/lib/page-settled";
|
||||||
|
import {
|
||||||
|
CAPTURE_RETRY_DELAY_MS,
|
||||||
|
DEFAULT_MAX_PAGE_TOKENS,
|
||||||
|
OBSERVATION_MIN_INTERVAL_MS,
|
||||||
|
} from "@/lib/record-constants";
|
||||||
|
import { registerObservation, type StateRegistryEntry } from "@/lib/trace-reducer";
|
||||||
|
import { captureVomObservation } from "@/tools/capture-vom-observation";
|
||||||
|
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||||
|
import type { CapturedNode } from "@/tools/vom/capture";
|
||||||
|
import type { DraftTraceStep, Step, StopReason, Trace, TraceState } from "@/transport/types";
|
||||||
|
import { reduceTraceSteps, resolveTraceStartUrl } from "./trace-reducer";
|
||||||
|
|
||||||
|
export interface LastSettledObservation {
|
||||||
|
stateId: string;
|
||||||
|
captured: CapturedNode[];
|
||||||
|
matchNodes: CapturedNode[];
|
||||||
|
documentTokenOwners: Map<string, number | null>;
|
||||||
|
refs: RenderedRef[];
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
vomText: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** A post-action observation that has been queued but not yet written down. */
|
||||||
|
interface PendingSettle {
|
||||||
|
draftIndex: number;
|
||||||
|
/** Set when a newer action has already decided where this step landed. */
|
||||||
|
cancelled: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Latest URL in an in-flight redirect chain awaiting coalesce + settle. */
|
||||||
|
export interface PendingRedirectLanding {
|
||||||
|
url: string;
|
||||||
|
/** Bumped on every hop so an in-flight `waitForPageSettled` can cancel. */
|
||||||
|
generation: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RecordingObservationState {
|
||||||
|
stateRegistry: Map<string, StateRegistryEntry>;
|
||||||
|
lastSettled: LastSettledObservation | null;
|
||||||
|
/** Latest token→owner map from the most recent observation capture. */
|
||||||
|
documentTokenOwners: Map<string, number | null>;
|
||||||
|
maxPageTokens: number;
|
||||||
|
redactValues: boolean;
|
||||||
|
lastCaptureAtMs: number;
|
||||||
|
/**
|
||||||
|
* Tail of the settle chain. Observations run one at a time so that a slow
|
||||||
|
* capture can never land after a faster one and report the pages out of the
|
||||||
|
* order the user visited them.
|
||||||
|
*/
|
||||||
|
settleQueue: Promise<void>;
|
||||||
|
/** Queued and in-flight settles, by draft index, so newer actions can supersede them. */
|
||||||
|
settles: Map<number, PendingSettle>;
|
||||||
|
stepAnnotations: Map<number, ObservationAnnotation[]>;
|
||||||
|
/**
|
||||||
|
* OAuth / server redirects are coalesced here: intermediate hops only update
|
||||||
|
* `url`+`generation`, and one `navigate` is emitted after the page settles.
|
||||||
|
*/
|
||||||
|
pendingRedirect: PendingRedirectLanding | null;
|
||||||
|
redirectFlushQueue: Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createObservationState(options?: {
|
||||||
|
maxPageTokens?: number;
|
||||||
|
redactValues?: boolean;
|
||||||
|
}): RecordingObservationState {
|
||||||
|
return {
|
||||||
|
stateRegistry: new Map(),
|
||||||
|
lastSettled: null,
|
||||||
|
documentTokenOwners: new Map(),
|
||||||
|
maxPageTokens: options?.maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS,
|
||||||
|
redactValues: options?.redactValues ?? false,
|
||||||
|
lastCaptureAtMs: 0,
|
||||||
|
settleQueue: Promise.resolve(),
|
||||||
|
settles: new Map(),
|
||||||
|
stepAnnotations: new Map(),
|
||||||
|
pendingRedirect: null,
|
||||||
|
redirectFlushQueue: Promise.resolve(),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function sleep(ms: number): Promise<void> {
|
||||||
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readTabMeta(
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
tabId: number,
|
||||||
|
): Promise<{ url: string; title?: string }> {
|
||||||
|
try {
|
||||||
|
const tab = await tabsApi.get(tabId);
|
||||||
|
return { url: tab.url ?? "about:blank", title: tab.title };
|
||||||
|
} catch {
|
||||||
|
return { url: "about:blank" };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function captureAndRegisterObservation(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
urlOverride?: string,
|
||||||
|
): Promise<LastSettledObservation> {
|
||||||
|
const now = Date.now();
|
||||||
|
const waitMs = Math.max(0, OBSERVATION_MIN_INTERVAL_MS - (now - obs.lastCaptureAtMs));
|
||||||
|
if (waitMs > 0) await sleep(waitMs);
|
||||||
|
|
||||||
|
const { url, title } = urlOverride
|
||||||
|
? { url: urlOverride, title: undefined }
|
||||||
|
: await readTabMeta(tabsApi, tabId);
|
||||||
|
const meta = urlOverride ? await readTabMeta(tabsApi, tabId) : { url, title };
|
||||||
|
const resolvedTitle = title ?? meta.title;
|
||||||
|
|
||||||
|
const rendered = await captureVomObservation(cdp, tabId, url, {
|
||||||
|
maxTokens: obs.maxPageTokens,
|
||||||
|
redactValues: obs.redactValues,
|
||||||
|
conditionalSurfaceProbe: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
const stateId = registerObservation(obs.stateRegistry, {
|
||||||
|
url,
|
||||||
|
title: resolvedTitle,
|
||||||
|
rawVomText: rendered.text,
|
||||||
|
truncated: rendered.truncated,
|
||||||
|
});
|
||||||
|
|
||||||
|
obs.lastCaptureAtMs = Date.now();
|
||||||
|
obs.documentTokenOwners = rendered.documentTokenOwners;
|
||||||
|
const settled: LastSettledObservation = {
|
||||||
|
stateId,
|
||||||
|
captured: rendered.captured,
|
||||||
|
matchNodes: rendered.matchNodes,
|
||||||
|
documentTokenOwners: rendered.documentTokenOwners,
|
||||||
|
refs: rendered.refs,
|
||||||
|
url,
|
||||||
|
title: resolvedTitle,
|
||||||
|
vomText: rendered.text,
|
||||||
|
};
|
||||||
|
obs.lastSettled = settled;
|
||||||
|
return settled;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Bind a draft to the page it was performed on: origin state, geometric target
|
||||||
|
* match, and the inline annotation. All three read the *current* observation,
|
||||||
|
* which is only the page the user acted on while the draft is still fresh —
|
||||||
|
* once the action settles, `lastSettled` is the destination page instead.
|
||||||
|
*/
|
||||||
|
export function applyTargetMatching(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
draft: DraftTraceStep,
|
||||||
|
stepId: number,
|
||||||
|
): void {
|
||||||
|
if (!obs.lastSettled) {
|
||||||
|
// Initial observation can fail or race the first action. Keep whatever
|
||||||
|
// the content script captured so the exported step is still teachable.
|
||||||
|
if ("target" in draft && "captureTarget" in draft) {
|
||||||
|
draft.target = fallbackDescriptor(draft.captureTarget);
|
||||||
|
}
|
||||||
|
rememberStepAnnotation(obs, stepId, draft);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
draft.preStateId = obs.lastSettled.stateId;
|
||||||
|
|
||||||
|
if ("target" in draft) {
|
||||||
|
draft.target =
|
||||||
|
"geometry" in draft && draft.geometry
|
||||||
|
? matchTarget({
|
||||||
|
geometry: draft.geometry,
|
||||||
|
captured: obs.lastSettled.matchNodes,
|
||||||
|
refs: obs.lastSettled.refs,
|
||||||
|
fallback: draft.captureTarget,
|
||||||
|
})
|
||||||
|
: // No geometry to match on, but the capture still knew what the user
|
||||||
|
// touched — keep that instead of an anonymous unmatched target.
|
||||||
|
fallbackDescriptor(draft.captureTarget);
|
||||||
|
}
|
||||||
|
rememberStepAnnotation(obs, stepId, draft);
|
||||||
|
}
|
||||||
|
|
||||||
|
function fillDetailForDraft(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
draft: DraftTraceStep,
|
||||||
|
): string | undefined {
|
||||||
|
if (obs.redactValues) return undefined;
|
||||||
|
if (draft.op === "fill") return JSON.stringify(draft.value);
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
|
function refLineForDraft(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
draft: DraftTraceStep,
|
||||||
|
): number | undefined {
|
||||||
|
if (!("target" in draft) || !obs.lastSettled) return undefined;
|
||||||
|
const targetRef = draft.target?.ref;
|
||||||
|
if (!targetRef) return undefined;
|
||||||
|
return obs.lastSettled.refs.find((r) => r.ref === targetRef)?.line;
|
||||||
|
}
|
||||||
|
|
||||||
|
function rememberStepAnnotation(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
stepId: number,
|
||||||
|
draft: DraftTraceStep,
|
||||||
|
): void {
|
||||||
|
const line = refLineForDraft(obs, draft);
|
||||||
|
if (line === undefined || draft.op === "navigate" || draft.op === "scroll") return;
|
||||||
|
if (!draft.preStateId) return;
|
||||||
|
const ann: ObservationAnnotation = {
|
||||||
|
stepId,
|
||||||
|
op: draft.op,
|
||||||
|
line,
|
||||||
|
stateId: draft.preStateId,
|
||||||
|
detail: fillDetailForDraft(obs, draft),
|
||||||
|
};
|
||||||
|
const bucket = obs.stepAnnotations.get(line) ?? [];
|
||||||
|
bucket.push(ann);
|
||||||
|
obs.stepAnnotations.set(line, bucket);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Record the step under the page it was performed on, not the one it led to. */
|
||||||
|
export function rememberStepOnPage(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
draft: DraftTraceStep,
|
||||||
|
stepId: number,
|
||||||
|
): void {
|
||||||
|
const stateId = draft.preStateId;
|
||||||
|
if (!stateId) return;
|
||||||
|
const entry = obs.stateRegistry.get(stateId);
|
||||||
|
if (entry && !entry.stepsHere.includes(stepId)) {
|
||||||
|
entry.stepsHere.push(stepId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Drop an in-flight redirect coalesce (e.g. a real navigate superseded it). */
|
||||||
|
export function clearPendingRedirectLanding(obs: RecordingObservationState): void {
|
||||||
|
obs.pendingRedirect = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function enqueueRedirectFlush(obs: RecordingObservationState, task: () => Promise<void>): void {
|
||||||
|
obs.redirectFlushQueue = obs.redirectFlushQueue.then(task, task).catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Remember a redirect hop and schedule a settle-then-emit of a single navigate
|
||||||
|
* to the final URL. Later hops only bump `generation` / replace `url`.
|
||||||
|
*/
|
||||||
|
export function scheduleRedirectLandingFlush(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
steps: DraftTraceStep[],
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
url: string,
|
||||||
|
options: { cancelled?: () => boolean } = {},
|
||||||
|
): void {
|
||||||
|
const generation = (obs.pendingRedirect?.generation ?? 0) + 1;
|
||||||
|
obs.pendingRedirect = { url, generation };
|
||||||
|
enqueueRedirectFlush(obs, () =>
|
||||||
|
coalesceRedirectLanding(obs, steps, cdp, tabId, tabsApi, options),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Drain any coalesced redirect so the next action matches against the real
|
||||||
|
* landing page. Safe to call when nothing is pending.
|
||||||
|
*/
|
||||||
|
export async function flushPendingRedirectLanding(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
steps: DraftTraceStep[],
|
||||||
|
cdp: CdpRunner | undefined,
|
||||||
|
tabId: number,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
options: { cancelled?: () => boolean } = {},
|
||||||
|
): Promise<void> {
|
||||||
|
if (obs.pendingRedirect && cdp) {
|
||||||
|
enqueueRedirectFlush(obs, () =>
|
||||||
|
coalesceRedirectLanding(obs, steps, cdp, tabId, tabsApi, options),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await obs.redirectFlushQueue;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wait until the redirect chain stops changing the document, then emit one
|
||||||
|
* `navigate` (cause `browser`) to the tab's final URL and settle its observation.
|
||||||
|
*/
|
||||||
|
async function coalesceRedirectLanding(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
steps: DraftTraceStep[],
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
options: { cancelled?: () => boolean },
|
||||||
|
): Promise<void> {
|
||||||
|
while (obs.pendingRedirect) {
|
||||||
|
if (options.cancelled?.()) {
|
||||||
|
obs.pendingRedirect = null;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const snap = obs.pendingRedirect;
|
||||||
|
const outcome = await waitForPageSettled(cdp, tabId, {
|
||||||
|
cancelled: () =>
|
||||||
|
!!options.cancelled?.() ||
|
||||||
|
obs.pendingRedirect === null ||
|
||||||
|
obs.pendingRedirect.generation !== snap.generation,
|
||||||
|
});
|
||||||
|
|
||||||
|
if (options.cancelled?.()) {
|
||||||
|
obs.pendingRedirect = null;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// A newer hop cancelled this wait — loop and settle against the latest URL.
|
||||||
|
if (outcome === "cancelled") continue;
|
||||||
|
if (!obs.pendingRedirect || obs.pendingRedirect.generation !== snap.generation) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
const finalUrl = (await readTabMeta(tabsApi, tabId)).url || snap.url;
|
||||||
|
obs.pendingRedirect = null;
|
||||||
|
|
||||||
|
if (!finalUrl || finalUrl === "about:blank") return;
|
||||||
|
if (obs.lastSettled?.url === finalUrl) return;
|
||||||
|
|
||||||
|
const last = steps[steps.length - 1];
|
||||||
|
if (last?.op === "navigate" && last.url === finalUrl) {
|
||||||
|
if (!last.postStateId) {
|
||||||
|
const draftIndex = steps.length - 1;
|
||||||
|
scheduleDraftSettle(obs, draftIndex, draftIndex + 1, cdp, tabId, tabsApi, steps);
|
||||||
|
await obs.settleQueue;
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (outcome === "timeout") {
|
||||||
|
console.debug(
|
||||||
|
`[bsk record] redirect landing still changing after settle budget; ` +
|
||||||
|
`recording navigate to ${finalUrl}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const draft: DraftTraceStep = {
|
||||||
|
op: "navigate",
|
||||||
|
url: finalUrl,
|
||||||
|
page_url: finalUrl,
|
||||||
|
cause: "browser",
|
||||||
|
preStateId: obs.lastSettled?.stateId,
|
||||||
|
};
|
||||||
|
steps.push(draft);
|
||||||
|
const draftIndex = steps.length - 1;
|
||||||
|
const stepId = draftIndex + 1;
|
||||||
|
rememberStepOnPage(obs, draft, stepId);
|
||||||
|
scheduleDraftSettle(obs, draftIndex, stepId, cdp, tabId, tabsApi, steps);
|
||||||
|
// Callers that flush before the next action need `lastSettled` to already
|
||||||
|
// be the landing page so target matching does not use the pre-redirect view.
|
||||||
|
await obs.settleQueue;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A capture that lands mid-navigation fails on a destroyed execution context.
|
||||||
|
* That is transient, so give it one more chance before the step has to fall
|
||||||
|
* back to a stale state.
|
||||||
|
*/
|
||||||
|
async function captureWithRetry(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
urlOverride?: string,
|
||||||
|
): Promise<LastSettledObservation> {
|
||||||
|
try {
|
||||||
|
return await captureAndRegisterObservation(obs, cdp, tabId, tabsApi, urlOverride);
|
||||||
|
} catch (err) {
|
||||||
|
console.debug("[bsk record] observation failed, retrying once", err);
|
||||||
|
await sleep(CAPTURE_RETRY_DELAY_MS);
|
||||||
|
return captureAndRegisterObservation(obs, cdp, tabId, tabsApi, urlOverride);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function settleDraftObservation(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
steps: DraftTraceStep[],
|
||||||
|
draftIndex: number,
|
||||||
|
pending?: PendingSettle,
|
||||||
|
): Promise<void> {
|
||||||
|
const cancelled = () => pending?.cancelled === true;
|
||||||
|
const outcome = await waitForPageSettled(cdp, tabId, { cancelled });
|
||||||
|
if (outcome === "cancelled") return;
|
||||||
|
if (outcome === "timeout") {
|
||||||
|
console.debug(
|
||||||
|
`[bsk record] step ${draftIndex + 1} was still changing the page after the settle ` +
|
||||||
|
"budget; observing it as it stands",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
const started = steps[draftIndex];
|
||||||
|
if (!started) return;
|
||||||
|
|
||||||
|
// The URL attached to the draft may only be an intermediate redirect hop.
|
||||||
|
// Once the page is settled, register the capture against live tab metadata.
|
||||||
|
const settled = await captureWithRetry(obs, cdp, tabId, tabsApi);
|
||||||
|
// The observation still counts as the latest view of the page even when this
|
||||||
|
// step no longer wants it — the next action will start from it.
|
||||||
|
if (cancelled()) return;
|
||||||
|
// Re-read the slot: a navigation observed while the capture ran may have
|
||||||
|
// rewritten this draft, and the observation belongs to whatever occupies the
|
||||||
|
// slot now — writing to the object we started with would strand it.
|
||||||
|
const draft = steps[draftIndex] ?? started;
|
||||||
|
draft.postStateId = settled.stateId;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Recover landing states from the shape of the recording itself: wherever the
|
||||||
|
* next action was performed is, by definition, where the previous one landed.
|
||||||
|
* Only the immediately following draft counts — a later one would claim a page
|
||||||
|
* that several unobserved actions away, which is worse than admitting we saw
|
||||||
|
* no change.
|
||||||
|
*/
|
||||||
|
export function inferMissingPostStates(steps: DraftTraceStep[]): void {
|
||||||
|
for (let i = 0; i < steps.length - 1; i += 1) {
|
||||||
|
const draft = steps[i];
|
||||||
|
const next = steps[i + 1];
|
||||||
|
if (!draft || !next || draft.postStateId || !next.preStateId) continue;
|
||||||
|
draft.postStateId = next.preStateId;
|
||||||
|
console.debug(
|
||||||
|
`[bsk record] step ${i + 1} (${draft.op}) had no post-action observation; ` +
|
||||||
|
`using where step ${i + 2} started (${next.preStateId})`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Last chance for the closing actions of a recording. Stopping right after the
|
||||||
|
* final action is normal, and so is that action navigating away, so take one
|
||||||
|
* final look at the page. Only trailing drafts may claim it: an earlier step
|
||||||
|
* did not land on whatever the user happens to be looking at when they stop.
|
||||||
|
*/
|
||||||
|
export async function settleUnsettledDrafts(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
steps: DraftTraceStep[],
|
||||||
|
): Promise<void> {
|
||||||
|
const trailing: DraftTraceStep[] = [];
|
||||||
|
for (let i = steps.length - 1; i >= 0; i -= 1) {
|
||||||
|
const draft = steps[i];
|
||||||
|
if (!draft || draft.postStateId) break;
|
||||||
|
trailing.push(draft);
|
||||||
|
}
|
||||||
|
if (trailing.length === 0) return;
|
||||||
|
|
||||||
|
let settled: LastSettledObservation;
|
||||||
|
try {
|
||||||
|
settled = await captureWithRetry(obs, cdp, tabId, tabsApi);
|
||||||
|
} catch (err) {
|
||||||
|
console.warn("[bsk record] final observation at stop failed", err);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
for (const draft of trailing) draft.postStateId = settled.stateId;
|
||||||
|
console.debug(
|
||||||
|
`[bsk record] settled ${trailing.length} trailing step(s) against the page at stop ` +
|
||||||
|
`(${settled.stateId} ${settled.url})`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Performing a new action ends the previous one: the page the user reached for
|
||||||
|
* is, by definition, where the earlier step left them. Taking the landing from
|
||||||
|
* the newer step also keeps the trace monotonic — a capture that finished late
|
||||||
|
* would otherwise credit an earlier step with a page that only exists because
|
||||||
|
* of the newer action.
|
||||||
|
*/
|
||||||
|
function supersedeEarlierSettles(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
draftIndex: number,
|
||||||
|
steps: DraftTraceStep[],
|
||||||
|
): void {
|
||||||
|
const landing = steps[draftIndex]?.preStateId;
|
||||||
|
for (const [index, pending] of obs.settles) {
|
||||||
|
if (index >= draftIndex) continue;
|
||||||
|
pending.cancelled = true;
|
||||||
|
obs.settles.delete(index);
|
||||||
|
|
||||||
|
const draft = steps[index];
|
||||||
|
if (!draft || draft.postStateId) continue;
|
||||||
|
if (!landing) continue;
|
||||||
|
draft.postStateId = landing;
|
||||||
|
console.debug(
|
||||||
|
`[bsk record] step ${index + 1} (${draft.op}) was still settling when step ` +
|
||||||
|
`${draftIndex + 1} started; landing it on ${landing}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function scheduleDraftSettle(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
draftIndex: number,
|
||||||
|
stepId: number,
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
tabsApi: ChromeTabsApi,
|
||||||
|
steps: DraftTraceStep[],
|
||||||
|
): void {
|
||||||
|
supersedeEarlierSettles(obs, draftIndex, steps);
|
||||||
|
|
||||||
|
// Rescheduling the same step (a navigation showed up after the action)
|
||||||
|
// replaces the pending observation rather than racing it.
|
||||||
|
const superseded = obs.settles.get(draftIndex);
|
||||||
|
if (superseded) superseded.cancelled = true;
|
||||||
|
|
||||||
|
const pending: PendingSettle = { draftIndex, cancelled: false };
|
||||||
|
obs.settles.set(draftIndex, pending);
|
||||||
|
|
||||||
|
enqueueSettle(obs, async () => {
|
||||||
|
if (pending.cancelled) return;
|
||||||
|
try {
|
||||||
|
await settleDraftObservation(obs, cdp, tabId, tabsApi, steps, draftIndex, pending);
|
||||||
|
} catch (err) {
|
||||||
|
// Recoverable: stop-time repair still gives the step a landing page.
|
||||||
|
const op = steps[draftIndex]?.op ?? "?";
|
||||||
|
console.warn(`[bsk record] post-action observation failed for step ${stepId} (${op})`, err);
|
||||||
|
} finally {
|
||||||
|
if (obs.settles.get(draftIndex) === pending) obs.settles.delete(draftIndex);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function enqueueSettle(obs: RecordingObservationState, task: () => Promise<void>): void {
|
||||||
|
obs.settleQueue = obs.settleQueue.then(task, task).catch(() => {});
|
||||||
|
}
|
||||||
|
|
||||||
|
export function cancelPendingSettles(obs: RecordingObservationState): void {
|
||||||
|
for (const pending of obs.settles.values()) pending.cancelled = true;
|
||||||
|
obs.settles.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Bound the drain loop so a self-rescheduling settle cannot block stop. */
|
||||||
|
const MAX_SETTLE_FLUSH_ROUNDS = 10;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Drain settle work before exporting, including work queued while draining —
|
||||||
|
* a navigation observed during the last capture schedules another one, and
|
||||||
|
* the trace would drop it if stop did not wait.
|
||||||
|
*/
|
||||||
|
export async function flushPendingSettles(obs: RecordingObservationState): Promise<void> {
|
||||||
|
for (let round = 0; round < MAX_SETTLE_FLUSH_ROUNDS; round += 1) {
|
||||||
|
const drained = obs.settleQueue;
|
||||||
|
await drained;
|
||||||
|
if (obs.settleQueue === drained) return;
|
||||||
|
}
|
||||||
|
console.warn("[bsk record] settle queue kept growing at stop; exporting what has been observed");
|
||||||
|
}
|
||||||
|
|
||||||
|
function finalizeStateBodies(
|
||||||
|
entries: StateRegistryEntry[],
|
||||||
|
annotationsByState: Map<string, ObservationAnnotation[]>,
|
||||||
|
stepIdByDraftId: Map<number, number>,
|
||||||
|
idByOldId: Map<string, string>,
|
||||||
|
): TraceState[] {
|
||||||
|
return entries.map((entry) => {
|
||||||
|
const id = idByOldId.get(entry.id) ?? entry.id;
|
||||||
|
const body = formatObservationFile({
|
||||||
|
stateId: id,
|
||||||
|
url: entry.url,
|
||||||
|
title: entry.title,
|
||||||
|
stepsHere: remapStepIds(entry.stepsHere, stepIdByDraftId),
|
||||||
|
body: entry.rawVomText,
|
||||||
|
annotations: annotationsByState.get(entry.id) ?? [],
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
id,
|
||||||
|
url: entry.url,
|
||||||
|
...(entry.title ? { title: entry.title } : {}),
|
||||||
|
body,
|
||||||
|
...(entry.truncated ? { truncated: true } : {}),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Draft ids only become step ids after collapsing and filtering. */
|
||||||
|
function remapStepIds(draftIds: number[], stepIdByDraftId: Map<number, number>): number[] {
|
||||||
|
const mapped = new Set<number>();
|
||||||
|
for (const draftId of draftIds) {
|
||||||
|
const stepId = stepIdByDraftId.get(draftId);
|
||||||
|
if (stepId !== undefined) mapped.add(stepId);
|
||||||
|
}
|
||||||
|
return [...mapped].sort((a, b) => a - b);
|
||||||
|
}
|
||||||
|
|
||||||
|
function collectAnnotationsByState(
|
||||||
|
obs: RecordingObservationState,
|
||||||
|
stepIdByDraftId: Map<number, number>,
|
||||||
|
): Map<string, ObservationAnnotation[]> {
|
||||||
|
const byState = new Map<string, ObservationAnnotation[]>();
|
||||||
|
for (const anns of obs.stepAnnotations.values()) {
|
||||||
|
for (const ann of anns) {
|
||||||
|
const stepId = stepIdByDraftId.get(ann.stepId);
|
||||||
|
if (stepId === undefined) continue;
|
||||||
|
const bucket = byState.get(ann.stateId) ?? [];
|
||||||
|
bucket.push({ ...ann, stepId });
|
||||||
|
byState.set(ann.stateId, bucket);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return byState;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Keep only the pages the published steps point at. Redirect hops and
|
||||||
|
* mid-load captures land in the registry too, and shipping them would invite
|
||||||
|
* a reader to treat a page the flow merely passed through as a real stop.
|
||||||
|
* With no steps at all the first observation is the whole artifact, so it
|
||||||
|
* stays.
|
||||||
|
*/
|
||||||
|
function selectPublishedStates(
|
||||||
|
registry: Map<string, StateRegistryEntry>,
|
||||||
|
steps: Step[],
|
||||||
|
): StateRegistryEntry[] {
|
||||||
|
const entries = [...registry.values()];
|
||||||
|
if (steps.length === 0) return entries.slice(0, 1);
|
||||||
|
const referenced = new Set<string>();
|
||||||
|
for (const step of steps) {
|
||||||
|
referenced.add(step.state);
|
||||||
|
referenced.add(step.result.state);
|
||||||
|
}
|
||||||
|
return entries.filter((entry) => referenced.has(entry.id));
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildTraceV3(input: {
|
||||||
|
obs: RecordingObservationState;
|
||||||
|
steps: DraftTraceStep[];
|
||||||
|
startedAt: string;
|
||||||
|
purpose?: string;
|
||||||
|
startUrl?: string;
|
||||||
|
stoppedBy: StopReason;
|
||||||
|
bskVersion: string;
|
||||||
|
frameCapture?: Trace["frame_capture"];
|
||||||
|
}): Trace {
|
||||||
|
const { steps, stepIdByDraftId } = reduceTraceSteps(input.steps, input.obs.stateRegistry);
|
||||||
|
const published = selectPublishedStates(input.obs.stateRegistry, steps);
|
||||||
|
// Renumber so the shipped dictionary reads s1..sN without holes where the
|
||||||
|
// dropped captures used to be.
|
||||||
|
const idByOldId = new Map(published.map((entry, index) => [entry.id, `s${index + 1}`]));
|
||||||
|
for (const step of steps) {
|
||||||
|
step.state = idByOldId.get(step.state) ?? step.state;
|
||||||
|
step.result.state = idByOldId.get(step.result.state) ?? step.result.state;
|
||||||
|
}
|
||||||
|
const annotationsByState = collectAnnotationsByState(input.obs, stepIdByDraftId);
|
||||||
|
const states = finalizeStateBodies(published, annotationsByState, stepIdByDraftId, idByOldId);
|
||||||
|
const startUrl = resolveTraceStartUrl(input.steps, input.startUrl, states);
|
||||||
|
return {
|
||||||
|
version: 3,
|
||||||
|
...(input.purpose ? { purpose: input.purpose } : {}),
|
||||||
|
recorded_at: new Date().toISOString(),
|
||||||
|
started_at: input.startedAt,
|
||||||
|
stopped_by: input.stoppedBy,
|
||||||
|
entry: { start_url: startUrl },
|
||||||
|
recorder: { bsk: input.bskVersion, vom: 1 },
|
||||||
|
...(input.frameCapture ? { frame_capture: input.frameCapture } : {}),
|
||||||
|
states,
|
||||||
|
steps,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -1,23 +1,29 @@
|
|||||||
import type { DraftTraceStep } from "@/transport/types";
|
import type { DraftTraceStep, NavigationCause } from "@/transport/types";
|
||||||
import type { RecordStepPayload } from "./record-bridge";
|
import type { RecordStepPayload } from "./record-bridge";
|
||||||
|
import { mapNavigationCause } from "./record-constants";
|
||||||
|
|
||||||
export interface RecordingStepBuffer {
|
export interface RecordingStepBuffer {
|
||||||
steps: DraftTraceStep[];
|
steps: DraftTraceStep[];
|
||||||
currentUrl?: string;
|
currentUrl?: string;
|
||||||
pendingNavigation: boolean;
|
pendingNavigation: boolean;
|
||||||
pendingNavigationDeadline?: number;
|
pendingNavigationDeadline?: number;
|
||||||
|
lastTransitionType?: string;
|
||||||
|
lastTransitionQualifiers?: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
|
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
|
||||||
|
|
||||||
function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
||||||
const pageUrl = payload.page_url;
|
const pageUrl = payload.page_url;
|
||||||
|
const geometry = payload.geometry;
|
||||||
switch (payload.op) {
|
switch (payload.op) {
|
||||||
case "click":
|
case "click":
|
||||||
return payload.target
|
return payload.target
|
||||||
? {
|
? {
|
||||||
op: "click",
|
op: "click",
|
||||||
target: payload.target,
|
target: { unmatched: true },
|
||||||
|
captureTarget: payload.target,
|
||||||
|
...(geometry ? { geometry } : {}),
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||||
}
|
}
|
||||||
: null;
|
: null;
|
||||||
@@ -25,7 +31,9 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
|||||||
return payload.target
|
return payload.target
|
||||||
? {
|
? {
|
||||||
op: "hover",
|
op: "hover",
|
||||||
target: payload.target,
|
target: { unmatched: true },
|
||||||
|
captureTarget: payload.target,
|
||||||
|
...(geometry ? { geometry } : {}),
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||||
}
|
}
|
||||||
: null;
|
: null;
|
||||||
@@ -33,8 +41,11 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
|||||||
return payload.target
|
return payload.target
|
||||||
? {
|
? {
|
||||||
op: "fill",
|
op: "fill",
|
||||||
target: payload.target,
|
target: { unmatched: true },
|
||||||
|
captureTarget: payload.target,
|
||||||
|
...(geometry ? { geometry } : {}),
|
||||||
value: payload.value ?? "",
|
value: payload.value ?? "",
|
||||||
|
...(payload.commit ? { commit: payload.commit } : {}),
|
||||||
...(payload.redacted ? { redacted: true } : {}),
|
...(payload.redacted ? { redacted: true } : {}),
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||||
}
|
}
|
||||||
@@ -44,7 +55,10 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
|||||||
? {
|
? {
|
||||||
op: "press",
|
op: "press",
|
||||||
key: payload.key,
|
key: payload.key,
|
||||||
...(payload.target ? { target: payload.target } : {}),
|
...(payload.target
|
||||||
|
? { target: { unmatched: true }, captureTarget: payload.target }
|
||||||
|
: {}),
|
||||||
|
...(geometry ? { geometry } : {}),
|
||||||
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
|
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||||
}
|
}
|
||||||
@@ -53,37 +67,69 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
|||||||
return payload.target && payload.values
|
return payload.target && payload.values
|
||||||
? {
|
? {
|
||||||
op: "select",
|
op: "select",
|
||||||
target: payload.target,
|
target: { unmatched: true },
|
||||||
|
captureTarget: payload.target,
|
||||||
|
...(geometry ? { geometry } : {}),
|
||||||
values: payload.values,
|
values: payload.values,
|
||||||
...(payload.labels?.length ? { labels: payload.labels } : {}),
|
...(payload.labels?.length ? { labels: payload.labels } : {}),
|
||||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||||
}
|
}
|
||||||
: null;
|
: null;
|
||||||
|
case "scroll":
|
||||||
|
return { op: "scroll", ...(pageUrl ? { page_url: pageUrl } : {}) };
|
||||||
case "navigate":
|
case "navigate":
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): boolean {
|
/** Annotate the latest action with `navigated_to`. Returns its index, or -1. */
|
||||||
|
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): number {
|
||||||
for (let i = buffer.steps.length - 1; i >= 0; i -= 1) {
|
for (let i = buffer.steps.length - 1; i >= 0; i -= 1) {
|
||||||
const step = buffer.steps[i];
|
const step = buffer.steps[i];
|
||||||
if (!step) continue;
|
if (!step) continue;
|
||||||
if (step.op === "click" || step.op === "press" || step.op === "select") {
|
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
||||||
buffer.steps[i] = { ...step, navigated_to: url };
|
// Mutate in place: an in-flight settle holds this object, and replacing
|
||||||
return true;
|
// it would strand that settle's observation on the discarded copy.
|
||||||
|
step.navigated_to = url;
|
||||||
|
return i;
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
}
|
}
|
||||||
return false;
|
return -1;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function causeFromBuffer(buffer: RecordingStepBuffer): NavigationCause | null {
|
||||||
|
return mapNavigationCause({
|
||||||
|
transitionType: buffer.lastTransitionType,
|
||||||
|
transitionQualifiers: buffer.lastTransitionQualifiers,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Outcome of folding a main-frame URL change into the draft buffer. */
|
||||||
|
export type NavigationObserveResult =
|
||||||
|
| { kind: "noop" }
|
||||||
|
| { kind: "annotated"; index: number }
|
||||||
|
| { kind: "appended"; index: number }
|
||||||
|
/**
|
||||||
|
* Redirect hop: do not emit a step yet. The recorder coalesces the chain and
|
||||||
|
* emits one `navigate` after `waitForPageSettled` on the final URL.
|
||||||
|
*/
|
||||||
|
| { kind: "coalesce_redirect"; url: string };
|
||||||
|
|
||||||
export function observeRecordedNavigation(
|
export function observeRecordedNavigation(
|
||||||
buffer: RecordingStepBuffer,
|
buffer: RecordingStepBuffer,
|
||||||
url: string,
|
url: string,
|
||||||
causedByAction?: boolean,
|
causedByAction?: boolean,
|
||||||
): void {
|
transitionType?: string,
|
||||||
if (!url || url === buffer.currentUrl) return;
|
transitionQualifiers?: string[],
|
||||||
|
): NavigationObserveResult {
|
||||||
|
if (!url || url === buffer.currentUrl) return { kind: "noop" };
|
||||||
buffer.currentUrl = url;
|
buffer.currentUrl = url;
|
||||||
|
if (transitionType !== undefined) buffer.lastTransitionType = transitionType;
|
||||||
|
if (transitionQualifiers !== undefined) {
|
||||||
|
buffer.lastTransitionQualifiers = transitionQualifiers;
|
||||||
|
}
|
||||||
|
|
||||||
const pendingIsCurrent =
|
const pendingIsCurrent =
|
||||||
buffer.pendingNavigation &&
|
buffer.pendingNavigation &&
|
||||||
(buffer.pendingNavigationDeadline === undefined ||
|
(buffer.pendingNavigationDeadline === undefined ||
|
||||||
@@ -92,40 +138,69 @@ export function observeRecordedNavigation(
|
|||||||
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
|
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
|
||||||
buffer.pendingNavigation = false;
|
buffer.pendingNavigation = false;
|
||||||
buffer.pendingNavigationDeadline = undefined;
|
buffer.pendingNavigationDeadline = undefined;
|
||||||
if (!annotateLastStepNavigation(buffer, url)) {
|
const annotatedIndex = annotateLastStepNavigation(buffer, url);
|
||||||
buffer.steps.push({
|
if (annotatedIndex >= 0) {
|
||||||
op: "navigate",
|
return { kind: "annotated", index: annotatedIndex };
|
||||||
url,
|
|
||||||
page_url: url,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
return;
|
const cause = causeFromBuffer(buffer);
|
||||||
|
if (cause === null) {
|
||||||
|
return { kind: "coalesce_redirect", url };
|
||||||
|
}
|
||||||
|
buffer.steps.push({
|
||||||
|
op: "navigate",
|
||||||
|
url,
|
||||||
|
page_url: url,
|
||||||
|
cause,
|
||||||
|
transitionType: buffer.lastTransitionType,
|
||||||
|
transitionQualifiers: buffer.lastTransitionQualifiers,
|
||||||
|
});
|
||||||
|
return { kind: "appended", index: buffer.steps.length - 1 };
|
||||||
}
|
}
|
||||||
|
|
||||||
buffer.pendingNavigation = false;
|
buffer.pendingNavigation = false;
|
||||||
buffer.pendingNavigationDeadline = undefined;
|
buffer.pendingNavigationDeadline = undefined;
|
||||||
|
const cause = causeFromBuffer(buffer);
|
||||||
|
if (cause === null) {
|
||||||
|
return { kind: "coalesce_redirect", url };
|
||||||
|
}
|
||||||
buffer.steps.push({
|
buffer.steps.push({
|
||||||
op: "navigate",
|
op: "navigate",
|
||||||
url,
|
url,
|
||||||
page_url: url,
|
page_url: url,
|
||||||
|
cause,
|
||||||
|
transitionType: buffer.lastTransitionType,
|
||||||
|
transitionQualifiers: buffer.lastTransitionQualifiers,
|
||||||
});
|
});
|
||||||
|
return { kind: "appended", index: buffer.steps.length - 1 };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Buffer one captured payload. Returns the index of the draft it appended, or
|
||||||
|
* `null` when the payload produced none (an unnamed click target, a navigation
|
||||||
|
* that only annotated the previous draft, …). Callers must not fall back to
|
||||||
|
* "the last draft" on `null`: that would re-match an already recorded step
|
||||||
|
* against a newer observation and overwrite its origin state.
|
||||||
|
*/
|
||||||
export function appendRecordedPayload(
|
export function appendRecordedPayload(
|
||||||
buffer: RecordingStepBuffer,
|
buffer: RecordingStepBuffer,
|
||||||
payload: RecordStepPayload,
|
payload: RecordStepPayload,
|
||||||
): void {
|
): number | null {
|
||||||
if (payload.op === "navigate") {
|
if (payload.op === "navigate") {
|
||||||
if (payload.url) {
|
if (!payload.url) return null;
|
||||||
observeRecordedNavigation(buffer, payload.url, payload.navigation_caused_by_action);
|
const result = observeRecordedNavigation(
|
||||||
}
|
buffer,
|
||||||
return;
|
payload.url,
|
||||||
|
payload.navigation_caused_by_action,
|
||||||
|
payload.transitionType,
|
||||||
|
payload.transitionQualifiers,
|
||||||
|
);
|
||||||
|
return result.kind === "appended" ? result.index : null;
|
||||||
}
|
}
|
||||||
const step = toDraftStep({
|
const step = toDraftStep({
|
||||||
...payload,
|
...payload,
|
||||||
page_url: payload.page_url ?? buffer.currentUrl,
|
page_url: payload.page_url ?? buffer.currentUrl,
|
||||||
});
|
});
|
||||||
if (!step) return;
|
if (!step) return null;
|
||||||
buffer.steps.push(step);
|
buffer.steps.push(step);
|
||||||
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
||||||
buffer.pendingNavigation = payload.expects_navigation === true;
|
buffer.pendingNavigation = payload.expects_navigation === true;
|
||||||
@@ -133,4 +208,5 @@ export function appendRecordedPayload(
|
|||||||
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
|
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
|
||||||
: undefined;
|
: undefined;
|
||||||
}
|
}
|
||||||
|
return buffer.steps.length - 1;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,4 +1,17 @@
|
|||||||
import type { DraftTraceStep, PageRef, SelectedOption, Step } from "@/transport/types";
|
import type {
|
||||||
|
DraftTraceStep,
|
||||||
|
FillCommit,
|
||||||
|
NavigationCause,
|
||||||
|
Step,
|
||||||
|
TraceState,
|
||||||
|
} from "@/transport/types";
|
||||||
|
import {
|
||||||
|
DEFAULT_FILL_COMMIT,
|
||||||
|
fnv1a64,
|
||||||
|
mapNavigationCause,
|
||||||
|
nextStateId,
|
||||||
|
resetStateIdCounterForTests,
|
||||||
|
} from "./record-constants";
|
||||||
|
|
||||||
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
|
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
|
||||||
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
|
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
|
||||||
@@ -12,206 +25,200 @@ export function shouldRecordPress(
|
|||||||
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
|
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
|
||||||
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
|
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
|
||||||
if (key === "Enter" || key === "Escape") return true;
|
if (key === "Enter" || key === "Escape") return true;
|
||||||
// Drop bare character typing — FillSession already records the value.
|
|
||||||
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
|
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
function shouldIncludeDraft(step: DraftTraceStep): boolean {
|
function shouldIncludeDraft(step: DraftTraceStep): boolean {
|
||||||
if (step.op === "fill" && !(step.value ?? "").trim() && !step.redacted) return false;
|
|
||||||
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
|
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
|
||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Collapse consecutive navigations to the last hop. */
|
/** A reduced draft plus the capture-order ids of every draft folded into it. */
|
||||||
function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
|
interface CollapsedDraft {
|
||||||
const out: DraftTraceStep[] = [];
|
draft: DraftTraceStep;
|
||||||
for (const step of steps) {
|
draftIds: number[];
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Collapse a redirect chain into a single hop.
|
||||||
|
*
|
||||||
|
* The surviving step keeps the *first* hop's origin state and cause — that is
|
||||||
|
* where the user was and why they left — and the *last* hop's destination, so
|
||||||
|
* the intermediate URL the browser only passed through never becomes the
|
||||||
|
* step's `state` or `to`.
|
||||||
|
*/
|
||||||
|
function collapseNavigations(steps: DraftTraceStep[]): CollapsedDraft[] {
|
||||||
|
const out: CollapsedDraft[] = [];
|
||||||
|
steps.forEach((step, index) => {
|
||||||
|
const draftId = index + 1;
|
||||||
const prev = out[out.length - 1];
|
const prev = out[out.length - 1];
|
||||||
if (step.op === "navigate" && prev?.op === "navigate") {
|
if (step.op === "navigate" && prev?.draft.op === "navigate") {
|
||||||
out[out.length - 1] = step;
|
prev.draft = {
|
||||||
continue;
|
...prev.draft,
|
||||||
|
url: step.url,
|
||||||
|
postStateId: step.postStateId ?? prev.draft.postStateId,
|
||||||
|
};
|
||||||
|
prev.draftIds.push(draftId);
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
out.push(step);
|
out.push({ draft: step, draftIds: [draftId] });
|
||||||
}
|
});
|
||||||
return out;
|
return out;
|
||||||
}
|
}
|
||||||
|
|
||||||
function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
export interface StateRegistryEntry extends TraceState {
|
||||||
const urls: string[] = [];
|
contentHash: string;
|
||||||
if (startUrl) urls.push(startUrl);
|
rawVomText: string;
|
||||||
for (const step of steps) {
|
stepsHere: number[];
|
||||||
if (step.op === "navigate") {
|
|
||||||
urls.push(step.url);
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
if ("page_url" in step && step.page_url) urls.push(step.page_url);
|
|
||||||
if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to);
|
|
||||||
}
|
|
||||||
const seen = new Set<string>();
|
|
||||||
const unique: string[] = [];
|
|
||||||
for (const url of urls) {
|
|
||||||
if (!url || seen.has(url)) continue;
|
|
||||||
seen.add(url);
|
|
||||||
unique.push(url);
|
|
||||||
}
|
|
||||||
return unique;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildPageRegistry(
|
export function stateDedupKey(body: string, url: string): string {
|
||||||
steps: DraftTraceStep[],
|
return `${fnv1a64(body)}:${url}`;
|
||||||
startUrl?: string,
|
|
||||||
): { pages: PageRef[]; urlToId: Map<string, string> } {
|
|
||||||
const urls = collectUrls(steps, startUrl);
|
|
||||||
const urlToId = new Map<string, string>();
|
|
||||||
const pages = urls.map((url, index) => {
|
|
||||||
const id = `p${index + 1}`;
|
|
||||||
urlToId.set(url, id);
|
|
||||||
return { id, url };
|
|
||||||
});
|
|
||||||
return { pages, urlToId };
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function pageIdFor(
|
/** Register or reuse a page observation in the states dictionary. */
|
||||||
url: string | undefined,
|
export function registerObservation(
|
||||||
urlToId: Map<string, string>,
|
registry: Map<string, StateRegistryEntry>,
|
||||||
fallbackUrl?: string,
|
input: {
|
||||||
|
url: string;
|
||||||
|
title?: string;
|
||||||
|
rawVomText: string;
|
||||||
|
truncated?: boolean;
|
||||||
|
},
|
||||||
): string {
|
): string {
|
||||||
if (url && urlToId.has(url)) return urlToId.get(url)!;
|
const key = stateDedupKey(input.rawVomText, input.url);
|
||||||
if (fallbackUrl && urlToId.has(fallbackUrl)) return urlToId.get(fallbackUrl)!;
|
for (const entry of registry.values()) {
|
||||||
return urlToId.values().next().value ?? "p1";
|
if (entry.contentHash === key) return entry.id;
|
||||||
|
}
|
||||||
|
const id = nextStateId();
|
||||||
|
registry.set(id, {
|
||||||
|
id,
|
||||||
|
url: input.url,
|
||||||
|
...(input.title ? { title: input.title } : {}),
|
||||||
|
truncated: input.truncated ?? false,
|
||||||
|
contentHash: key,
|
||||||
|
rawVomText: input.rawVomText,
|
||||||
|
stepsHere: [],
|
||||||
|
});
|
||||||
|
return id;
|
||||||
}
|
}
|
||||||
|
|
||||||
function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined {
|
function toSelection(
|
||||||
if (step.op === "navigate") return step.page_url ?? step.url;
|
values: string[],
|
||||||
if ("page_url" in step && step.page_url) return step.page_url;
|
labels?: string[],
|
||||||
return fallbackUrl;
|
): Array<{ value: string; label?: string }> {
|
||||||
}
|
|
||||||
|
|
||||||
function effectForNavigation(
|
|
||||||
navigatedTo: string | undefined,
|
|
||||||
urlToId: Map<string, string>,
|
|
||||||
): Step["effect"] {
|
|
||||||
if (!navigatedTo) return undefined;
|
|
||||||
const pageId = urlToId.get(navigatedTo);
|
|
||||||
if (!pageId) return undefined;
|
|
||||||
return { navigated_to: pageId };
|
|
||||||
}
|
|
||||||
|
|
||||||
function withEffect(step: Step, effect: Step["effect"]): Step {
|
|
||||||
if (!effect) return step;
|
|
||||||
return { ...step, effect };
|
|
||||||
}
|
|
||||||
|
|
||||||
function toSelection(values: string[], labels?: string[]): SelectedOption[] {
|
|
||||||
return values.map((value, index) => ({
|
return values.map((value, index) => ({
|
||||||
value,
|
value,
|
||||||
...(labels?.[index] ? { label: labels[index] } : {}),
|
...(labels?.[index] ? { label: labels[index] } : {}),
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
function toV2Step(
|
function navigateCauseForDraft(
|
||||||
step: DraftTraceStep,
|
draft: Extract<DraftTraceStep, { op: "navigate" }>,
|
||||||
id: number,
|
): NavigationCause {
|
||||||
urlToId: Map<string, string>,
|
if (draft.cause) return draft.cause;
|
||||||
fallbackUrl?: string,
|
const mapped = mapNavigationCause({
|
||||||
): Step | null {
|
transitionType: draft.transitionType,
|
||||||
if (!shouldIncludeDraft(step)) return null;
|
transitionQualifiers: draft.transitionQualifiers,
|
||||||
|
});
|
||||||
|
return mapped ?? "browser";
|
||||||
|
}
|
||||||
|
|
||||||
const pageUrl = pageUrlForDraft(step, fallbackUrl);
|
function toV3Step(draft: DraftTraceStep, id: number): Step | null {
|
||||||
const page = pageIdFor(pageUrl, urlToId, fallbackUrl);
|
if (!shouldIncludeDraft(draft)) return null;
|
||||||
|
// An action the user really performed must not disappear because one of its
|
||||||
|
// observations is missing — a capture racing a document swap is routine, and
|
||||||
|
// the last step of a recording is the most exposed to it. Fall back to the
|
||||||
|
// endpoint we do know; only a step with no observation at all has nothing to
|
||||||
|
// point at.
|
||||||
|
const preState = draft.preStateId ?? draft.postStateId;
|
||||||
|
const postState = draft.postStateId ?? draft.preStateId;
|
||||||
|
if (!preState || !postState) return null;
|
||||||
|
|
||||||
switch (step.op) {
|
const common = { id, state: preState, result: { state: postState } };
|
||||||
|
|
||||||
|
switch (draft.op) {
|
||||||
case "navigate":
|
case "navigate":
|
||||||
return {
|
return {
|
||||||
op: "navigate",
|
op: "navigate",
|
||||||
id,
|
...common,
|
||||||
page: pageIdFor(step.url, urlToId, fallbackUrl),
|
to: draft.url,
|
||||||
to: step.url,
|
cause: navigateCauseForDraft(draft),
|
||||||
};
|
};
|
||||||
case "click":
|
case "click":
|
||||||
return withEffect(
|
return { op: "click", ...common, target: draft.target };
|
||||||
{
|
|
||||||
op: "click",
|
|
||||||
id,
|
|
||||||
page,
|
|
||||||
target: step.target,
|
|
||||||
},
|
|
||||||
effectForNavigation(step.navigated_to, urlToId),
|
|
||||||
);
|
|
||||||
case "hover":
|
case "hover":
|
||||||
return {
|
return { op: "hover", ...common, target: draft.target };
|
||||||
op: "hover",
|
|
||||||
id,
|
|
||||||
page,
|
|
||||||
target: step.target,
|
|
||||||
};
|
|
||||||
case "fill":
|
case "fill":
|
||||||
return {
|
return {
|
||||||
op: "fill",
|
op: "fill",
|
||||||
id,
|
...common,
|
||||||
page,
|
target: draft.target,
|
||||||
target: step.target,
|
value: draft.value,
|
||||||
value: step.value,
|
commit: draft.commit ?? DEFAULT_FILL_COMMIT,
|
||||||
...(step.redacted ? { redacted: true } : {}),
|
...(draft.redacted ? { redacted: true } : {}),
|
||||||
};
|
};
|
||||||
case "press":
|
case "press":
|
||||||
return withEffect(
|
return {
|
||||||
{
|
op: "press",
|
||||||
op: "press",
|
...common,
|
||||||
id,
|
key: draft.key,
|
||||||
page,
|
...(draft.target ? { target: draft.target } : {}),
|
||||||
key: step.key,
|
...(draft.modifiers?.length ? { modifiers: draft.modifiers } : {}),
|
||||||
...(step.target ? { target: step.target } : {}),
|
};
|
||||||
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
|
|
||||||
},
|
|
||||||
effectForNavigation(step.navigated_to, urlToId),
|
|
||||||
);
|
|
||||||
case "select":
|
case "select":
|
||||||
return withEffect(
|
return {
|
||||||
{
|
op: "select",
|
||||||
op: "select",
|
...common,
|
||||||
id,
|
target: draft.target,
|
||||||
page,
|
selection: toSelection(draft.values, draft.labels),
|
||||||
target: step.target,
|
};
|
||||||
selection: toSelection(step.values, step.labels),
|
case "scroll":
|
||||||
},
|
return { op: "scroll", ...common };
|
||||||
effectForNavigation(step.navigated_to, urlToId),
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface ReducedTrace {
|
export interface ReducedTrace {
|
||||||
pages: PageRef[];
|
states: TraceState[];
|
||||||
steps: Step[];
|
steps: Step[];
|
||||||
|
/**
|
||||||
|
* Capture-order draft id (`draftIndex + 1`) → published step id. Collapsed
|
||||||
|
* and dropped drafts make the two diverge, so anything the recorder tagged
|
||||||
|
* with a draft id (`steps_here`, inline annotations) must be remapped.
|
||||||
|
*/
|
||||||
|
stepIdByDraftId: Map<number, number>;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Compile capture drafts into record-only trace v2 steps.
|
* Compile capture drafts into record-only trace v3 steps.
|
||||||
* Variable inputs are NOT classified here — executing agents infer that at run time.
|
* Drafts must already carry `preStateId` / `postStateId` from the recorder.
|
||||||
*/
|
*/
|
||||||
export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): ReducedTrace {
|
export function reduceTraceSteps(
|
||||||
|
steps: DraftTraceStep[],
|
||||||
|
registry: Map<string, StateRegistryEntry>,
|
||||||
|
): ReducedTrace {
|
||||||
const collapsed = collapseNavigations(steps);
|
const collapsed = collapseNavigations(steps);
|
||||||
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
|
|
||||||
const out: Step[] = [];
|
const out: Step[] = [];
|
||||||
|
const stepIdByDraftId = new Map<number, number>();
|
||||||
let id = 1;
|
let id = 1;
|
||||||
let lastUrl = startUrl;
|
for (const { draft, draftIds } of collapsed) {
|
||||||
for (const draft of collapsed) {
|
const step = toV3Step(draft, id);
|
||||||
if (draft.op === "navigate") lastUrl = draft.url;
|
|
||||||
else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to;
|
|
||||||
else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url;
|
|
||||||
const step = toV2Step(draft, id, urlToId, lastUrl);
|
|
||||||
if (!step) continue;
|
if (!step) continue;
|
||||||
out.push(step);
|
out.push(step);
|
||||||
|
for (const draftId of draftIds) stepIdByDraftId.set(draftId, id);
|
||||||
id += 1;
|
id += 1;
|
||||||
}
|
}
|
||||||
return { pages, steps: out };
|
const states = [...registry.values()].map(
|
||||||
|
({ contentHash: _hash, rawVomText: _body, stepsHere: _steps, ...state }) => state,
|
||||||
|
);
|
||||||
|
return { states, steps: out, stepIdByDraftId };
|
||||||
}
|
}
|
||||||
|
|
||||||
export function resolveTraceStartUrl(
|
export function resolveTraceStartUrl(
|
||||||
drafts: DraftTraceStep[],
|
drafts: DraftTraceStep[],
|
||||||
startUrl?: string,
|
startUrl?: string,
|
||||||
pages?: PageRef[],
|
states?: TraceState[],
|
||||||
): string {
|
): string {
|
||||||
if (startUrl) return startUrl;
|
if (startUrl) return startUrl;
|
||||||
const navigate = drafts.find((step): step is Extract<DraftTraceStep, { op: "navigate" }> => {
|
const navigate = drafts.find((step): step is Extract<DraftTraceStep, { op: "navigate" }> => {
|
||||||
@@ -221,5 +228,9 @@ export function resolveTraceStartUrl(
|
|||||||
for (const draft of drafts) {
|
for (const draft of drafts) {
|
||||||
if ("page_url" in draft && draft.page_url) return draft.page_url;
|
if ("page_url" in draft && draft.page_url) return draft.page_url;
|
||||||
}
|
}
|
||||||
return pages?.[0]?.url ?? "about:blank";
|
return states?.[0]?.url ?? "about:blank";
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export type { FillCommit, NavigationCause };
|
||||||
|
/** Test seam */
|
||||||
|
export { resetStateIdCounterForTests };
|
||||||
|
|||||||
@@ -0,0 +1,86 @@
|
|||||||
|
import { describe, expect, it } from "vitest";
|
||||||
|
import { RECORD_DOCUMENT_TOKEN_ATTR } from "@/lib/record-document-token";
|
||||||
|
import type { CapturedNode } from "@/tools/vom/capture";
|
||||||
|
import { collectDocumentTokenOwners } from "../capture-vom-observation";
|
||||||
|
|
||||||
|
function htmlNode(
|
||||||
|
backendNodeId: number,
|
||||||
|
token: string,
|
||||||
|
ownerFrameBackendNodeId: number | null = null,
|
||||||
|
): CapturedNode {
|
||||||
|
return {
|
||||||
|
backendNodeId,
|
||||||
|
parentBackendNodeId: null,
|
||||||
|
tag: "html",
|
||||||
|
attrs: { [RECORD_DOCUMENT_TOKEN_ATTR]: token },
|
||||||
|
rect: { x: 0, y: 0, w: 1280, h: 720 },
|
||||||
|
paintOrder: 0,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
...(ownerFrameBackendNodeId === null ? {} : { ownerFrameBackendNodeId }),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
describe("collectDocumentTokenOwners", () => {
|
||||||
|
it("maps document tokens to their immediate iframe owner backend node", () => {
|
||||||
|
const topToken = "top-doc-token";
|
||||||
|
const childToken = "child-doc-token";
|
||||||
|
const owners = collectDocumentTokenOwners({
|
||||||
|
viewport: { width: 1280, height: 720 },
|
||||||
|
nodes: [htmlNode(1, topToken, null)],
|
||||||
|
iframeNodes: new Map([
|
||||||
|
[
|
||||||
|
13,
|
||||||
|
[
|
||||||
|
htmlNode(101, childToken, 13),
|
||||||
|
{
|
||||||
|
backendNodeId: 102,
|
||||||
|
parentBackendNodeId: 101,
|
||||||
|
tag: "input",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 10, y: 20, w: 200, h: 30 },
|
||||||
|
paintOrder: 1,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
ownerFrameBackendNodeId: 13,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
]),
|
||||||
|
excludedBackendNodeIds: new Set(),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(owners.get(topToken)).toBeNull();
|
||||||
|
expect(owners.get(childToken)).toBe(13);
|
||||||
|
});
|
||||||
|
|
||||||
|
it("maps nested iframe documents to their direct owner, not the outer iframe", () => {
|
||||||
|
const nestedToken = "nested-doc-token";
|
||||||
|
const owners = collectDocumentTokenOwners({
|
||||||
|
viewport: { width: 1280, height: 720 },
|
||||||
|
nodes: [htmlNode(1, "top-token", null)],
|
||||||
|
iframeNodes: new Map([
|
||||||
|
[
|
||||||
|
13,
|
||||||
|
[
|
||||||
|
{
|
||||||
|
backendNodeId: 23,
|
||||||
|
parentBackendNodeId: 101,
|
||||||
|
tag: "iframe",
|
||||||
|
attrs: {},
|
||||||
|
rect: { x: 0, y: 0, w: 400, h: 300 },
|
||||||
|
paintOrder: 1,
|
||||||
|
position: "static",
|
||||||
|
pointerEvents: "auto",
|
||||||
|
ownerFrameBackendNodeId: 13,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
],
|
||||||
|
[23, [htmlNode(201, nestedToken, 23)]],
|
||||||
|
]),
|
||||||
|
excludedBackendNodeIds: new Set(),
|
||||||
|
});
|
||||||
|
|
||||||
|
expect(owners.get(nestedToken)).toBe(23);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -1034,7 +1034,7 @@ describe("buildVomScene", () => {
|
|||||||
expect(rendered).toContain(' @e2 textbox "验证码"');
|
expect(rendered).toContain(' @e2 textbox "验证码"');
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not mark non-password inputs sensitive from password-like labels", () => {
|
it("marks current-password autocomplete as sensitive even for text inputs", () => {
|
||||||
const axNodes: CdpAxNode[] = [
|
const axNodes: CdpAxNode[] = [
|
||||||
{
|
{
|
||||||
nodeId: "1",
|
nodeId: "1",
|
||||||
@@ -1069,7 +1069,7 @@ describe("buildVomScene", () => {
|
|||||||
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
|
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
|
||||||
expect.objectContaining({
|
expect.objectContaining({
|
||||||
id: 10,
|
id: 10,
|
||||||
sensitive: false,
|
sensitive: true,
|
||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
});
|
});
|
||||||
@@ -1364,7 +1364,9 @@ describe("buildVomScene", () => {
|
|||||||
);
|
);
|
||||||
const rendered = renderVom(scene);
|
const rendered = renderVom(scene);
|
||||||
expect(rendered.text).toContain('@e1 button "close"');
|
expect(rendered.text).toContain('@e1 button "close"');
|
||||||
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
|
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
|
||||||
|
{ ref: "e1", backendNodeId: 20 },
|
||||||
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not promote a clickable container that wraps a real interactive control", () => {
|
it("does not promote a clickable container that wraps a real interactive control", () => {
|
||||||
@@ -1508,7 +1510,9 @@ describe("buildVomScene", () => {
|
|||||||
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic");
|
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic");
|
||||||
const rendered = renderVom(scene);
|
const rendered = renderVom(scene);
|
||||||
expect(rendered.text).toContain('@e1 button "收藏"');
|
expect(rendered.text).toContain('@e1 button "收藏"');
|
||||||
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
|
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
|
||||||
|
{ ref: "e1", backendNodeId: 20 },
|
||||||
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("builds active scope blocks from active aria-controls relationships", () => {
|
it("builds active scope blocks from active aria-controls relationships", () => {
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,124 @@
|
|||||||
|
import { type RenderedRef, renderVom, type VomOptions } from "@browser-skill/vom";
|
||||||
|
import { readRecordDocumentTokenFromAttrs } from "@/lib/record-document-token";
|
||||||
|
import type { CdpAxNode } from "./observation";
|
||||||
|
import { buildVomScene } from "./observation";
|
||||||
|
import type { CdpRunner } from "./shared";
|
||||||
|
import {
|
||||||
|
type CapturedNode,
|
||||||
|
type CapturedSurfaceProbe,
|
||||||
|
type CapturedViewModel,
|
||||||
|
captureViewModel,
|
||||||
|
collectOverlayExcludedBackendIds,
|
||||||
|
} from "./vom/capture";
|
||||||
|
|
||||||
|
export interface CaptureVomObservationOptions extends VomOptions {
|
||||||
|
conditionalSurfaceProbe?: boolean;
|
||||||
|
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CaptureVomObservationResult {
|
||||||
|
text: string;
|
||||||
|
refs: RenderedRef[];
|
||||||
|
truncated: boolean;
|
||||||
|
captured: CapturedNode[];
|
||||||
|
/** All nodes including iframe sub-documents, for geometric target matching. */
|
||||||
|
matchNodes: CapturedNode[];
|
||||||
|
/** Maps ephemeral record document tokens to immediate iframe owner backend ids. */
|
||||||
|
documentTokenOwners: Map<string, number | null>;
|
||||||
|
surfaceProbes?: CapturedSurfaceProbe[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function flattenCapturedNodes(vm: CapturedViewModel): CapturedNode[] {
|
||||||
|
return [...vm.nodes, ...[...vm.iframeNodes.values()].flat()];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function collectDocumentTokenOwners(vm: CapturedViewModel): Map<string, number | null> {
|
||||||
|
const owners = new Map<string, number | null>();
|
||||||
|
for (const node of flattenCapturedNodes(vm)) {
|
||||||
|
if (node.tag !== "html") continue;
|
||||||
|
const token = readRecordDocumentTokenFromAttrs(node.attrs);
|
||||||
|
if (!token) continue;
|
||||||
|
owners.set(token, node.ownerFrameBackendNodeId ?? null);
|
||||||
|
}
|
||||||
|
return owners;
|
||||||
|
}
|
||||||
|
|
||||||
|
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
|
||||||
|
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
|
||||||
|
}
|
||||||
|
|
||||||
|
interface LayoutMetricsViewportReply {
|
||||||
|
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||||
|
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function fallbackCapturedViewModel(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
): Promise<CapturedViewModel> {
|
||||||
|
let viewport = { width: 0, height: 0 };
|
||||||
|
try {
|
||||||
|
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
|
||||||
|
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||||
|
viewport = {
|
||||||
|
width: vpSrc.clientWidth ?? 0,
|
||||||
|
height: vpSrc.clientHeight ?? 0,
|
||||||
|
};
|
||||||
|
} catch {
|
||||||
|
// viewport stays zero-sized
|
||||||
|
}
|
||||||
|
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId);
|
||||||
|
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
|
||||||
|
}
|
||||||
|
|
||||||
|
async function captureForVom(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
conditionalSurfaceProbe: boolean,
|
||||||
|
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>,
|
||||||
|
): Promise<CapturedViewModel> {
|
||||||
|
try {
|
||||||
|
return await captureViewModel(cdp, tabId, {
|
||||||
|
conditionalSurfaceProbe,
|
||||||
|
hoverProbeBypassOverlay,
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
return fallbackCapturedViewModel(cdp, tabId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Pure VOM capture pipeline shared by snapshot, observe, and recording. */
|
||||||
|
export async function captureVomObservation(
|
||||||
|
cdp: CdpRunner,
|
||||||
|
tabId: number,
|
||||||
|
url: string | undefined,
|
||||||
|
opts: CaptureVomObservationOptions = {},
|
||||||
|
): Promise<CaptureVomObservationResult> {
|
||||||
|
await cdp.ensureAttachedToUrl?.(tabId, url);
|
||||||
|
await cdp.send<unknown>(tabId, "Accessibility.enable", {});
|
||||||
|
const result = await cdp.send<{ nodes: CdpAxNode[] }>(tabId, "Accessibility.getFullAXTree", {});
|
||||||
|
const axNodes = result.nodes ?? [];
|
||||||
|
const captured = await captureForVom(
|
||||||
|
cdp,
|
||||||
|
tabId,
|
||||||
|
opts.conditionalSurfaceProbe ?? false,
|
||||||
|
opts.hoverProbeBypassOverlay,
|
||||||
|
);
|
||||||
|
const scene = buildVomScene(axNodes, captured, { pageUrl: url });
|
||||||
|
const rendered = renderVom(scene, {
|
||||||
|
maxDepth: opts.maxDepth,
|
||||||
|
maxTokens: opts.maxTokens,
|
||||||
|
redactValues: opts.redactValues,
|
||||||
|
activeRegionPolicy: opts.activeRegionPolicy ?? true,
|
||||||
|
});
|
||||||
|
const matchNodes = flattenCapturedNodes(captured);
|
||||||
|
return {
|
||||||
|
text: rendered.text,
|
||||||
|
refs: rendered.refs,
|
||||||
|
truncated: rendered.truncated,
|
||||||
|
captured: captured.nodes,
|
||||||
|
matchNodes,
|
||||||
|
documentTokenOwners: collectDocumentTokenOwners(captured),
|
||||||
|
surfaceProbes: captured.surfaceProbes,
|
||||||
|
};
|
||||||
|
}
|
||||||
@@ -8,7 +8,6 @@ import {
|
|||||||
applyVomInteractionRecovery,
|
applyVomInteractionRecovery,
|
||||||
type CondSurface,
|
type CondSurface,
|
||||||
isVomReferenceNode,
|
isVomReferenceNode,
|
||||||
renderVom,
|
|
||||||
type VomNode,
|
type VomNode,
|
||||||
type VomScene,
|
type VomScene,
|
||||||
} from "@browser-skill/vom";
|
} from "@browser-skill/vom";
|
||||||
@@ -29,6 +28,7 @@ import type {
|
|||||||
SnapshotParams,
|
SnapshotParams,
|
||||||
SnapshotResult,
|
SnapshotResult,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
|
import { captureVomObservation } from "./capture-vom-observation";
|
||||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||||
import { nodeBoundingRect, scrollNodeIntoView } from "./element-geometry";
|
import { nodeBoundingRect, scrollNodeIntoView } from "./element-geometry";
|
||||||
import { rpcError } from "./errors";
|
import { rpcError } from "./errors";
|
||||||
@@ -44,12 +44,7 @@ import {
|
|||||||
type ToolEffect,
|
type ToolEffect,
|
||||||
} from "./shared";
|
} from "./shared";
|
||||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||||
import {
|
import { type CapturedNode, type CapturedViewModel } from "./vom/capture";
|
||||||
type CapturedNode,
|
|
||||||
type CapturedViewModel,
|
|
||||||
captureViewModel,
|
|
||||||
collectOverlayExcludedBackendIds,
|
|
||||||
} from "./vom/capture";
|
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Shared helpers (legacy aliases — observation.ts kept exporting these
|
// Shared helpers (legacy aliases — observation.ts kept exporting these
|
||||||
@@ -350,7 +345,18 @@ function isModalSignal(
|
|||||||
|
|
||||||
function isSensitive(capturedNode: CapturedNode | undefined): boolean {
|
function isSensitive(capturedNode: CapturedNode | undefined): boolean {
|
||||||
const attrs = capturedNode?.attrs ?? {};
|
const attrs = capturedNode?.attrs ?? {};
|
||||||
return (attrs.type ?? "").toLowerCase() === "password";
|
const type = (attrs.type ?? "").toLowerCase();
|
||||||
|
if (type === "password") return true;
|
||||||
|
const autocomplete = (attrs.autocomplete ?? "").toLowerCase();
|
||||||
|
if (autocomplete.startsWith("cc-")) return true;
|
||||||
|
if (
|
||||||
|
autocomplete === "one-time-code" ||
|
||||||
|
autocomplete === "current-password" ||
|
||||||
|
autocomplete === "new-password"
|
||||||
|
) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface AxNodeSignals {
|
interface AxNodeSignals {
|
||||||
@@ -400,7 +406,13 @@ const AX_TEXT_STOP_ROLES = new Set([
|
|||||||
"columnheader",
|
"columnheader",
|
||||||
"rowheader",
|
"rowheader",
|
||||||
]);
|
]);
|
||||||
const SENSITIVE_AX_INPUT_TYPES = new Set(["password", "credit-card"]);
|
const SENSITIVE_AX_INPUT_TYPES = new Set([
|
||||||
|
"password",
|
||||||
|
"credit-card",
|
||||||
|
"one-time-code",
|
||||||
|
"current-password",
|
||||||
|
"new-password",
|
||||||
|
]);
|
||||||
|
|
||||||
function axPropertyString(axNode: CdpAxNode, name: string): string | undefined {
|
function axPropertyString(axNode: CdpAxNode, name: string): string | undefined {
|
||||||
const prop = axNode.properties?.find((item) => item.name === name);
|
const prop = axNode.properties?.find((item) => item.name === name);
|
||||||
@@ -1314,50 +1326,6 @@ export async function handleGetHtml(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
|
|
||||||
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
|
|
||||||
}
|
|
||||||
|
|
||||||
interface LayoutMetricsViewportReply {
|
|
||||||
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
|
||||||
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
|
||||||
}
|
|
||||||
|
|
||||||
async function fallbackCapturedViewModel(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
tabId: number,
|
|
||||||
): Promise<CapturedViewModel> {
|
|
||||||
let viewport = { width: 0, height: 0 };
|
|
||||||
try {
|
|
||||||
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
|
|
||||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
|
||||||
viewport = {
|
|
||||||
width: vpSrc.clientWidth ?? 0,
|
|
||||||
height: vpSrc.clientHeight ?? 0,
|
|
||||||
};
|
|
||||||
} catch {
|
|
||||||
// viewport stays zero-sized
|
|
||||||
}
|
|
||||||
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId);
|
|
||||||
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
|
|
||||||
}
|
|
||||||
|
|
||||||
async function captureForVom(
|
|
||||||
cdp: CdpRunner,
|
|
||||||
tabId: number,
|
|
||||||
conditionalSurfaceProbe: boolean,
|
|
||||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>,
|
|
||||||
): Promise<CapturedViewModel> {
|
|
||||||
try {
|
|
||||||
return await captureViewModel(cdp, tabId, {
|
|
||||||
conditionalSurfaceProbe,
|
|
||||||
hoverProbeBypassOverlay,
|
|
||||||
});
|
|
||||||
} catch {
|
|
||||||
return fallbackCapturedViewModel(cdp, tabId);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
async function handleVomObservation(
|
async function handleVomObservation(
|
||||||
manager: SessionManager,
|
manager: SessionManager,
|
||||||
params: SnapshotParams | ObserveParams,
|
params: SnapshotParams | ObserveParams,
|
||||||
@@ -1383,27 +1351,14 @@ async function handleVomObservation(
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
|
||||||
await deps.cdp.send<unknown>(target.tabId, "Accessibility.enable", {});
|
|
||||||
const result = await deps.cdp.send<{ nodes: CdpAxNode[] }>(
|
|
||||||
target.tabId,
|
|
||||||
"Accessibility.getFullAXTree",
|
|
||||||
{},
|
|
||||||
);
|
|
||||||
const axNodes = result.nodes ?? [];
|
|
||||||
const effectiveConditionalSurfaceProbe =
|
const effectiveConditionalSurfaceProbe =
|
||||||
deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe;
|
deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe;
|
||||||
const captured = await captureForVom(
|
const rendered = await captureVomObservation(deps.cdp, target.tabId, target.url, {
|
||||||
deps.cdp,
|
|
||||||
target.tabId,
|
|
||||||
effectiveConditionalSurfaceProbe,
|
|
||||||
deps.hoverProbeBypassOverlay,
|
|
||||||
);
|
|
||||||
const scene = buildVomScene(axNodes, captured, { pageUrl: target.url });
|
|
||||||
const rendered = renderVom(scene, {
|
|
||||||
maxDepth: params.max_depth,
|
maxDepth: params.max_depth,
|
||||||
maxTokens: params.max_tokens,
|
maxTokens: params.max_tokens,
|
||||||
activeRegionPolicy: true,
|
activeRegionPolicy: true,
|
||||||
|
conditionalSurfaceProbe: effectiveConditionalSurfaceProbe,
|
||||||
|
hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay,
|
||||||
});
|
});
|
||||||
ctx.refStore.replace(
|
ctx.refStore.replace(
|
||||||
rendered.refs.map(
|
rendered.refs.map(
|
||||||
@@ -1418,7 +1373,7 @@ async function handleVomObservation(
|
|||||||
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
|
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
|
||||||
? {
|
? {
|
||||||
debug: {
|
debug: {
|
||||||
surface_probes: (captured.surfaceProbes ?? []).map((probe) => ({
|
surface_probes: (rendered.surfaceProbes ?? []).map((probe) => ({
|
||||||
trigger_backend_node_id: probe.triggerBackendNodeId,
|
trigger_backend_node_id: probe.triggerBackendNodeId,
|
||||||
...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}),
|
...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}),
|
||||||
trigger_action: probe.triggerAction,
|
trigger_action: probe.triggerAction,
|
||||||
|
|||||||
@@ -7,21 +7,53 @@ import {
|
|||||||
isRecordQueryMessage,
|
isRecordQueryMessage,
|
||||||
isRecordStepMessage,
|
isRecordStepMessage,
|
||||||
RECORD_CANCEL,
|
RECORD_CANCEL,
|
||||||
|
RECORD_CAPTURE_STATUS,
|
||||||
RECORD_START,
|
RECORD_START,
|
||||||
RECORD_STEP,
|
RECORD_STEP,
|
||||||
RECORD_STOP,
|
RECORD_STOP,
|
||||||
type RecordCancelMessage,
|
type RecordCancelMessage,
|
||||||
|
type RecordCaptureStatusMessage,
|
||||||
type RecordFinishMessage,
|
type RecordFinishMessage,
|
||||||
type RecordQueryResponse,
|
type RecordQueryResponse,
|
||||||
type RecordStartAck,
|
type RecordStartAck,
|
||||||
type RecordStartMessage,
|
type RecordStartMessage,
|
||||||
|
type RecordStepPayload,
|
||||||
type RecordStopMessage,
|
type RecordStopMessage,
|
||||||
} from "@/lib/record-bridge";
|
} from "@/lib/record-bridge";
|
||||||
|
import {
|
||||||
|
armAllDocumentsForTab,
|
||||||
|
armDocumentCapture,
|
||||||
|
cancelDocumentCapture,
|
||||||
|
type ArmedDocument,
|
||||||
|
listInjectableFrames,
|
||||||
|
type RecordFrameSendDeps,
|
||||||
|
stopDocumentCapture,
|
||||||
|
} from "@/lib/record-frame-arming";
|
||||||
|
import { DEFAULT_MAX_PAGE_TOKENS } from "@/lib/record-constants";
|
||||||
|
import {
|
||||||
|
applyTargetMatching,
|
||||||
|
buildTraceV3,
|
||||||
|
cancelPendingSettles,
|
||||||
|
captureAndRegisterObservation,
|
||||||
|
clearPendingRedirectLanding,
|
||||||
|
createObservationState,
|
||||||
|
flushPendingRedirectLanding,
|
||||||
|
flushPendingSettles,
|
||||||
|
inferMissingPostStates,
|
||||||
|
type RecordingObservationState,
|
||||||
|
rememberStepOnPage,
|
||||||
|
scheduleDraftSettle,
|
||||||
|
scheduleRedirectLandingFlush,
|
||||||
|
settleUnsettledDrafts,
|
||||||
|
} from "@/lib/record-observation";
|
||||||
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording-step-buffer";
|
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording-step-buffer";
|
||||||
import { reduceTraceSteps, resolveTraceStartUrl } from "@/lib/trace-reducer";
|
|
||||||
import type { SessionManager } from "@/session-manager/manager";
|
import type { SessionManager } from "@/session-manager/manager";
|
||||||
|
import { EXTENSION_VERSION } from "@/transport/handshake";
|
||||||
import type {
|
import type {
|
||||||
DraftTraceStep,
|
DraftTraceStep,
|
||||||
|
FrameCaptureFailure,
|
||||||
|
FrameCaptureInfo,
|
||||||
|
FrameCaptureStatus,
|
||||||
RecordAwaitParams,
|
RecordAwaitParams,
|
||||||
RecordAwaitResult,
|
RecordAwaitResult,
|
||||||
RecordStartParams,
|
RecordStartParams,
|
||||||
@@ -29,6 +61,7 @@ import type {
|
|||||||
RecordStopParams,
|
RecordStopParams,
|
||||||
RecordStopResult,
|
RecordStopResult,
|
||||||
RpcError,
|
RpcError,
|
||||||
|
StopReason,
|
||||||
Trace,
|
Trace,
|
||||||
} from "@/transport/types";
|
} from "@/transport/types";
|
||||||
import { handleNavigate } from "./navigation";
|
import { handleNavigate } from "./navigation";
|
||||||
@@ -58,6 +91,37 @@ interface ActiveRecording {
|
|||||||
currentUrl?: string;
|
currentUrl?: string;
|
||||||
pendingNavigation: boolean;
|
pendingNavigation: boolean;
|
||||||
pendingNavigationDeadline?: number;
|
pendingNavigationDeadline?: number;
|
||||||
|
observation: RecordingObservationState;
|
||||||
|
stoppedBy: StopReason;
|
||||||
|
maxPageTokens: number;
|
||||||
|
redactValues: boolean;
|
||||||
|
/** Documents whose capture acknowledged START and still need STOP. */
|
||||||
|
armedDocuments: Map<string, ArmedDocument>;
|
||||||
|
frameCaptureFailures: FrameCaptureFailure[];
|
||||||
|
frameCaptureStatus: FrameCaptureStatus;
|
||||||
|
/** When false, ignore subframe rearm requests during finish. */
|
||||||
|
acceptingNewFrames: boolean;
|
||||||
|
/** Rearms already sending START when finish begins. */
|
||||||
|
rearmCallbacks: Set<Promise<void>>;
|
||||||
|
/** Navigation callbacks tracked from event receipt through action enqueue. */
|
||||||
|
navigationCallbacks: Set<Promise<void>>;
|
||||||
|
/** Synchronous intake gate closed only after finish drains to stability. */
|
||||||
|
acceptingNavigation: boolean;
|
||||||
|
/**
|
||||||
|
* Serializes step appends with navigation observation so a click is always
|
||||||
|
* in `steps` before a same-turn `webNavigation` tries to annotate it.
|
||||||
|
*/
|
||||||
|
actionQueue: Promise<void>;
|
||||||
|
/**
|
||||||
|
* CDP runner captured at `record_start`. The message listeners are attached
|
||||||
|
* once at service-worker startup, where no runner exists yet, so observation
|
||||||
|
* capture must go through the recording instead of the listener deps.
|
||||||
|
*/
|
||||||
|
cdp?: CdpRunner;
|
||||||
|
}
|
||||||
|
|
||||||
|
function enqueueRecordingAction(recording: ActiveRecording, task: () => Promise<void>): void {
|
||||||
|
recording.actionQueue = recording.actionQueue.then(task, task).catch(() => {});
|
||||||
}
|
}
|
||||||
|
|
||||||
const recordings = new Map<string, ActiveRecording>();
|
const recordings = new Map<string, ActiveRecording>();
|
||||||
@@ -78,6 +142,9 @@ function sleep(ms: number): Promise<void> {
|
|||||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Recording producer version mirrored into trace.recorder.bsk. */
|
||||||
|
export const BSK_TRACE_VERSION = EXTENSION_VERSION;
|
||||||
|
|
||||||
/** Injectable http(s) landing page when `tool.record_start` omits `url`. */
|
/** Injectable http(s) landing page when `tool.record_start` omits `url`. */
|
||||||
export const RECORD_DEFAULT_START_URL = "https://example.com/";
|
export const RECORD_DEFAULT_START_URL = "https://example.com/";
|
||||||
|
|
||||||
@@ -155,16 +222,95 @@ async function sendRecordStartWithAck(
|
|||||||
}
|
}
|
||||||
|
|
||||||
function buildTrace(recording: ActiveRecording): Trace {
|
function buildTrace(recording: ActiveRecording): Trace {
|
||||||
const { pages, steps } = reduceTraceSteps(recording.steps, recording.startUrl);
|
const frameCapture: FrameCaptureInfo | undefined =
|
||||||
const startUrl = resolveTraceStartUrl(recording.steps, recording.startUrl, pages);
|
recording.frameCaptureFailures.length > 0 || recording.frameCaptureStatus === "partial"
|
||||||
return {
|
? {
|
||||||
|
status: recording.frameCaptureStatus,
|
||||||
|
...(recording.frameCaptureFailures.length > 0
|
||||||
|
? { failures: recording.frameCaptureFailures }
|
||||||
|
: {}),
|
||||||
|
}
|
||||||
|
: undefined;
|
||||||
|
return buildTraceV3({
|
||||||
|
obs: recording.observation,
|
||||||
|
steps: recording.steps,
|
||||||
|
startedAt: recording.startedAt,
|
||||||
...(recording.purpose ? { purpose: recording.purpose } : {}),
|
...(recording.purpose ? { purpose: recording.purpose } : {}),
|
||||||
recorded_at: new Date().toISOString(),
|
startUrl: recording.startUrl,
|
||||||
started_at: recording.startedAt,
|
stoppedBy: recording.stoppedBy,
|
||||||
entry: { start_url: startUrl },
|
bskVersion: BSK_TRACE_VERSION,
|
||||||
pages,
|
...(frameCapture ? { frameCapture } : {}),
|
||||||
steps,
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function recordFrameFailure(recording: ActiveRecording, failure: FrameCaptureFailure): void {
|
||||||
|
recording.frameCaptureFailures.push(failure);
|
||||||
|
recording.frameCaptureStatus = "partial";
|
||||||
|
}
|
||||||
|
|
||||||
|
function enrichRecordedStep(
|
||||||
|
recording: ActiveRecording,
|
||||||
|
step: RecordStepPayload,
|
||||||
|
): RecordStepPayload {
|
||||||
|
const enriched: RecordStepPayload = { ...step };
|
||||||
|
if (enriched.geometry && enriched.documentToken) {
|
||||||
|
const owner = recording.observation.documentTokenOwners.get(enriched.documentToken);
|
||||||
|
if (owner !== undefined) {
|
||||||
|
enriched.geometry = { ...enriched.geometry, ownerFrameBackendNodeId: owner };
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!enriched.page_url && recording.currentUrl) {
|
||||||
|
enriched.page_url = recording.currentUrl;
|
||||||
|
}
|
||||||
|
return enriched;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function publishCaptureStatus(
|
||||||
|
recording: ActiveRecording,
|
||||||
|
deps: RecordDeps,
|
||||||
|
): Promise<void> {
|
||||||
|
if (!deps.broadcastCaptureStatus) return;
|
||||||
|
const msg: RecordCaptureStatusMessage = {
|
||||||
|
type: RECORD_CAPTURE_STATUS,
|
||||||
|
status: recording.frameCaptureStatus,
|
||||||
|
...(recording.frameCaptureFailures.length > 0
|
||||||
|
? { failures: recording.frameCaptureFailures }
|
||||||
|
: {}),
|
||||||
};
|
};
|
||||||
|
try {
|
||||||
|
await deps.broadcastCaptureStatus(recording.tabId, msg);
|
||||||
|
} catch {
|
||||||
|
// Overlay is best-effort.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function processRecordedStep(
|
||||||
|
recording: ActiveRecording,
|
||||||
|
deps: RecordDeps,
|
||||||
|
draftIndex: number,
|
||||||
|
): Promise<void> {
|
||||||
|
const cdp = recording.cdp ?? deps.cdp;
|
||||||
|
if (!cdp) return;
|
||||||
|
const draft = recording.steps[draftIndex];
|
||||||
|
if (!draft) return;
|
||||||
|
const stepId = draftIndex + 1;
|
||||||
|
|
||||||
|
if (draft.op === "navigate") {
|
||||||
|
draft.preStateId = recording.observation.lastSettled?.stateId;
|
||||||
|
} else {
|
||||||
|
applyTargetMatching(recording.observation, draft, stepId);
|
||||||
|
}
|
||||||
|
rememberStepOnPage(recording.observation, draft, stepId);
|
||||||
|
|
||||||
|
scheduleDraftSettle(
|
||||||
|
recording.observation,
|
||||||
|
draftIndex,
|
||||||
|
stepId,
|
||||||
|
cdp,
|
||||||
|
recording.tabId,
|
||||||
|
deps.tabsApi,
|
||||||
|
recording.steps,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RecordDeps {
|
export interface RecordDeps {
|
||||||
@@ -173,17 +319,67 @@ export interface RecordDeps {
|
|||||||
tabId: number,
|
tabId: number,
|
||||||
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
|
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
|
||||||
): Promise<unknown>;
|
): Promise<unknown>;
|
||||||
|
sendToDocument?(
|
||||||
|
tabId: number,
|
||||||
|
documentId: string,
|
||||||
|
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
|
||||||
|
): Promise<unknown>;
|
||||||
|
getAllFrames?(tabId: number): Promise<
|
||||||
|
Array<{
|
||||||
|
frameId: number;
|
||||||
|
documentId?: string;
|
||||||
|
url?: string;
|
||||||
|
parentFrameId: number;
|
||||||
|
}>
|
||||||
|
>;
|
||||||
|
broadcastCaptureStatus?(
|
||||||
|
tabId: number,
|
||||||
|
msg: RecordCaptureStatusMessage,
|
||||||
|
): Promise<void>;
|
||||||
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||||
cdp?: CdpRunner;
|
cdp?: CdpRunner;
|
||||||
signal?: AbortSignal;
|
signal?: AbortSignal;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function frameSendDeps(deps: RecordDeps): RecordFrameSendDeps {
|
||||||
|
return {
|
||||||
|
sendToDocument: (tabId, documentId, msg) => {
|
||||||
|
if (deps.sendToDocument) return deps.sendToDocument(tabId, documentId, msg);
|
||||||
|
return deps.sendToTab(tabId, msg);
|
||||||
|
},
|
||||||
|
getAllFrames: async (tabId) => {
|
||||||
|
if (deps.getAllFrames) return deps.getAllFrames(tabId);
|
||||||
|
if (!chrome.webNavigation?.getAllFrames) return [];
|
||||||
|
const frames = await chrome.webNavigation.getAllFrames({ tabId });
|
||||||
|
return (frames ?? []).map((frame) => ({
|
||||||
|
frameId: frame.frameId,
|
||||||
|
documentId: frame.documentId,
|
||||||
|
url: frame.url,
|
||||||
|
parentFrameId: frame.parentFrameId,
|
||||||
|
}));
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
let defaultDeps: RecordDeps | null = null;
|
let defaultDeps: RecordDeps | null = null;
|
||||||
function getDefaultDeps(): RecordDeps {
|
function getDefaultDeps(): RecordDeps {
|
||||||
if (!defaultDeps) {
|
if (!defaultDeps) {
|
||||||
defaultDeps = {
|
defaultDeps = {
|
||||||
tabsApi: chromeTabsApi,
|
tabsApi: chromeTabsApi,
|
||||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||||
|
sendToDocument: (tabId, documentId, msg) =>
|
||||||
|
chrome.tabs.sendMessage(tabId, msg, { documentId }),
|
||||||
|
getAllFrames: async (tabId) => {
|
||||||
|
if (!chrome.webNavigation?.getAllFrames) return [];
|
||||||
|
const frames = await chrome.webNavigation.getAllFrames({ tabId });
|
||||||
|
return (frames ?? []).map((frame) => ({
|
||||||
|
frameId: frame.frameId,
|
||||||
|
documentId: frame.documentId,
|
||||||
|
url: frame.url,
|
||||||
|
parentFrameId: frame.parentFrameId,
|
||||||
|
}));
|
||||||
|
},
|
||||||
|
broadcastCaptureStatus: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg, { frameId: 0 }),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
return defaultDeps;
|
return defaultDeps;
|
||||||
@@ -241,16 +437,29 @@ export function releaseBrowserObservationListenersIfIdle(): void {
|
|||||||
detachBrowserObservation = null;
|
detachBrowserObservation = null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export function attachRecordStepListener(): () => void {
|
export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||||
const listener = (
|
const listener = (
|
||||||
message: unknown,
|
message: unknown,
|
||||||
_sender: chrome.runtime.MessageSender,
|
sender: chrome.runtime.MessageSender,
|
||||||
_sendResponse: () => void,
|
_sendResponse: () => void,
|
||||||
) => {
|
) => {
|
||||||
if (!isRecordStepMessage(message)) return false;
|
if (!isRecordStepMessage(message)) return false;
|
||||||
for (const recording of recordings.values()) {
|
for (const recording of recordings.values()) {
|
||||||
if (recording.requestId !== message.requestId) continue;
|
if (recording.requestId !== message.requestId) continue;
|
||||||
appendRecordedPayload(recording, message.step);
|
enqueueRecordingAction(recording, async () => {
|
||||||
|
const cdp = recording.cdp ?? deps.cdp;
|
||||||
|
await flushPendingRedirectLanding(
|
||||||
|
recording.observation,
|
||||||
|
recording.steps,
|
||||||
|
cdp,
|
||||||
|
recording.tabId,
|
||||||
|
deps.tabsApi,
|
||||||
|
{ cancelled: () => recording.settled },
|
||||||
|
);
|
||||||
|
const enriched = enrichRecordedStep(recording, message.step);
|
||||||
|
const draftIndex = appendRecordedPayload(recording, enriched);
|
||||||
|
if (draftIndex !== null) await processRecordedStep(recording, deps, draftIndex);
|
||||||
|
});
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
return false;
|
return false;
|
||||||
@@ -277,7 +486,11 @@ export function attachRecordFinishListener(deps: RecordDeps = getDefaultDeps()):
|
|||||||
|
|
||||||
function findRecordingByTabId(tabId: number): ActiveRecording | null {
|
function findRecordingByTabId(tabId: number): ActiveRecording | null {
|
||||||
for (const recording of recordings.values()) {
|
for (const recording of recordings.values()) {
|
||||||
if (recording.tabId === tabId && !recording.settled) return recording;
|
if (recording.settled) continue;
|
||||||
|
if (recording.tabId === tabId) return recording;
|
||||||
|
for (const armed of recording.armedDocuments.values()) {
|
||||||
|
if (armed.tabId === tabId) return recording;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
@@ -325,71 +538,135 @@ async function clearRearmTimersForRecording(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function stopRecordingOnAllAgentTabs(
|
async function stopRecordingOnAllDocuments(
|
||||||
recording: ActiveRecording,
|
recording: ActiveRecording,
|
||||||
deps: RecordDeps,
|
deps: RecordDeps,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
|
recording.acceptingNewFrames = false;
|
||||||
const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
|
const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
|
||||||
let tabIds = [recording.tabId];
|
const frameDeps = frameSendDeps(deps);
|
||||||
try {
|
const armed = [...recording.armedDocuments.values()];
|
||||||
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
|
for (const doc of armed) {
|
||||||
tabIds = [
|
const result = await stopDocumentCapture(doc, stopMsg, frameDeps);
|
||||||
...new Set([
|
if ("ok" in result) {
|
||||||
recording.tabId,
|
recording.armedDocuments.delete(doc.documentId);
|
||||||
...tabs.flatMap((tab) => (typeof tab.id === "number" ? [tab.id] : [])),
|
continue;
|
||||||
]),
|
|
||||||
];
|
|
||||||
} catch {
|
|
||||||
// Fall back to the current recording tab.
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const tabId of tabIds) {
|
|
||||||
try {
|
|
||||||
const response = await deps.sendToTab(tabId, stopMsg);
|
|
||||||
if (tabId === recording.tabId && !isRecordStartAck(response)) {
|
|
||||||
throw new Error("content script did not confirm recorded steps");
|
|
||||||
}
|
|
||||||
} catch {
|
|
||||||
if (tabId === recording.tabId) {
|
|
||||||
throw new Error("failed to flush recorded steps");
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (deps.bypassOverlay) {
|
recordFrameFailure(recording, result);
|
||||||
try {
|
}
|
||||||
await deps.bypassOverlay(tabId, false);
|
if (recording.armedDocuments.size > 0) {
|
||||||
} catch {
|
for (const doc of [...recording.armedDocuments.values()]) {
|
||||||
// Best-effort cleanup.
|
recordFrameFailure(
|
||||||
}
|
recording,
|
||||||
|
{
|
||||||
|
reason: "flush_failed",
|
||||||
|
frame_id: doc.frameId,
|
||||||
|
document_id: doc.documentId,
|
||||||
|
url: doc.url,
|
||||||
|
detail: "document still armed after STOP",
|
||||||
|
},
|
||||||
|
);
|
||||||
|
}
|
||||||
|
recording.armedDocuments.clear();
|
||||||
|
}
|
||||||
|
await publishCaptureStatus(recording, deps);
|
||||||
|
if (deps.bypassOverlay) {
|
||||||
|
try {
|
||||||
|
await deps.bypassOverlay(recording.tabId, false);
|
||||||
|
} catch {
|
||||||
|
// Best-effort cleanup.
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function rearmRecording(
|
async function cancelRecordingOnAllDocuments(
|
||||||
|
recording: ActiveRecording,
|
||||||
|
deps: RecordDeps,
|
||||||
|
): Promise<void> {
|
||||||
|
const cancelMsg: RecordCancelMessage = { type: RECORD_CANCEL, requestId: recording.requestId };
|
||||||
|
const frameDeps = frameSendDeps(deps);
|
||||||
|
for (const doc of [...recording.armedDocuments.values()]) {
|
||||||
|
await cancelDocumentCapture(doc, cancelMsg, frameDeps);
|
||||||
|
}
|
||||||
|
recording.armedDocuments.clear();
|
||||||
|
}
|
||||||
|
|
||||||
|
async function rearmRecordingDocuments(
|
||||||
recording: ActiveRecording,
|
recording: ActiveRecording,
|
||||||
targetTabId: number,
|
targetTabId: number,
|
||||||
deps: RecordDeps,
|
deps: RecordDeps,
|
||||||
|
targetDocumentId?: string,
|
||||||
): Promise<boolean> {
|
): Promise<boolean> {
|
||||||
// Do NOT toggle automation-bypass here: each retry used to increment the
|
let resolveTracked!: () => void;
|
||||||
// content-script counter, and a single stop decrement left the ControlOverlay
|
const tracked = new Promise<void>((resolve) => {
|
||||||
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
|
resolveTracked = resolve;
|
||||||
// already hides the control chrome while activeRecord is set.
|
});
|
||||||
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
|
recording.rearmCallbacks.add(tracked);
|
||||||
const startMsg: RecordStartMessage = {
|
|
||||||
type: RECORD_START,
|
const frameDeps = frameSendDeps(deps);
|
||||||
requestId: recording.requestId,
|
try {
|
||||||
startedAtMs: recording.startedAtMs,
|
const attemptLimit = targetDocumentId ? 1 : RECORD_REARM_MAX_ATTEMPTS;
|
||||||
};
|
for (let attempt = 0; attempt < attemptLimit; attempt += 1) {
|
||||||
try {
|
if (recording.settled || recording.finishing || !recording.acceptingNewFrames) return false;
|
||||||
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
|
const { injectable, failures } = await listInjectableFrames(targetTabId, frameDeps);
|
||||||
recording.tabId = targetTabId;
|
for (const failure of failures) recordFrameFailure(recording, failure);
|
||||||
return true;
|
const targets = targetDocumentId
|
||||||
} catch {
|
? injectable.filter((frame) => frame.documentId === targetDocumentId)
|
||||||
|
: injectable.filter((frame) => !recording.armedDocuments.has(frame.documentId ?? ""));
|
||||||
|
if (targets.length === 0) {
|
||||||
|
if (targetDocumentId) {
|
||||||
|
recordFrameFailure(recording, {
|
||||||
|
reason: "rearm_failed",
|
||||||
|
frame_id: 0,
|
||||||
|
document_id: targetDocumentId,
|
||||||
|
detail: "document not injectable",
|
||||||
|
});
|
||||||
|
await publishCaptureStatus(recording, deps);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
recording.tabId = targetTabId;
|
||||||
|
return injectable.some((frame) => recording.armedDocuments.has(frame.documentId ?? ""));
|
||||||
|
}
|
||||||
|
|
||||||
|
let armedAny = false;
|
||||||
|
for (const frame of targets) {
|
||||||
|
const startMsg: RecordStartMessage = {
|
||||||
|
type: RECORD_START,
|
||||||
|
requestId: recording.requestId,
|
||||||
|
startedAtMs: recording.startedAtMs,
|
||||||
|
showOverlay: frame.frameId === 0,
|
||||||
|
frameMode: frame.frameId === 0 ? "top" : "child",
|
||||||
|
};
|
||||||
|
const result = await armDocumentCapture(targetTabId, frame, startMsg, frameDeps);
|
||||||
|
if (result.armed) {
|
||||||
|
recording.armedDocuments.set(result.armed.documentId, result.armed);
|
||||||
|
armedAny = true;
|
||||||
|
}
|
||||||
|
if (result.failure) recordFrameFailure(recording, result.failure);
|
||||||
|
}
|
||||||
|
if (armedAny) {
|
||||||
|
recording.tabId = targetTabId;
|
||||||
|
await publishCaptureStatus(recording, deps);
|
||||||
|
if (deps.cdp && recording.acceptingNewFrames) {
|
||||||
|
void captureAndRegisterObservation(
|
||||||
|
recording.observation,
|
||||||
|
deps.cdp,
|
||||||
|
targetTabId,
|
||||||
|
deps.tabsApi,
|
||||||
|
).catch(() => {});
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
|
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
|
||||||
await sleep(RECORD_REARM_RETRY_DELAY_MS);
|
await sleep(RECORD_REARM_RETRY_DELAY_MS);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
await publishCaptureStatus(recording, deps);
|
||||||
|
return false;
|
||||||
|
} finally {
|
||||||
|
recording.rearmCallbacks.delete(tracked);
|
||||||
|
resolveTracked();
|
||||||
}
|
}
|
||||||
return false;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
|
function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
|
||||||
@@ -401,7 +678,7 @@ function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
|
|||||||
rearmTimers.delete(tabId);
|
rearmTimers.delete(tabId);
|
||||||
void (async () => {
|
void (async () => {
|
||||||
const current = await findRecordingForTab(tabId, deps);
|
const current = await findRecordingForTab(tabId, deps);
|
||||||
if (current) await rearmRecording(current, tabId, deps);
|
if (current) await rearmRecordingDocuments(current, tabId, deps);
|
||||||
})();
|
})();
|
||||||
}, RECORD_REARM_DEBOUNCE_MS),
|
}, RECORD_REARM_DEBOUNCE_MS),
|
||||||
);
|
);
|
||||||
@@ -432,11 +709,86 @@ export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): ()
|
|||||||
}
|
}
|
||||||
|
|
||||||
export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||||
const observeMainFrame = (tabId: number, url?: string, causedByAction?: boolean) => {
|
const observeMainFrame = (
|
||||||
|
tabId: number,
|
||||||
|
url?: string,
|
||||||
|
causedByAction?: boolean,
|
||||||
|
transitionType?: string,
|
||||||
|
transitionQualifiers?: string[],
|
||||||
|
) => {
|
||||||
|
if (!url) return;
|
||||||
|
const direct = findRecordingByTabId(tabId);
|
||||||
|
const candidates = direct
|
||||||
|
? direct.acceptingNavigation
|
||||||
|
? [direct]
|
||||||
|
: []
|
||||||
|
: [...recordings.values()].filter(
|
||||||
|
(recording) => !recording.settled && recording.acceptingNavigation,
|
||||||
|
);
|
||||||
|
if (candidates.length === 0) return;
|
||||||
|
|
||||||
|
let resolveTracked!: () => void;
|
||||||
|
const tracked = new Promise<void>((resolve) => {
|
||||||
|
resolveTracked = resolve;
|
||||||
|
});
|
||||||
|
for (const candidate of candidates) candidate.navigationCallbacks.add(tracked);
|
||||||
|
|
||||||
void (async () => {
|
void (async () => {
|
||||||
const recording = await findRecordingForTab(tabId, deps);
|
try {
|
||||||
if (recording && url) {
|
const recording = await findRecordingForTab(tabId, deps);
|
||||||
observeRecordedNavigation(recording, url, causedByAction);
|
if (!recording || !recording.acceptingNavigation || !candidates.includes(recording)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
enqueueRecordingAction(recording, async () => {
|
||||||
|
const result = observeRecordedNavigation(
|
||||||
|
recording,
|
||||||
|
url,
|
||||||
|
causedByAction,
|
||||||
|
transitionType,
|
||||||
|
transitionQualifiers,
|
||||||
|
);
|
||||||
|
const cdp = recording.cdp ?? deps.cdp;
|
||||||
|
if (!cdp) return;
|
||||||
|
|
||||||
|
if (result.kind === "coalesce_redirect") {
|
||||||
|
scheduleRedirectLandingFlush(
|
||||||
|
recording.observation,
|
||||||
|
recording.steps,
|
||||||
|
cdp,
|
||||||
|
tabId,
|
||||||
|
deps.tabsApi,
|
||||||
|
result.url,
|
||||||
|
{ cancelled: () => recording.settled },
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (result.kind === "noop") return;
|
||||||
|
|
||||||
|
// A concrete navigation supersedes any in-flight redirect coalesce.
|
||||||
|
clearPendingRedirectLanding(recording.observation);
|
||||||
|
|
||||||
|
const draftIndex = result.index;
|
||||||
|
const lastDraft = recording.steps[draftIndex];
|
||||||
|
if (!lastDraft) return;
|
||||||
|
|
||||||
|
if (result.kind === "appended") {
|
||||||
|
lastDraft.preStateId = recording.observation.lastSettled?.stateId;
|
||||||
|
rememberStepOnPage(recording.observation, lastDraft, draftIndex + 1);
|
||||||
|
}
|
||||||
|
scheduleDraftSettle(
|
||||||
|
recording.observation,
|
||||||
|
draftIndex,
|
||||||
|
draftIndex + 1,
|
||||||
|
cdp,
|
||||||
|
tabId,
|
||||||
|
deps.tabsApi,
|
||||||
|
recording.steps,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
for (const candidate of candidates) candidate.navigationCallbacks.delete(tracked);
|
||||||
|
resolveTracked();
|
||||||
}
|
}
|
||||||
})();
|
})();
|
||||||
};
|
};
|
||||||
@@ -447,29 +799,36 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
|
|||||||
})();
|
})();
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const onSubFrameComplete = (tabId: number, documentId?: string) => {
|
||||||
|
if (!documentId) return;
|
||||||
|
void (async () => {
|
||||||
|
const recording = await findRecordingForTab(tabId, deps);
|
||||||
|
if (!recording || !recording.acceptingNewFrames) return;
|
||||||
|
await rearmRecordingDocuments(recording, tabId, deps, documentId);
|
||||||
|
})();
|
||||||
|
};
|
||||||
|
|
||||||
if (chrome.webNavigation?.onCompleted) {
|
if (chrome.webNavigation?.onCompleted) {
|
||||||
const completedListener = (
|
const completedListener = (
|
||||||
details: chrome.webNavigation.WebNavigationFramedCallbackDetails,
|
details: chrome.webNavigation.WebNavigationFramedCallbackDetails,
|
||||||
) => {
|
) => {
|
||||||
if (details.frameId !== 0) return;
|
if (details.frameId === 0) {
|
||||||
onMainFrameComplete(details.tabId, details.url);
|
onMainFrameComplete(details.tabId, details.url);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
onSubFrameComplete(details.tabId, details.documentId);
|
||||||
};
|
};
|
||||||
const committedListener = (
|
const committedListener = (
|
||||||
details: chrome.webNavigation.WebNavigationTransitionCallbackDetails,
|
details: chrome.webNavigation.WebNavigationTransitionCallbackDetails,
|
||||||
) => {
|
) => {
|
||||||
if (details.frameId !== 0) return;
|
if (details.frameId !== 0) return;
|
||||||
const causedByAction =
|
observeMainFrame(
|
||||||
details.transitionType === "link" || details.transitionType === "form_submit"
|
details.tabId,
|
||||||
? true
|
details.url,
|
||||||
: details.transitionType === "typed" ||
|
undefined,
|
||||||
details.transitionType === "auto_bookmark" ||
|
details.transitionType,
|
||||||
details.transitionType === "generated" ||
|
details.transitionQualifiers,
|
||||||
details.transitionType === "keyword" ||
|
);
|
||||||
details.transitionType === "keyword_generated" ||
|
|
||||||
details.transitionType === "reload"
|
|
||||||
? false
|
|
||||||
: undefined;
|
|
||||||
observeMainFrame(details.tabId, details.url, causedByAction);
|
|
||||||
};
|
};
|
||||||
chrome.webNavigation.onCompleted.addListener(completedListener);
|
chrome.webNavigation.onCompleted.addListener(completedListener);
|
||||||
chrome.webNavigation.onCommitted?.addListener(committedListener);
|
chrome.webNavigation.onCommitted?.addListener(committedListener);
|
||||||
@@ -487,6 +846,46 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
|
|||||||
return () => chrome.tabs.onUpdated.removeListener(listener);
|
return () => chrome.tabs.onUpdated.removeListener(listener);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const MAX_FINISH_DRAIN_ROUNDS = 10;
|
||||||
|
|
||||||
|
async function drainRecordingToStability(
|
||||||
|
recording: ActiveRecording,
|
||||||
|
deps: RecordDeps,
|
||||||
|
): Promise<boolean> {
|
||||||
|
const cdp = recording.cdp ?? deps.cdp;
|
||||||
|
for (let round = 0; round < MAX_FINISH_DRAIN_ROUNDS; round += 1) {
|
||||||
|
await Promise.all([...recording.navigationCallbacks]);
|
||||||
|
const actionTail = recording.actionQueue;
|
||||||
|
await actionTail;
|
||||||
|
await flushPendingRedirectLanding(
|
||||||
|
recording.observation,
|
||||||
|
recording.steps,
|
||||||
|
cdp,
|
||||||
|
recording.tabId,
|
||||||
|
deps.tabsApi,
|
||||||
|
{ cancelled: () => recording.settled },
|
||||||
|
);
|
||||||
|
await flushPendingSettles(recording.observation);
|
||||||
|
const redirectTail = recording.observation.redirectFlushQueue;
|
||||||
|
const settleTail = recording.observation.settleQueue;
|
||||||
|
await Promise.all([redirectTail, settleTail]);
|
||||||
|
|
||||||
|
if (
|
||||||
|
recording.navigationCallbacks.size === 0 &&
|
||||||
|
recording.actionQueue === actionTail &&
|
||||||
|
recording.observation.redirectFlushQueue === redirectTail &&
|
||||||
|
recording.observation.settleQueue === settleTail
|
||||||
|
) {
|
||||||
|
// No event or promise continuation can interleave with this synchronous
|
||||||
|
// check-and-close, so work accepted before the cutoff is fully drained.
|
||||||
|
recording.acceptingNavigation = false;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
console.warn("[bsk record] navigation/action queues did not stabilize at stop");
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||||
const listener = (
|
const listener = (
|
||||||
message: unknown,
|
message: unknown,
|
||||||
@@ -505,11 +904,15 @@ export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()):
|
|||||||
sendResponse({ active: false });
|
sendResponse({ active: false });
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
await rearmRecording(recording, tabId, deps);
|
await rearmRecordingDocuments(recording, tabId, deps, sender.documentId);
|
||||||
sendResponse({
|
sendResponse({
|
||||||
active: true,
|
active: true,
|
||||||
requestId: recording.requestId,
|
requestId: recording.requestId,
|
||||||
startedAtMs: recording.startedAtMs,
|
startedAtMs: recording.startedAtMs,
|
||||||
|
captureStatus: recording.frameCaptureStatus,
|
||||||
|
...(recording.frameCaptureFailures.length > 0
|
||||||
|
? { captureFailures: recording.frameCaptureFailures }
|
||||||
|
: {}),
|
||||||
});
|
});
|
||||||
})();
|
})();
|
||||||
return true;
|
return true;
|
||||||
@@ -527,23 +930,46 @@ async function finishRecordingByRequest(
|
|||||||
if (recording.requestId !== requestId || recording.settled) continue;
|
if (recording.requestId !== requestId || recording.settled) continue;
|
||||||
const match = await findRecordingForTab(tabId, deps);
|
const match = await findRecordingForTab(tabId, deps);
|
||||||
if (match !== recording) continue;
|
if (match !== recording) continue;
|
||||||
await finishRecording(sessionId, deps);
|
await finishRecording(sessionId, deps, "user_finish");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async function finishRecording(sessionId: string, deps: RecordDeps): Promise<Trace | null> {
|
async function finishRecording(
|
||||||
|
sessionId: string,
|
||||||
|
deps: RecordDeps,
|
||||||
|
stoppedBy: StopReason,
|
||||||
|
): Promise<Trace | null> {
|
||||||
const recording = recordings.get(sessionId);
|
const recording = recordings.get(sessionId);
|
||||||
if (!recording || recording.settled || recording.finishing) return null;
|
if (!recording || recording.settled) return null;
|
||||||
|
if (recording.finishing) return recording.finishPromise;
|
||||||
recording.finishing = true;
|
recording.finishing = true;
|
||||||
|
recording.stoppedBy = stoppedBy;
|
||||||
|
|
||||||
await clearRearmTimersForRecording(recording, deps);
|
await clearRearmTimersForRecording(recording, deps);
|
||||||
|
await Promise.all([...recording.rearmCallbacks]);
|
||||||
try {
|
try {
|
||||||
await stopRecordingOnAllAgentTabs(recording, deps);
|
await stopRecordingOnAllDocuments(recording, deps);
|
||||||
} catch {
|
} catch {
|
||||||
recording.finishing = false;
|
recording.finishing = false;
|
||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
if (!(await drainRecordingToStability(recording, deps))) {
|
||||||
|
recording.finishing = false;
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
cancelPendingSettles(recording.observation);
|
||||||
|
inferMissingPostStates(recording.steps);
|
||||||
|
const cdp = recording.cdp ?? deps.cdp;
|
||||||
|
if (cdp) {
|
||||||
|
await settleUnsettledDrafts(
|
||||||
|
recording.observation,
|
||||||
|
cdp,
|
||||||
|
recording.tabId,
|
||||||
|
deps.tabsApi,
|
||||||
|
recording.steps,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
recording.settled = true;
|
recording.settled = true;
|
||||||
recordings.delete(sessionId);
|
recordings.delete(sessionId);
|
||||||
@@ -582,6 +1008,8 @@ export async function handleRecordStart(
|
|||||||
});
|
});
|
||||||
const navigateUrl = params.url ?? RECORD_DEFAULT_START_URL;
|
const navigateUrl = params.url ?? RECORD_DEFAULT_START_URL;
|
||||||
const startedAtMs = Date.now();
|
const startedAtMs = Date.now();
|
||||||
|
const maxPageTokens = params.max_page_tokens;
|
||||||
|
const redactValues = params.redact_values ?? false;
|
||||||
recordings.set(params.session_id, {
|
recordings.set(params.session_id, {
|
||||||
requestId,
|
requestId,
|
||||||
tabId: target.tabId,
|
tabId: target.tabId,
|
||||||
@@ -599,23 +1027,30 @@ export async function handleRecordStart(
|
|||||||
currentUrl: navigateUrl,
|
currentUrl: navigateUrl,
|
||||||
pendingNavigation: false,
|
pendingNavigation: false,
|
||||||
pendingNavigationDeadline: undefined,
|
pendingNavigationDeadline: undefined,
|
||||||
|
observation: createObservationState({ maxPageTokens, redactValues }),
|
||||||
|
stoppedBy: "user_finish",
|
||||||
|
maxPageTokens: maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS,
|
||||||
|
redactValues,
|
||||||
|
armedDocuments: new Map(),
|
||||||
|
frameCaptureFailures: [],
|
||||||
|
frameCaptureStatus: "complete",
|
||||||
|
acceptingNewFrames: true,
|
||||||
|
rearmCallbacks: new Set(),
|
||||||
|
navigationCallbacks: new Set(),
|
||||||
|
acceptingNavigation: true,
|
||||||
|
actionQueue: Promise.resolve(),
|
||||||
|
...(deps.cdp ? { cdp: deps.cdp } : {}),
|
||||||
});
|
});
|
||||||
// Observe navigations for the whole recording lifetime; attach before
|
// Observe navigations for the whole recording lifetime; attach before
|
||||||
// optional navigate so the destination load can rearm capture.
|
// optional navigate so the destination load can rearm capture.
|
||||||
ensureBrowserObservationListeners(deps);
|
ensureBrowserObservationListeners(deps);
|
||||||
|
|
||||||
const abortPending = async (notifyContent: boolean) => {
|
const abortPending = async (notifyContent: boolean) => {
|
||||||
|
const pending = recordings.get(params.session_id);
|
||||||
recordings.delete(params.session_id);
|
recordings.delete(params.session_id);
|
||||||
releaseBrowserObservationListenersIfIdle();
|
releaseBrowserObservationListenersIfIdle();
|
||||||
if (notifyContent) {
|
if (notifyContent && pending) {
|
||||||
try {
|
await cancelRecordingOnAllDocuments(pending, deps);
|
||||||
await deps.sendToTab(target.tabId, {
|
|
||||||
type: RECORD_CANCEL,
|
|
||||||
requestId,
|
|
||||||
});
|
|
||||||
} catch {
|
|
||||||
// Content may never have received RECORD_START.
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
if (deps.bypassOverlay) {
|
if (deps.bypassOverlay) {
|
||||||
try {
|
try {
|
||||||
@@ -718,10 +1153,28 @@ export async function handleRecordStart(
|
|||||||
if (cancelled) return cancelled;
|
if (cancelled) return cancelled;
|
||||||
}
|
}
|
||||||
|
|
||||||
const startMsg: RecordStartMessage = { type: RECORD_START, requestId, startedAtMs };
|
const startMsg: RecordStartMessage = {
|
||||||
|
type: RECORD_START,
|
||||||
|
requestId,
|
||||||
|
startedAtMs,
|
||||||
|
showOverlay: true,
|
||||||
|
frameMode: "top",
|
||||||
|
};
|
||||||
|
|
||||||
try {
|
try {
|
||||||
await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
|
const frameDeps = frameSendDeps(deps);
|
||||||
|
const { armed, failures } = await armAllDocumentsForTab(target.tabId, startMsg, frameDeps);
|
||||||
|
for (const failure of failures) recordFrameFailure(active, failure);
|
||||||
|
for (const doc of armed) active.armedDocuments.set(doc.documentId, doc);
|
||||||
|
if (active.armedDocuments.size === 0) {
|
||||||
|
await abortPending(true);
|
||||||
|
return {
|
||||||
|
code: "protocol_error",
|
||||||
|
message:
|
||||||
|
"failed to start recording in any frame — reload the BrowserSkill extension, then retry",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
await publishCaptureStatus(active, deps);
|
||||||
} catch {
|
} catch {
|
||||||
await abortPending(true);
|
await abortPending(true);
|
||||||
return {
|
return {
|
||||||
@@ -731,6 +1184,23 @@ export async function handleRecordStart(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (deps.cdp) {
|
||||||
|
const activeRecording = recordings.get(params.session_id);
|
||||||
|
if (activeRecording) {
|
||||||
|
try {
|
||||||
|
await captureAndRegisterObservation(
|
||||||
|
activeRecording.observation,
|
||||||
|
deps.cdp,
|
||||||
|
target.tabId,
|
||||||
|
deps.tabsApi,
|
||||||
|
startUrl,
|
||||||
|
);
|
||||||
|
} catch {
|
||||||
|
// Proceed without initial observation; steps may be dropped by reducer.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
{
|
{
|
||||||
const cancelled = await abortIfCancelled(true);
|
const cancelled = await abortIfCancelled(true);
|
||||||
if (cancelled) return cancelled;
|
if (cancelled) return cancelled;
|
||||||
@@ -755,7 +1225,7 @@ export async function handleRecordStop(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const trace = await finishRecording(params.session_id, deps);
|
const trace = await finishRecording(params.session_id, deps, "cli_stop");
|
||||||
if (!trace) {
|
if (!trace) {
|
||||||
return {
|
return {
|
||||||
code: "protocol_error",
|
code: "protocol_error",
|
||||||
|
|||||||
@@ -677,7 +677,7 @@ describe("captureViewModel", () => {
|
|||||||
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
|
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("subtracts scroll offset to make rects viewport-relative", async () => {
|
it("preserves document rects while deriving viewport rects after scroll", async () => {
|
||||||
const cdp = {
|
const cdp = {
|
||||||
send: vi.fn(async (_t: number, method: string) => {
|
send: vi.fn(async (_t: number, method: string) => {
|
||||||
if (method === "DOMSnapshot.enable") return {};
|
if (method === "DOMSnapshot.enable") return {};
|
||||||
@@ -691,11 +691,10 @@ describe("captureViewModel", () => {
|
|||||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||||
};
|
};
|
||||||
const { nodes } = await captureViewModel(cdp, 4);
|
const { nodes } = await captureViewModel(cdp, 4);
|
||||||
const div = nodes.find((n) => n.backendNodeId === 12);
|
const input = nodes.find((n) => n.backendNodeId === 13);
|
||||||
// local rect preserves viewport-relative scroll math.
|
expect(input?.documentRect).toEqual({ x: 400, y: 300, w: 200, h: 40 });
|
||||||
expect(div?.localRect?.y).toBe(-200);
|
expect(input?.localRect).toEqual({ x: 400, y: 100, w: 200, h: 40 });
|
||||||
// exported rect is clipped to the top-level viewport before VOM consumes it.
|
expect(input?.rect).toEqual({ x: 400, y: 100, w: 200, h: 40 });
|
||||||
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
|
|
||||||
});
|
});
|
||||||
|
|
||||||
it("normalizes device-pixel bounds by devicePixelRatio", async () => {
|
it("normalizes device-pixel bounds by devicePixelRatio", async () => {
|
||||||
|
|||||||
@@ -25,6 +25,8 @@ export interface CapturedNode {
|
|||||||
rect: Rect | null;
|
rect: Rect | null;
|
||||||
/** Frame-local viewport-relative CSS px before top-level translation/clipping. */
|
/** Frame-local viewport-relative CSS px before top-level translation/clipping. */
|
||||||
localRect?: Rect | null;
|
localRect?: Rect | null;
|
||||||
|
/** Frame-local document-relative CSS px as reported by DOMSnapshot. */
|
||||||
|
documentRect?: Rect | null;
|
||||||
paintOrder: number;
|
paintOrder: number;
|
||||||
position: string;
|
position: string;
|
||||||
pointerEvents: string;
|
pointerEvents: string;
|
||||||
@@ -788,6 +790,7 @@ function parseDocumentNodes(
|
|||||||
|
|
||||||
let rect: Rect | null = null;
|
let rect: Rect | null = null;
|
||||||
let localRect: Rect | null = null;
|
let localRect: Rect | null = null;
|
||||||
|
let documentRect: Rect | null = null;
|
||||||
let paintOrder = 0;
|
let paintOrder = 0;
|
||||||
let position = "static";
|
let position = "static";
|
||||||
let pointerEvents = "auto";
|
let pointerEvents = "auto";
|
||||||
@@ -796,12 +799,18 @@ function parseDocumentNodes(
|
|||||||
if (li !== undefined) {
|
if (li !== undefined) {
|
||||||
const b = dl?.bounds?.[li];
|
const b = dl?.bounds?.[li];
|
||||||
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
|
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
|
||||||
localRect = {
|
documentRect = {
|
||||||
x: b[0] / dpr - context.scrollX,
|
x: b[0] / dpr,
|
||||||
y: b[1] / dpr - context.scrollY,
|
y: b[1] / dpr,
|
||||||
w: b[2] / dpr,
|
w: b[2] / dpr,
|
||||||
h: b[3] / dpr,
|
h: b[3] / dpr,
|
||||||
};
|
};
|
||||||
|
localRect = {
|
||||||
|
x: documentRect.x - context.scrollX,
|
||||||
|
y: documentRect.y - context.scrollY,
|
||||||
|
w: documentRect.w,
|
||||||
|
h: documentRect.h,
|
||||||
|
};
|
||||||
rect = rectInTop(localRect, context);
|
rect = rectInTop(localRect, context);
|
||||||
}
|
}
|
||||||
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
||||||
@@ -825,6 +834,7 @@ function parseDocumentNodes(
|
|||||||
attrs,
|
attrs,
|
||||||
rect,
|
rect,
|
||||||
localRect,
|
localRect,
|
||||||
|
documentRect,
|
||||||
paintOrder,
|
paintOrder,
|
||||||
position,
|
position,
|
||||||
pointerEvents,
|
pointerEvents,
|
||||||
|
|||||||
@@ -65,6 +65,11 @@ function deferredFakeTransport(): { transport: Transport; emit: (frame: Protocol
|
|||||||
}
|
}
|
||||||
|
|
||||||
describe("performHandshake", () => {
|
describe("performHandshake", () => {
|
||||||
|
it("advertises the Trace v3 compatibility boundary", () => {
|
||||||
|
expect(PROTOCOL_VERSION).toBe("1.1");
|
||||||
|
expect(MIN_COMPATIBLE_PROTOCOL).toBe("1.1");
|
||||||
|
});
|
||||||
|
|
||||||
it("sends system.handshake with identity and both compat fields", async () => {
|
it("sends system.handshake with identity and both compat fields", async () => {
|
||||||
let sentFrame: ProtocolFrame | null = null;
|
let sentFrame: ProtocolFrame | null = null;
|
||||||
const transport = fakeTransport((req) => {
|
const transport = fakeTransport((req) => {
|
||||||
@@ -74,9 +79,9 @@ describe("performHandshake", () => {
|
|||||||
result: {
|
result: {
|
||||||
server: "browser-skill-daemon",
|
server: "browser-skill-daemon",
|
||||||
version: "0.1.0",
|
version: "0.1.0",
|
||||||
protocol_version: "1.0",
|
protocol_version: "1.1",
|
||||||
min_compatible_peer: "0.0.0",
|
min_compatible_peer: "0.0.0",
|
||||||
min_compatible_protocol: "1.0",
|
min_compatible_protocol: "1.1",
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
@@ -102,7 +107,7 @@ describe("performHandshake", () => {
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
expect(outcome.result.min_compatible_peer).toBe("0.0.0");
|
expect(outcome.result.min_compatible_peer).toBe("0.0.0");
|
||||||
expect(outcome.result.min_compatible_protocol).toBe("1.0");
|
expect(outcome.result.min_compatible_protocol).toBe("1.1");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("accepts legacy daemon reply with only min_compatible_peer", async () => {
|
it("accepts legacy daemon reply with only min_compatible_peer", async () => {
|
||||||
@@ -129,8 +134,8 @@ describe("performHandshake", () => {
|
|||||||
const response = {
|
const response = {
|
||||||
server: "browser-skill-daemon",
|
server: "browser-skill-daemon",
|
||||||
version: "0.1.0",
|
version: "0.1.0",
|
||||||
protocol_version: "1.0",
|
protocol_version: "1.1",
|
||||||
min_compatible_protocol: "1.0",
|
min_compatible_protocol: "1.1",
|
||||||
} satisfies HandshakeResult;
|
} satisfies HandshakeResult;
|
||||||
const transport = fakeTransport((req) => ({
|
const transport = fakeTransport((req) => ({
|
||||||
id: (req as { id: string }).id,
|
id: (req as { id: string }).id,
|
||||||
@@ -145,7 +150,7 @@ describe("performHandshake", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
expect(outcome.result.min_compatible_peer).toBeUndefined();
|
expect(outcome.result.min_compatible_peer).toBeUndefined();
|
||||||
expect(outcome.result.min_compatible_protocol).toBe("1.0");
|
expect(outcome.result.min_compatible_protocol).toBe("1.1");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("rejects when the daemon responds with an error", async () => {
|
it("rejects when the daemon responds with an error", async () => {
|
||||||
@@ -178,9 +183,9 @@ describe("performHandshake", () => {
|
|||||||
result: {
|
result: {
|
||||||
server: "browser-skill-daemon",
|
server: "browser-skill-daemon",
|
||||||
version: "0.1.0",
|
version: "0.1.0",
|
||||||
protocol_version: "1.0",
|
protocol_version: "1.1",
|
||||||
min_compatible_peer: "0.0.0",
|
min_compatible_peer: "0.0.0",
|
||||||
min_compatible_protocol: "1.0",
|
min_compatible_protocol: "1.1",
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
emit({
|
emit({
|
||||||
@@ -188,14 +193,14 @@ describe("performHandshake", () => {
|
|||||||
result: {
|
result: {
|
||||||
server: "browser-skill-daemon",
|
server: "browser-skill-daemon",
|
||||||
version: "0.1.0",
|
version: "0.1.0",
|
||||||
protocol_version: "1.0",
|
protocol_version: "1.1",
|
||||||
min_compatible_peer: "0.0.0",
|
min_compatible_peer: "0.0.0",
|
||||||
min_compatible_protocol: "1.0",
|
min_compatible_protocol: "1.1",
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
await expect(pending).resolves.toMatchObject({
|
await expect(pending).resolves.toMatchObject({
|
||||||
result: { server: "browser-skill-daemon", protocol_version: "1.0" },
|
result: { server: "browser-skill-daemon", protocol_version: "1.1" },
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ import type {
|
|||||||
ResponseFrame,
|
ResponseFrame,
|
||||||
} from "./types";
|
} from "./types";
|
||||||
|
|
||||||
export const PROTOCOL_VERSION = "1.0";
|
export const PROTOCOL_VERSION = "1.1";
|
||||||
/**
|
/**
|
||||||
* Extension semver, injected at build time from `package.json` via
|
* Extension semver, injected at build time from `package.json` via
|
||||||
* Vite's `define` (see `wxt.config.ts` and `vitest.config.ts`).
|
* Vite's `define` (see `wxt.config.ts` and `vitest.config.ts`).
|
||||||
@@ -18,7 +18,7 @@ export const EXTENSION_VERSION: string =
|
|||||||
* Lowest **protocol** version this extension accepts (e.g. `"1.0"`).
|
* Lowest **protocol** version this extension accepts (e.g. `"1.0"`).
|
||||||
* Must stay in sync with daemon `MIN_COMPATIBLE_PROTOCOL`.
|
* Must stay in sync with daemon `MIN_COMPATIBLE_PROTOCOL`.
|
||||||
*/
|
*/
|
||||||
export const MIN_COMPATIBLE_PROTOCOL = "1.0";
|
export const MIN_COMPATIBLE_PROTOCOL = "1.1";
|
||||||
/**
|
/**
|
||||||
* **Deprecated** — legacy app-semver floor for wire compat with old
|
* **Deprecated** — legacy app-semver floor for wire compat with old
|
||||||
* daemons. New code sends `"0.0.0"`; compat decisions ignore this.
|
* daemons. New code sends `"0.0.0"`; compat decisions ignore this.
|
||||||
|
|||||||
@@ -3,6 +3,8 @@
|
|||||||
// stay structural (interface, not class) so the same JSON parses on both
|
// stay structural (interface, not class) so the same JSON parses on both
|
||||||
// sides without extra adapters.
|
// sides without extra adapters.
|
||||||
|
|
||||||
|
import type { CaptureTargetDescriptor } from "@/lib/describe-target";
|
||||||
|
|
||||||
export type RpcId = string;
|
export type RpcId = string;
|
||||||
|
|
||||||
export type ErrorCode =
|
export type ErrorCode =
|
||||||
@@ -656,26 +658,72 @@ export interface EmulateResult {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// --------------------------------------------------------------------------
|
// --------------------------------------------------------------------------
|
||||||
// Semantic record payloads — mirror bsk-protocol record.rs (Trace v2)
|
// Semantic record payloads — mirror bsk-protocol record.rs (Trace v3)
|
||||||
// --------------------------------------------------------------------------
|
// --------------------------------------------------------------------------
|
||||||
|
|
||||||
|
export const TRACE_VERSION = 3;
|
||||||
|
export const VOM_FORMAT_VERSION = 1;
|
||||||
|
|
||||||
export interface TargetDescriptor {
|
export interface TargetDescriptor {
|
||||||
|
ref?: string;
|
||||||
role?: string;
|
role?: string;
|
||||||
name?: string;
|
name?: string;
|
||||||
tag: string;
|
ctx?: string;
|
||||||
name_attr?: string;
|
unmatched?: boolean;
|
||||||
placeholder?: string;
|
|
||||||
nearby_label?: string;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface TraceEntry {
|
export interface TraceEntry {
|
||||||
start_url: string;
|
start_url: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface PageRef {
|
export interface RecorderInfo {
|
||||||
|
bsk: string;
|
||||||
|
vom: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type FrameCaptureStatus = "complete" | "partial";
|
||||||
|
|
||||||
|
export type FrameCaptureFailureReason =
|
||||||
|
| "not_injectable"
|
||||||
|
| "arm_failed"
|
||||||
|
| "rearm_failed"
|
||||||
|
| "drain_failed"
|
||||||
|
| "flush_failed";
|
||||||
|
|
||||||
|
export interface FrameCaptureFailure {
|
||||||
|
reason: FrameCaptureFailureReason;
|
||||||
|
frame_id: number;
|
||||||
|
document_id?: string;
|
||||||
|
url?: string;
|
||||||
|
detail?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FrameCaptureInfo {
|
||||||
|
status: FrameCaptureStatus;
|
||||||
|
failures?: FrameCaptureFailure[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export type StopReason = "user_finish" | "cli_stop";
|
||||||
|
|
||||||
|
export interface TraceState {
|
||||||
id: string;
|
id: string;
|
||||||
url: string;
|
url: string;
|
||||||
title?: string;
|
title?: string;
|
||||||
|
/** Wire-only: full page observation text. */
|
||||||
|
body?: string;
|
||||||
|
/** Disk-only: filename under the bundle `pages/` directory. */
|
||||||
|
page?: string;
|
||||||
|
truncated?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface StepResult {
|
||||||
|
state: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface StepCommon {
|
||||||
|
id: number;
|
||||||
|
state: string;
|
||||||
|
result: StepResult;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface SelectedOption {
|
export interface SelectedOption {
|
||||||
@@ -683,66 +731,140 @@ export interface SelectedOption {
|
|||||||
label?: string;
|
label?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface StepEffect {
|
export type NavigationCause =
|
||||||
navigated_to: string;
|
| "user_typed"
|
||||||
|
| "link"
|
||||||
|
| "form_submit"
|
||||||
|
| "reload"
|
||||||
|
| "history"
|
||||||
|
| "script"
|
||||||
|
| "browser";
|
||||||
|
|
||||||
|
export type FillCommit = "enter" | "suggestion" | "blur";
|
||||||
|
|
||||||
|
/** Geometry reported by the content script at capture time. */
|
||||||
|
export interface CaptureGeometry {
|
||||||
|
rect: { x: number; y: number; w: number; h: number };
|
||||||
|
scrollX: number;
|
||||||
|
scrollY: number;
|
||||||
|
position: string;
|
||||||
|
tag: string;
|
||||||
|
ownerFrameBackendNodeId?: number | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface StepCommon {
|
/** Capture/buffer draft before v3 reduction. */
|
||||||
id: number;
|
|
||||||
page: string;
|
|
||||||
effect?: StepEffect;
|
|
||||||
}
|
|
||||||
|
|
||||||
/** Capture/buffer draft before v2 reduction. */
|
|
||||||
export type DraftTraceStep =
|
export type DraftTraceStep =
|
||||||
| {
|
| {
|
||||||
op: "click";
|
op: "click";
|
||||||
target: TargetDescriptor;
|
target: TargetDescriptor;
|
||||||
|
captureTarget?: CaptureTargetDescriptor;
|
||||||
|
geometry?: CaptureGeometry;
|
||||||
navigated_to?: string;
|
navigated_to?: string;
|
||||||
page_url?: string;
|
page_url?: string;
|
||||||
|
title?: string;
|
||||||
|
preStateId?: string;
|
||||||
|
postStateId?: string;
|
||||||
|
observationBody?: string;
|
||||||
|
observationHash?: string;
|
||||||
|
truncated?: boolean;
|
||||||
}
|
}
|
||||||
| {
|
| {
|
||||||
op: "hover";
|
op: "hover";
|
||||||
target: TargetDescriptor;
|
target: TargetDescriptor;
|
||||||
|
captureTarget?: CaptureTargetDescriptor;
|
||||||
|
geometry?: CaptureGeometry;
|
||||||
page_url?: string;
|
page_url?: string;
|
||||||
|
title?: string;
|
||||||
|
preStateId?: string;
|
||||||
|
postStateId?: string;
|
||||||
|
observationBody?: string;
|
||||||
|
observationHash?: string;
|
||||||
|
truncated?: boolean;
|
||||||
}
|
}
|
||||||
| {
|
| {
|
||||||
op: "fill";
|
op: "fill";
|
||||||
target: TargetDescriptor;
|
target: TargetDescriptor;
|
||||||
|
captureTarget?: CaptureTargetDescriptor;
|
||||||
|
geometry?: CaptureGeometry;
|
||||||
value: string;
|
value: string;
|
||||||
|
commit?: FillCommit;
|
||||||
redacted?: boolean;
|
redacted?: boolean;
|
||||||
|
navigated_to?: string;
|
||||||
page_url?: string;
|
page_url?: string;
|
||||||
|
title?: string;
|
||||||
|
preStateId?: string;
|
||||||
|
postStateId?: string;
|
||||||
|
observationBody?: string;
|
||||||
|
observationHash?: string;
|
||||||
|
truncated?: boolean;
|
||||||
}
|
}
|
||||||
| {
|
| {
|
||||||
op: "press";
|
op: "press";
|
||||||
key: string;
|
key: string;
|
||||||
target?: TargetDescriptor;
|
target?: TargetDescriptor;
|
||||||
|
captureTarget?: CaptureTargetDescriptor;
|
||||||
|
geometry?: CaptureGeometry;
|
||||||
modifiers?: KeyModifier[];
|
modifiers?: KeyModifier[];
|
||||||
navigated_to?: string;
|
navigated_to?: string;
|
||||||
page_url?: string;
|
page_url?: string;
|
||||||
|
title?: string;
|
||||||
|
preStateId?: string;
|
||||||
|
postStateId?: string;
|
||||||
|
observationBody?: string;
|
||||||
|
observationHash?: string;
|
||||||
|
truncated?: boolean;
|
||||||
}
|
}
|
||||||
| {
|
| {
|
||||||
op: "select";
|
op: "select";
|
||||||
target: TargetDescriptor;
|
target: TargetDescriptor;
|
||||||
|
captureTarget?: CaptureTargetDescriptor;
|
||||||
|
geometry?: CaptureGeometry;
|
||||||
values: string[];
|
values: string[];
|
||||||
labels?: string[];
|
labels?: string[];
|
||||||
navigated_to?: string;
|
navigated_to?: string;
|
||||||
page_url?: string;
|
page_url?: string;
|
||||||
|
title?: string;
|
||||||
|
preStateId?: string;
|
||||||
|
postStateId?: string;
|
||||||
|
observationBody?: string;
|
||||||
|
observationHash?: string;
|
||||||
|
truncated?: boolean;
|
||||||
|
}
|
||||||
|
| {
|
||||||
|
op: "scroll";
|
||||||
|
page_url?: string;
|
||||||
|
title?: string;
|
||||||
|
preStateId?: string;
|
||||||
|
postStateId?: string;
|
||||||
|
observationBody?: string;
|
||||||
|
observationHash?: string;
|
||||||
|
truncated?: boolean;
|
||||||
}
|
}
|
||||||
| {
|
| {
|
||||||
op: "navigate";
|
op: "navigate";
|
||||||
url: string;
|
url: string;
|
||||||
|
cause?: NavigationCause;
|
||||||
page_url?: string;
|
page_url?: string;
|
||||||
|
title?: string;
|
||||||
|
preStateId?: string;
|
||||||
|
postStateId?: string;
|
||||||
|
observationBody?: string;
|
||||||
|
observationHash?: string;
|
||||||
|
truncated?: boolean;
|
||||||
|
/** Raw webNavigation transition metadata for cause mapping. */
|
||||||
|
transitionType?: string;
|
||||||
|
transitionQualifiers?: string[];
|
||||||
};
|
};
|
||||||
|
|
||||||
/** Exported record-only step (trace v2). */
|
/** Exported record-only step (trace v3). */
|
||||||
export type Step =
|
export type Step =
|
||||||
| ({ op: "navigate" } & StepCommon & { to: string })
|
| ({ op: "navigate" } & StepCommon & { to: string; cause: NavigationCause })
|
||||||
| ({ op: "click" } & StepCommon & { target: TargetDescriptor })
|
| ({ op: "click" } & StepCommon & { target: TargetDescriptor })
|
||||||
| ({ op: "hover" } & StepCommon & { target: TargetDescriptor })
|
| ({ op: "hover" } & StepCommon & { target: TargetDescriptor })
|
||||||
| ({ op: "fill" } & StepCommon & {
|
| ({ op: "fill" } & StepCommon & {
|
||||||
target: TargetDescriptor;
|
target: TargetDescriptor;
|
||||||
value: string;
|
value: string;
|
||||||
|
commit: FillCommit;
|
||||||
redacted?: boolean;
|
redacted?: boolean;
|
||||||
})
|
})
|
||||||
| ({ op: "select" } & StepCommon & {
|
| ({ op: "select" } & StepCommon & {
|
||||||
@@ -753,14 +875,20 @@ export type Step =
|
|||||||
key: string;
|
key: string;
|
||||||
modifiers?: KeyModifier[];
|
modifiers?: KeyModifier[];
|
||||||
target?: TargetDescriptor;
|
target?: TargetDescriptor;
|
||||||
});
|
})
|
||||||
|
| ({ op: "scroll" } & StepCommon);
|
||||||
|
|
||||||
export interface Trace {
|
export interface Trace {
|
||||||
|
version: number;
|
||||||
recorded_at: string;
|
recorded_at: string;
|
||||||
started_at?: string;
|
started_at?: string;
|
||||||
purpose?: string;
|
purpose?: string;
|
||||||
|
stopped_by: StopReason;
|
||||||
entry: TraceEntry;
|
entry: TraceEntry;
|
||||||
pages: PageRef[];
|
recorder: RecorderInfo;
|
||||||
|
/** Present when one or more frames could not be armed, rearmed, or flushed. */
|
||||||
|
frame_capture?: FrameCaptureInfo;
|
||||||
|
states: TraceState[];
|
||||||
steps: Step[];
|
steps: Step[];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -769,6 +897,8 @@ export interface RecordStartParams {
|
|||||||
tab_id?: number;
|
tab_id?: number;
|
||||||
url?: string;
|
url?: string;
|
||||||
purpose?: string;
|
purpose?: string;
|
||||||
|
max_page_tokens?: number;
|
||||||
|
redact_values?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface RecordStartResult {
|
export interface RecordStartResult {
|
||||||
|
|||||||
@@ -265,19 +265,27 @@ retry — complete the task autonomously or stop gracefully.
|
|||||||
|
|
||||||
### Recording — `bsk record`
|
### Recording — `bsk record`
|
||||||
|
|
||||||
Capture the user's own actions in the Agent Window to a `trace.json`, for later LLM-driven automation:
|
Capture the user's own actions in the Agent Window to a **trace v3 bundle**, for later LLM-driven automation:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
bsk record start --browser <instance-id-or-label> [--url https://…] [--purpose "publish a wiki doc"] [--output trace.json]
|
bsk record start --browser <instance-id-or-label> \
|
||||||
|
[--url https://…] [--purpose "publish a wiki doc"] \
|
||||||
|
[--max-page-tokens 3000] [--redact-values] \
|
||||||
|
[--output trace]
|
||||||
# `--url` is optional; default https://example.com/ when omitted (must be http(s)).
|
# `--url` is optional; default https://example.com/ when omitted (must be http(s)).
|
||||||
# Blocks until the user clicks Finish in the recording panel, then writes ./trace.json and closes the window.
|
# Blocks until the user clicks Finish in the recording panel, then writes:
|
||||||
|
# trace/trace.json — action chain + state index (version 3)
|
||||||
|
# trace/pages/ — one `.vom.txt` observe snapshot per settled page state
|
||||||
|
|
||||||
bsk record stop [--output trace.json] # terminal fallback if the browser panel is unavailable
|
bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable
|
||||||
```
|
```
|
||||||
|
|
||||||
- The trace is a **record-only action log** (a `pages[]` dictionary + `navigate`/`click`/`fill`/`select`/`press` steps with `target` descriptors). It records *what the user did*; deciding which inputs are variable is left to the executing agent.
|
- **Bundle layout:** `--output` is a directory (default `./trace`) containing `trace.json` and `pages/`. `trace.json` lists `states[]` (page observation ids) and `steps[]` (each step binds `state` = observe snapshot *before* the action and `result.state` = snapshot *after* settle). Page bodies live in `pages/sN.vom.txt` using the same VOM format as `bsk observe`.
|
||||||
|
- Each `states[]` entry is one **settled page observation** (captured after recording start, navigation landing, or action settle — not on a timer).
|
||||||
|
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
|
||||||
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
||||||
- There is **no** `bsk replay` — to redo a flow, read the trace and reuse the existing `session` / `snapshot` / `@eN` / `click` / `fill` tools. Follow **Stop when the goal is met**.
|
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
|
||||||
|
- Recording captures interactions in the **top frame and injectable child frames** (same-origin and cross-origin where the extension can attach). Embedded frames that cannot be injected are omitted; the trace may include `frame_capture.status: "partial"` with failure details.
|
||||||
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
||||||
|
|
||||||
## Error handling
|
## Error handling
|
||||||
|
|||||||
@@ -480,7 +480,7 @@ mod m2_tests {
|
|||||||
fn fake_status(browsers: Vec<BrowserStatusEntry>, skew: Vec<VersionSkewEntry>) -> StatusResult {
|
fn fake_status(browsers: Vec<BrowserStatusEntry>, skew: Vec<VersionSkewEntry>) -> StatusResult {
|
||||||
StatusResult {
|
StatusResult {
|
||||||
daemon_version: env!("CARGO_PKG_VERSION").into(),
|
daemon_version: env!("CARGO_PKG_VERSION").into(),
|
||||||
protocol_version: "1.0".into(),
|
protocol_version: "1.1".into(),
|
||||||
pid: 1,
|
pid: 1,
|
||||||
uptime_secs: 0,
|
uptime_secs: 0,
|
||||||
ws_port: 0,
|
ws_port: 0,
|
||||||
@@ -545,7 +545,7 @@ mod m2_tests {
|
|||||||
session_count: 0,
|
session_count: 0,
|
||||||
connected_at_ms: 1,
|
connected_at_ms: 1,
|
||||||
version_skew: false,
|
version_skew: false,
|
||||||
extension_protocol_version: "1.0".into(),
|
extension_protocol_version: "1.1".into(),
|
||||||
}],
|
}],
|
||||||
Vec::new(),
|
Vec::new(),
|
||||||
);
|
);
|
||||||
@@ -566,7 +566,7 @@ mod m2_tests {
|
|||||||
session_count: 0,
|
session_count: 0,
|
||||||
connected_at_ms: 1,
|
connected_at_ms: 1,
|
||||||
version_skew: true,
|
version_skew: true,
|
||||||
extension_protocol_version: "1.1".into(),
|
extension_protocol_version: "1.2".into(),
|
||||||
}],
|
}],
|
||||||
vec![VersionSkewEntry {
|
vec![VersionSkewEntry {
|
||||||
instance_id: "alpha".into(),
|
instance_id: "alpha".into(),
|
||||||
@@ -574,8 +574,8 @@ mod m2_tests {
|
|||||||
label: "Personal".into(),
|
label: "Personal".into(),
|
||||||
server_version: env!("CARGO_PKG_VERSION").into(),
|
server_version: env!("CARGO_PKG_VERSION").into(),
|
||||||
client_version: "0.0.9".into(),
|
client_version: "0.0.9".into(),
|
||||||
server_protocol_version: "1.0".into(),
|
server_protocol_version: "1.1".into(),
|
||||||
client_protocol_version: "1.1".into(),
|
client_protocol_version: "1.2".into(),
|
||||||
}],
|
}],
|
||||||
);
|
);
|
||||||
let check = check_browsers_protocol_compatible(Some(&status));
|
let check = check_browsers_protocol_compatible(Some(&status));
|
||||||
|
|||||||
@@ -1,15 +1,17 @@
|
|||||||
//! `bsk record start|stop` — capture user actions in the Agent Window.
|
//! `bsk record start|stop` — capture user actions in the Agent Window.
|
||||||
|
|
||||||
use std::fs;
|
use std::collections::HashSet;
|
||||||
use std::path::PathBuf;
|
use std::fs::{self, OpenOptions};
|
||||||
|
use std::io;
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use anyhow::Context;
|
use anyhow::Context;
|
||||||
use bsk_protocol::Method;
|
|
||||||
use bsk_protocol::tools::{
|
use bsk_protocol::tools::{
|
||||||
RecordAwaitParams, RecordAwaitResult, RecordStartParams, RecordStartResult, RecordStopParams,
|
FrameCaptureStatus, RecordAwaitParams, RecordAwaitResult, RecordStartParams, RecordStartResult,
|
||||||
RecordStopResult, Trace,
|
RecordStopParams, RecordStopResult, TRACE_VERSION, Trace, TraceState,
|
||||||
};
|
};
|
||||||
|
use bsk_protocol::{ErrorCode, Method};
|
||||||
use clap::{Args, Subcommand};
|
use clap::{Args, Subcommand};
|
||||||
|
|
||||||
use crate::cli::TOOL_IPC_TIMEOUT;
|
use crate::cli::TOOL_IPC_TIMEOUT;
|
||||||
@@ -18,10 +20,34 @@ use crate::cli::ensure_daemon::ensure_daemon;
|
|||||||
use crate::cli::error::{CliError, Format};
|
use crate::cli::error::{CliError, Format};
|
||||||
use crate::cli::record_state;
|
use crate::cli::record_state;
|
||||||
use crate::cli::session::{SessionStartOptions, start_session, stop_session};
|
use crate::cli::session::{SessionStartOptions, start_session, stop_session};
|
||||||
|
use crate::daemon::ipc::STOP_IN_PROGRESS_REASON;
|
||||||
|
|
||||||
/// Max wait for the user to click 结束 in the browser (24 hours).
|
/// Max wait for the user to click 结束 in the browser (24 hours).
|
||||||
const RECORD_AWAIT_TIMEOUT_MS: u32 = 86_400_000;
|
const RECORD_AWAIT_TIMEOUT_MS: u32 = 86_400_000;
|
||||||
|
|
||||||
|
/// Tear down the recording session, tolerating the teardown the other half of
|
||||||
|
/// the pair already started. `record start` blocks in `record_await` and stops
|
||||||
|
/// the session once the trace arrives; the documented `record stop` fallback
|
||||||
|
/// stops it too. Whichever loses that race must not report a failure — the
|
||||||
|
/// trace is already written by then.
|
||||||
|
fn stop_recording_session(sock: PathBuf, session_id: &str) -> Result<(), CliError> {
|
||||||
|
match stop_session(sock, session_id) {
|
||||||
|
Ok(_) => Ok(()),
|
||||||
|
Err(err) if session_teardown_raced(&err) => Ok(()),
|
||||||
|
Err(err) => Err(err),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn session_teardown_raced(err: &CliError) -> bool {
|
||||||
|
if err.code() == Some(ErrorCode::NotFound) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
err.data()
|
||||||
|
.and_then(|data| data.get("reason"))
|
||||||
|
.and_then(serde_json::Value::as_str)
|
||||||
|
== Some(STOP_IN_PROGRESS_REASON)
|
||||||
|
}
|
||||||
|
|
||||||
/// `bsk record …` subcommand tree.
|
/// `bsk record …` subcommand tree.
|
||||||
#[derive(Debug, Clone, Args)]
|
#[derive(Debug, Clone, Args)]
|
||||||
pub struct RecordCmd {
|
pub struct RecordCmd {
|
||||||
@@ -58,15 +84,25 @@ pub struct RecordStartArgs {
|
|||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
pub purpose: Option<String>,
|
pub purpose: Option<String>,
|
||||||
|
|
||||||
/// Output path for the trace JSON (default `./trace.json`).
|
/// Max VOM tokens per page observation file (default 3000).
|
||||||
#[arg(long, default_value = "trace.json")]
|
#[arg(long = "max-page-tokens")]
|
||||||
|
pub max_page_tokens: Option<u32>,
|
||||||
|
|
||||||
|
/// Redact all form values in page observations (`[filled]` only).
|
||||||
|
#[arg(long = "redact-values")]
|
||||||
|
pub redact_values: bool,
|
||||||
|
|
||||||
|
/// Output directory for the trace bundle (default `./trace`).
|
||||||
|
/// Writes `<dir>/trace.json` and `<dir>/pages/*.vom.txt`.
|
||||||
|
#[arg(long, default_value = "trace")]
|
||||||
pub output: PathBuf,
|
pub output: PathBuf,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Args)]
|
#[derive(Debug, Clone, Args)]
|
||||||
pub struct RecordStopArgs {
|
pub struct RecordStopArgs {
|
||||||
/// Output path for the trace JSON (default `./trace.json`).
|
/// Output directory for the trace bundle (default `./trace`).
|
||||||
#[arg(long, default_value = "trace.json")]
|
/// Writes `<dir>/trace.json` and `<dir>/pages/*.vom.txt`.
|
||||||
|
#[arg(long, default_value = "trace")]
|
||||||
pub output: PathBuf,
|
pub output: PathBuf,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -98,6 +134,8 @@ fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError>
|
|||||||
tab_id: args.tab_id,
|
tab_id: args.tab_id,
|
||||||
url: args.url,
|
url: args.url,
|
||||||
purpose: args.purpose.clone(),
|
purpose: args.purpose.clone(),
|
||||||
|
max_page_tokens: args.max_page_tokens,
|
||||||
|
redact_values: Some(args.redact_values),
|
||||||
};
|
};
|
||||||
let start_result = business_rpc::call::<RecordStartParams, RecordStartResult>(
|
let start_result = business_rpc::call::<RecordStartParams, RecordStartResult>(
|
||||||
info.sock_path.clone(),
|
info.sock_path.clone(),
|
||||||
@@ -142,13 +180,13 @@ fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError>
|
|||||||
// Keep write/render inside a Result so `?` cannot skip session teardown.
|
// Keep write/render inside a Result so `?` cannot skip session teardown.
|
||||||
let run_result: Result<(), CliError> = match await_result {
|
let run_result: Result<(), CliError> = match await_result {
|
||||||
Ok(result) => (|| {
|
Ok(result) => (|| {
|
||||||
write_trace_file(&args.output, &result.trace)?;
|
let pages_dir = write_trace_bundle(&args.output, &result.trace)?;
|
||||||
render_finish(&result.trace, &args.output, format)
|
render_finish(&result.trace, &args.output, &pages_dir, format)
|
||||||
})(),
|
})(),
|
||||||
Err(err) => Err(err),
|
Err(err) => Err(err),
|
||||||
};
|
};
|
||||||
|
|
||||||
let session_stop_result = stop_session(info.sock_path, &session.session_id);
|
let session_stop_result = stop_recording_session(info.sock_path, &session.session_id);
|
||||||
record_state::clear();
|
record_state::clear();
|
||||||
|
|
||||||
run_result?;
|
run_result?;
|
||||||
@@ -161,22 +199,23 @@ fn dispatch_stop(args: RecordStopArgs, format: Format) -> Result<(), CliError> {
|
|||||||
let info = ensure_daemon().context("ensure daemon is running")?;
|
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||||
let session_id = state.session_id.clone();
|
let session_id = state.session_id.clone();
|
||||||
|
|
||||||
|
let params = RecordStopParams {
|
||||||
|
session_id: session_id.clone(),
|
||||||
|
};
|
||||||
|
let result = business_rpc::call::<RecordStopParams, RecordStopResult>(
|
||||||
|
info.sock_path.clone(),
|
||||||
|
"record-stop",
|
||||||
|
Method::ToolRecordStop,
|
||||||
|
Some(params),
|
||||||
|
TOOL_IPC_TIMEOUT,
|
||||||
|
)?;
|
||||||
|
|
||||||
let run_result: Result<(), CliError> = (|| {
|
let run_result: Result<(), CliError> = (|| {
|
||||||
let params = RecordStopParams {
|
let pages_dir = write_trace_bundle(&args.output, &result.trace)?;
|
||||||
session_id: session_id.clone(),
|
render_stop(&result, &args.output, &pages_dir, format)
|
||||||
};
|
|
||||||
let result = business_rpc::call::<RecordStopParams, RecordStopResult>(
|
|
||||||
info.sock_path.clone(),
|
|
||||||
"record-stop",
|
|
||||||
Method::ToolRecordStop,
|
|
||||||
Some(params),
|
|
||||||
TOOL_IPC_TIMEOUT,
|
|
||||||
)?;
|
|
||||||
write_trace_file(&args.output, &result.trace)?;
|
|
||||||
render_stop(&result, &args.output, format)
|
|
||||||
})();
|
})();
|
||||||
|
|
||||||
let session_stop_result = stop_session(info.sock_path, &session_id);
|
let session_stop_result = stop_recording_session(info.sock_path, &session_id);
|
||||||
record_state::clear();
|
record_state::clear();
|
||||||
|
|
||||||
run_result?;
|
run_result?;
|
||||||
@@ -190,30 +229,313 @@ fn record_await_ipc_timeout(timeout_ms: u32) -> Duration {
|
|||||||
.unwrap_or(Duration::from_secs(u64::from(timeout_ms / 1_000) + 15))
|
.unwrap_or(Duration::from_secs(u64::from(timeout_ms / 1_000) + 15))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn write_trace_file(output: &PathBuf, trace: &Trace) -> Result<(), CliError> {
|
fn pages_dir_for_output(output_dir: &Path) -> PathBuf {
|
||||||
let json = serde_json::to_string_pretty(trace)
|
output_dir.join("pages")
|
||||||
.context("serialize trace JSON")
|
}
|
||||||
.map_err(CliError::Local)?;
|
|
||||||
if let Some(parent) = output.parent() {
|
fn trace_json_path(output_dir: &Path) -> PathBuf {
|
||||||
if !parent.as_os_str().is_empty() {
|
output_dir.join("trace.json")
|
||||||
fs::create_dir_all(parent)
|
}
|
||||||
.with_context(|| format!("create output directory {}", parent.display()))
|
|
||||||
.map_err(CliError::Local)?;
|
fn is_canonical_state_id(id: &str) -> bool {
|
||||||
|
let Some(number) = id.strip_prefix('s') else {
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
let mut digits = number.bytes();
|
||||||
|
matches!(digits.next(), Some(b'1'..=b'9')) && digits.all(|byte| byte.is_ascii_digit())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_bundle_directory(path: &Path, description: &str) -> Result<(), CliError> {
|
||||||
|
match fs::symlink_metadata(path) {
|
||||||
|
Ok(metadata) if metadata.file_type().is_symlink() => Err(CliError::Local(anyhow::anyhow!(
|
||||||
|
"{description} {} must not be a symlink",
|
||||||
|
path.display()
|
||||||
|
))),
|
||||||
|
Ok(metadata) if metadata.is_dir() => Ok(()),
|
||||||
|
Ok(_) => Err(CliError::Local(anyhow::anyhow!(
|
||||||
|
"{description} {} is not a directory",
|
||||||
|
path.display()
|
||||||
|
))),
|
||||||
|
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
|
||||||
|
Err(err) => Err(CliError::Local(
|
||||||
|
anyhow::Error::new(err).context(format!("inspect {description} {}", path.display())),
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_replaceable_file(path: &Path, description: &str) -> Result<(), CliError> {
|
||||||
|
match fs::symlink_metadata(path) {
|
||||||
|
Ok(metadata) if metadata.file_type().is_file() => Ok(()),
|
||||||
|
Ok(_) => Err(CliError::Local(anyhow::anyhow!(
|
||||||
|
"{description} {} is not a regular file",
|
||||||
|
path.display()
|
||||||
|
))),
|
||||||
|
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
|
||||||
|
Err(err) => Err(CliError::Local(
|
||||||
|
anyhow::Error::new(err).context(format!("inspect {description} {}", path.display())),
|
||||||
|
)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn commit_staged_file(staged: &Path, target: &Path, backup: &Path) -> io::Result<Option<PathBuf>> {
|
||||||
|
let old_file = match fs::symlink_metadata(target) {
|
||||||
|
Ok(_) => {
|
||||||
|
fs::rename(target, backup)?;
|
||||||
|
Some(backup.to_path_buf())
|
||||||
|
}
|
||||||
|
Err(err) if err.kind() == io::ErrorKind::NotFound => None,
|
||||||
|
Err(err) => return Err(err),
|
||||||
|
};
|
||||||
|
|
||||||
|
if let Err(err) = fs::rename(staged, target) {
|
||||||
|
if let Some(old_file) = old_file.as_deref() {
|
||||||
|
if let Err(restore_err) = fs::rename(old_file, target) {
|
||||||
|
return Err(io::Error::new(
|
||||||
|
restore_err.kind(),
|
||||||
|
format!(
|
||||||
|
"{err}; additionally failed to restore {}: {restore_err}",
|
||||||
|
target.display()
|
||||||
|
),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return Err(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(old_file)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn rollback_installed_files(installed: &[(PathBuf, Option<PathBuf>)]) -> io::Result<()> {
|
||||||
|
let mut rollback_error = None;
|
||||||
|
for (target, backup) in installed.iter().rev() {
|
||||||
|
match fs::remove_file(target) {
|
||||||
|
Ok(()) => {}
|
||||||
|
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
|
||||||
|
Err(err) => {
|
||||||
|
rollback_error.get_or_insert(err);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(backup) = backup {
|
||||||
|
if let Err(err) = fs::rename(backup, target) {
|
||||||
|
rollback_error.get_or_insert(err);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
match rollback_error {
|
||||||
|
Some(err) => Err(err),
|
||||||
|
None => Ok(()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Drop page files left by an earlier, longer recording without deleting the
|
||||||
|
/// directory itself: `record start` and the `record stop` fallback export the
|
||||||
|
/// same recording concurrently, and a `remove_dir_all` here would fail the
|
||||||
|
/// writer whose files the other one just erased.
|
||||||
|
fn prune_stale_pages(pages_dir: &Path, keep: &HashSet<String>) -> Result<(), CliError> {
|
||||||
|
let entries = match fs::read_dir(pages_dir) {
|
||||||
|
Ok(entries) => entries,
|
||||||
|
Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(()),
|
||||||
|
Err(err) => {
|
||||||
|
return Err(CliError::Local(
|
||||||
|
anyhow::Error::new(err).context(format!("read pages dir {}", pages_dir.display())),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
for entry in entries.flatten() {
|
||||||
|
let name = entry.file_name();
|
||||||
|
let Some(name) = name.to_str() else { continue };
|
||||||
|
if !name.ends_with(".vom.txt") || keep.contains(name) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
match fs::remove_file(entry.path()) {
|
||||||
|
Ok(()) => {}
|
||||||
|
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
|
||||||
|
Err(err) => {
|
||||||
|
return Err(CliError::Local(
|
||||||
|
anyhow::Error::new(err)
|
||||||
|
.context(format!("remove stale page {}", entry.path().display())),
|
||||||
|
));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
fs::write(output, format!("{json}\n"))
|
|
||||||
.with_context(|| format!("write trace to {}", output.display()))
|
|
||||||
.map_err(CliError::Local)?;
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn render_finish(trace: &Trace, output: &PathBuf, format: Format) -> Result<(), CliError> {
|
fn write_trace_bundle(output_dir: &Path, trace: &Trace) -> Result<PathBuf, CliError> {
|
||||||
|
let pages_dir = pages_dir_for_output(output_dir);
|
||||||
|
let trace_path = trace_json_path(output_dir);
|
||||||
|
|
||||||
|
if trace.version != TRACE_VERSION {
|
||||||
|
return Err(CliError::Local(anyhow::anyhow!(
|
||||||
|
"trace version {} is not supported (expected {})",
|
||||||
|
trace.version,
|
||||||
|
TRACE_VERSION
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut state_ids = HashSet::with_capacity(trace.states.len());
|
||||||
|
for state in &trace.states {
|
||||||
|
if !is_canonical_state_id(&state.id) {
|
||||||
|
return Err(CliError::Local(anyhow::anyhow!(
|
||||||
|
"state id {:?} is not canonical (expected s1, s2, ...)",
|
||||||
|
state.id
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
if !state_ids.insert(state.id.as_str()) {
|
||||||
|
return Err(CliError::Local(anyhow::anyhow!(
|
||||||
|
"duplicate state id {:?}",
|
||||||
|
state.id
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut disk_states: Vec<TraceState> = Vec::with_capacity(trace.states.len());
|
||||||
|
let mut page_writes = Vec::new();
|
||||||
|
let mut keep = HashSet::new();
|
||||||
|
for state in &trace.states {
|
||||||
|
let mut disk = state.clone();
|
||||||
|
if let Some(body) = state.body.as_deref() {
|
||||||
|
let filename = format!("{}.vom.txt", state.id);
|
||||||
|
let page_path = pages_dir.join(&filename);
|
||||||
|
if page_path.parent() != Some(pages_dir.as_path()) {
|
||||||
|
return Err(CliError::Local(anyhow::anyhow!(
|
||||||
|
"page path {} escapes pages directory {}",
|
||||||
|
page_path.display(),
|
||||||
|
pages_dir.display()
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
keep.insert(filename.clone());
|
||||||
|
page_writes.push((filename.clone(), body));
|
||||||
|
disk.body = None;
|
||||||
|
disk.page = Some(filename);
|
||||||
|
}
|
||||||
|
disk_states.push(disk);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut disk_trace = trace.clone();
|
||||||
|
disk_trace.states = disk_states;
|
||||||
|
let json = serde_json::to_string_pretty(&disk_trace)
|
||||||
|
.context("serialize trace JSON")
|
||||||
|
.map_err(CliError::Local)?;
|
||||||
|
|
||||||
|
validate_bundle_directory(output_dir, "output bundle directory")?;
|
||||||
|
fs::create_dir_all(output_dir)
|
||||||
|
.with_context(|| format!("create output dir {}", output_dir.display()))
|
||||||
|
.map_err(CliError::Local)?;
|
||||||
|
|
||||||
|
validate_bundle_directory(&pages_dir, "pages directory")?;
|
||||||
|
|
||||||
|
// `record start` and `record stop` can export the same recording at once.
|
||||||
|
// Serialize writers without replacing the shared directory underneath one
|
||||||
|
// another; the lock is released when this file handle is dropped.
|
||||||
|
let lock_path = output_dir.join(".bsk-record-export.lock");
|
||||||
|
let lock_file = OpenOptions::new()
|
||||||
|
.read(true)
|
||||||
|
.write(true)
|
||||||
|
.create(true)
|
||||||
|
.truncate(false)
|
||||||
|
.open(&lock_path)
|
||||||
|
.with_context(|| format!("open export lock {}", lock_path.display()))
|
||||||
|
.map_err(CliError::Local)?;
|
||||||
|
fs2::FileExt::lock_exclusive(&lock_file)
|
||||||
|
.with_context(|| format!("lock export bundle {}", output_dir.display()))
|
||||||
|
.map_err(CliError::Local)?;
|
||||||
|
|
||||||
|
fs::create_dir_all(&pages_dir)
|
||||||
|
.with_context(|| format!("create pages dir {}", pages_dir.display()))
|
||||||
|
.map_err(CliError::Local)?;
|
||||||
|
|
||||||
|
for (filename, _) in &page_writes {
|
||||||
|
validate_replaceable_file(&pages_dir.join(filename), "page observation")?;
|
||||||
|
}
|
||||||
|
validate_replaceable_file(&trace_path, "trace JSON")?;
|
||||||
|
|
||||||
|
let transaction_id = uuid::Uuid::new_v4();
|
||||||
|
let staging_dir = output_dir.join(format!(".bsk-record-stage-{transaction_id}"));
|
||||||
|
let staging_pages = staging_dir.join("pages");
|
||||||
|
fs::create_dir_all(&staging_pages)
|
||||||
|
.with_context(|| format!("create staging dir {}", staging_pages.display()))
|
||||||
|
.map_err(CliError::Local)?;
|
||||||
|
|
||||||
|
let stage_result: Result<(), CliError> = (|| {
|
||||||
|
for (filename, body) in &page_writes {
|
||||||
|
let staged_path = staging_pages.join(filename);
|
||||||
|
fs::write(&staged_path, body)
|
||||||
|
.with_context(|| format!("write staged page observation {}", staged_path.display()))
|
||||||
|
.map_err(CliError::Local)?;
|
||||||
|
}
|
||||||
|
let staged_trace = staging_dir.join("trace.json");
|
||||||
|
fs::write(&staged_trace, format!("{json}\n"))
|
||||||
|
.with_context(|| format!("write staged trace to {}", staged_trace.display()))
|
||||||
|
.map_err(CliError::Local)?;
|
||||||
|
Ok(())
|
||||||
|
})();
|
||||||
|
if let Err(err) = stage_result {
|
||||||
|
let _ = fs::remove_dir_all(&staging_dir);
|
||||||
|
return Err(err);
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut replacements: Vec<(PathBuf, PathBuf, PathBuf)> =
|
||||||
|
Vec::with_capacity(page_writes.len() + 1);
|
||||||
|
for (filename, _) in &page_writes {
|
||||||
|
replacements.push((
|
||||||
|
staging_pages.join(filename),
|
||||||
|
pages_dir.join(filename),
|
||||||
|
pages_dir.join(format!(".{filename}.{transaction_id}.bak")),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
replacements.push((
|
||||||
|
staging_dir.join("trace.json"),
|
||||||
|
trace_path.clone(),
|
||||||
|
output_dir.join(format!(".trace.json.{transaction_id}.bak")),
|
||||||
|
));
|
||||||
|
|
||||||
|
let mut installed = Vec::with_capacity(replacements.len());
|
||||||
|
for (staged, target, backup) in replacements {
|
||||||
|
match commit_staged_file(&staged, &target, &backup) {
|
||||||
|
Ok(old_file) => installed.push((target, old_file)),
|
||||||
|
Err(err) => {
|
||||||
|
let rollback_result = rollback_installed_files(&installed);
|
||||||
|
let _ = fs::remove_dir_all(&staging_dir);
|
||||||
|
let message = match rollback_result {
|
||||||
|
Ok(()) => format!("commit replacement {}: {err}", target.display()),
|
||||||
|
Err(rollback_err) => format!(
|
||||||
|
"commit replacement {}: {err}; rollback also failed: {rollback_err}",
|
||||||
|
target.display()
|
||||||
|
),
|
||||||
|
};
|
||||||
|
return Err(CliError::Local(anyhow::anyhow!(message)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (_, backup) in &installed {
|
||||||
|
if let Some(backup) = backup {
|
||||||
|
let _ = fs::remove_file(backup);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let _ = fs::remove_dir_all(&staging_dir);
|
||||||
|
prune_stale_pages(&pages_dir, &keep)?;
|
||||||
|
|
||||||
|
Ok(pages_dir)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn render_finish(
|
||||||
|
trace: &Trace,
|
||||||
|
output_dir: &PathBuf,
|
||||||
|
pages_dir: &PathBuf,
|
||||||
|
format: Format,
|
||||||
|
) -> Result<(), CliError> {
|
||||||
|
let trace_path = trace_json_path(output_dir);
|
||||||
match format {
|
match format {
|
||||||
Format::Json => {
|
Format::Json => {
|
||||||
println!(
|
println!(
|
||||||
"{}",
|
"{}",
|
||||||
serde_json::to_string(&serde_json::json!({
|
serde_json::to_string(&serde_json::json!({
|
||||||
"output": output,
|
"output": output_dir,
|
||||||
|
"trace_json": trace_path,
|
||||||
|
"pages_dir": pages_dir,
|
||||||
"trace": trace,
|
"trace": trace,
|
||||||
"window_closed": true,
|
"window_closed": true,
|
||||||
}))
|
}))
|
||||||
@@ -221,7 +543,20 @@ fn render_finish(trace: &Trace, output: &PathBuf, format: Format) -> Result<(),
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
Format::Human => {
|
Format::Human => {
|
||||||
println!("saved {} steps to {}", trace.steps.len(), output.display());
|
println!(
|
||||||
|
"saved {} steps and {} states to {} (+ {})",
|
||||||
|
trace.steps.len(),
|
||||||
|
trace.states.len(),
|
||||||
|
trace_path.display(),
|
||||||
|
pages_dir.display()
|
||||||
|
);
|
||||||
|
if let Some(frame_capture) = &trace.frame_capture {
|
||||||
|
if matches!(frame_capture.status, FrameCaptureStatus::Partial) {
|
||||||
|
eprintln!(
|
||||||
|
"warning: trace is partial — some embedded frames were not recorded (see frame_capture.failures in trace.json)"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -230,33 +565,252 @@ fn render_finish(trace: &Trace, output: &PathBuf, format: Format) -> Result<(),
|
|||||||
fn render_stop(
|
fn render_stop(
|
||||||
result: &RecordStopResult,
|
result: &RecordStopResult,
|
||||||
output: &PathBuf,
|
output: &PathBuf,
|
||||||
|
pages_dir: &PathBuf,
|
||||||
format: Format,
|
format: Format,
|
||||||
) -> Result<(), CliError> {
|
) -> Result<(), CliError> {
|
||||||
render_finish(&result.trace, output, format)
|
render_finish(&result.trace, output, pages_dir, format)
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
use bsk_protocol::tools::{
|
||||||
|
NavigationCause, RecorderInfo, Step, StepCommon, StepResult, StopReason, TraceEntry,
|
||||||
|
VOM_FORMAT_VERSION,
|
||||||
|
};
|
||||||
|
|
||||||
#[test]
|
fn sample_trace(state_id: &str, body: &str) -> Trace {
|
||||||
fn default_output_is_trace_json() {
|
Trace {
|
||||||
let args = RecordStopArgs {
|
version: TRACE_VERSION,
|
||||||
output: PathBuf::from("trace.json"),
|
recorded_at: "2026-08-10T02:12:55.360Z".into(),
|
||||||
};
|
stopped_by: StopReason::UserFinish,
|
||||||
assert_eq!(args.output, PathBuf::from("trace.json"));
|
entry: TraceEntry {
|
||||||
|
start_url: "https://example.com/".into(),
|
||||||
|
},
|
||||||
|
recorder: RecorderInfo {
|
||||||
|
bsk: "0.1.10".into(),
|
||||||
|
vom: VOM_FORMAT_VERSION,
|
||||||
|
},
|
||||||
|
frame_capture: None,
|
||||||
|
states: vec![TraceState {
|
||||||
|
id: state_id.into(),
|
||||||
|
url: "https://example.com/".into(),
|
||||||
|
title: Some("Example".into()),
|
||||||
|
body: Some(body.into()),
|
||||||
|
page: None,
|
||||||
|
truncated: false,
|
||||||
|
}],
|
||||||
|
steps: vec![Step::Navigate {
|
||||||
|
common: StepCommon {
|
||||||
|
id: 1,
|
||||||
|
state: state_id.into(),
|
||||||
|
result: StepResult {
|
||||||
|
state: state_id.into(),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
to: "https://example.com/".into(),
|
||||||
|
cause: NavigationCause::UserTyped,
|
||||||
|
}],
|
||||||
|
purpose: None,
|
||||||
|
started_at: None,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn start_args_default_output_is_trace_json() {
|
fn concurrent_teardown_is_not_a_record_failure() {
|
||||||
|
let stopping = CliError::from_rpc(bsk_protocol::RpcError {
|
||||||
|
code: ErrorCode::Timeout,
|
||||||
|
message: "session stop is already in progress".into(),
|
||||||
|
data: Some(serde_json::json!({ "reason": STOP_IN_PROGRESS_REASON })),
|
||||||
|
});
|
||||||
|
assert!(session_teardown_raced(&stopping));
|
||||||
|
|
||||||
|
let gone = CliError::from_rpc(bsk_protocol::RpcError {
|
||||||
|
code: ErrorCode::NotFound,
|
||||||
|
message: "session is not registered".into(),
|
||||||
|
data: None,
|
||||||
|
});
|
||||||
|
assert!(session_teardown_raced(&gone));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unrelated_stop_timeout_still_fails() {
|
||||||
|
let stalled = CliError::from_rpc(bsk_protocol::RpcError {
|
||||||
|
code: ErrorCode::Timeout,
|
||||||
|
message: "tool.session_stop timed out".into(),
|
||||||
|
data: None,
|
||||||
|
});
|
||||||
|
assert!(!session_teardown_raced(&stalled));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn default_output_is_trace_dir() {
|
||||||
|
let args = RecordStopArgs {
|
||||||
|
output: PathBuf::from("trace"),
|
||||||
|
};
|
||||||
|
assert_eq!(args.output, PathBuf::from("trace"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn start_args_default_output_is_trace_dir() {
|
||||||
let args = RecordStartArgs {
|
let args = RecordStartArgs {
|
||||||
browser: None,
|
browser: None,
|
||||||
tab_id: None,
|
tab_id: None,
|
||||||
url: None,
|
url: None,
|
||||||
purpose: None,
|
purpose: None,
|
||||||
output: PathBuf::from("trace.json"),
|
max_page_tokens: None,
|
||||||
|
redact_values: false,
|
||||||
|
output: PathBuf::from("trace"),
|
||||||
};
|
};
|
||||||
assert_eq!(args.output, PathBuf::from("trace.json"));
|
assert_eq!(args.output, PathBuf::from("trace"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn write_trace_bundle_writes_dir_layout() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let output = dir.path().join("flow");
|
||||||
|
let trace = sample_trace("s1", "@vom 1\nL1 page");
|
||||||
|
let pages_dir = write_trace_bundle(&output, &trace).unwrap();
|
||||||
|
let trace_path = output.join("trace.json");
|
||||||
|
assert!(trace_path.exists());
|
||||||
|
assert_eq!(pages_dir, output.join("pages"));
|
||||||
|
assert!(pages_dir.join("s1.vom.txt").exists());
|
||||||
|
let disk: serde_json::Value =
|
||||||
|
serde_json::from_str(&fs::read_to_string(&trace_path).unwrap()).unwrap();
|
||||||
|
assert_eq!(disk["states"][0]["page"], "s1.vom.txt");
|
||||||
|
assert!(disk["states"][0].get("body").is_none());
|
||||||
|
assert_eq!(
|
||||||
|
fs::read_to_string(pages_dir.join("s1.vom.txt")).unwrap(),
|
||||||
|
"@vom 1\nL1 page"
|
||||||
|
);
|
||||||
|
|
||||||
|
// Re-exporting drops pages from the previous run but keeps the
|
||||||
|
// directory in place for the concurrent writer.
|
||||||
|
fs::write(pages_dir.join("s9.vom.txt"), "stale").unwrap();
|
||||||
|
fs::write(pages_dir.join("notes.md"), "keep me").unwrap();
|
||||||
|
write_trace_bundle(&output, &trace).unwrap();
|
||||||
|
assert!(!pages_dir.join("s9.vom.txt").exists());
|
||||||
|
assert!(pages_dir.join("s1.vom.txt").exists());
|
||||||
|
assert!(pages_dir.join("notes.md").exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn write_trace_bundle_rejects_traversal_state_id() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let output = dir.path().join("flow");
|
||||||
|
let trace = sample_trace("../escaped", "@vom 1\nL1 page");
|
||||||
|
|
||||||
|
let err = write_trace_bundle(&output, &trace).unwrap_err();
|
||||||
|
|
||||||
|
assert!(err.to_string().contains("state id"));
|
||||||
|
assert!(!output.join("escaped.vom.txt").exists());
|
||||||
|
assert!(!output.join("trace.json").exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn write_trace_bundle_rejects_duplicate_state_id() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let output = dir.path().join("flow");
|
||||||
|
let mut trace = sample_trace("s1", "@vom 1\nL1 first");
|
||||||
|
trace.states.push(TraceState {
|
||||||
|
body: Some("@vom 1\nL1 duplicate".into()),
|
||||||
|
..trace.states[0].clone()
|
||||||
|
});
|
||||||
|
|
||||||
|
let err = write_trace_bundle(&output, &trace).unwrap_err();
|
||||||
|
|
||||||
|
assert!(err.to_string().contains("duplicate state id"));
|
||||||
|
assert!(!output.exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unsupported_version_preserves_existing_bundle() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let output = dir.path().join("flow");
|
||||||
|
write_trace_bundle(&output, &sample_trace("s1", "original")).unwrap();
|
||||||
|
let original_json = fs::read(output.join("trace.json")).unwrap();
|
||||||
|
|
||||||
|
let mut replacement = sample_trace("s2", "replacement");
|
||||||
|
replacement.version += 1;
|
||||||
|
let err = write_trace_bundle(&output, &replacement).unwrap_err();
|
||||||
|
|
||||||
|
assert!(err.to_string().contains("not supported"));
|
||||||
|
assert_eq!(fs::read(output.join("trace.json")).unwrap(), original_json);
|
||||||
|
assert_eq!(
|
||||||
|
fs::read_to_string(output.join("pages/s1.vom.txt")).unwrap(),
|
||||||
|
"original"
|
||||||
|
);
|
||||||
|
assert!(!output.join("pages/s2.vom.txt").exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn page_replacement_failure_preserves_existing_bundle() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let output = dir.path().join("flow");
|
||||||
|
write_trace_bundle(&output, &sample_trace("s1", "original")).unwrap();
|
||||||
|
let original_json = fs::read(output.join("trace.json")).unwrap();
|
||||||
|
fs::write(output.join("pages/notes.md"), "keep me").unwrap();
|
||||||
|
fs::create_dir(output.join("pages/s2.vom.txt")).unwrap();
|
||||||
|
|
||||||
|
let err = write_trace_bundle(&output, &sample_trace("s2", "replacement")).unwrap_err();
|
||||||
|
|
||||||
|
assert!(err.to_string().contains("page"));
|
||||||
|
assert_eq!(fs::read(output.join("trace.json")).unwrap(), original_json);
|
||||||
|
assert_eq!(
|
||||||
|
fs::read_to_string(output.join("pages/s1.vom.txt")).unwrap(),
|
||||||
|
"original"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
fs::read_to_string(output.join("pages/notes.md")).unwrap(),
|
||||||
|
"keep me"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[test]
|
||||||
|
fn write_trace_bundle_rejects_symlinked_pages_directory() {
|
||||||
|
use std::os::unix::fs::symlink;
|
||||||
|
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let output = dir.path().join("flow");
|
||||||
|
let external_pages = dir.path().join("external-pages");
|
||||||
|
fs::create_dir_all(&output).unwrap();
|
||||||
|
fs::create_dir_all(&external_pages).unwrap();
|
||||||
|
fs::write(external_pages.join("s9.vom.txt"), "external").unwrap();
|
||||||
|
symlink(&external_pages, output.join("pages")).unwrap();
|
||||||
|
|
||||||
|
let err = write_trace_bundle(&output, &sample_trace("s1", "replacement")).unwrap_err();
|
||||||
|
|
||||||
|
assert!(err.to_string().contains("symlink"));
|
||||||
|
assert_eq!(
|
||||||
|
fs::read_to_string(external_pages.join("s9.vom.txt")).unwrap(),
|
||||||
|
"external"
|
||||||
|
);
|
||||||
|
assert!(!external_pages.join("s1.vom.txt").exists());
|
||||||
|
assert!(!output.join("trace.json").exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[test]
|
||||||
|
fn write_trace_bundle_rejects_symlinked_output_directory() {
|
||||||
|
use std::os::unix::fs::symlink;
|
||||||
|
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let external_output = dir.path().join("external-output");
|
||||||
|
let output = dir.path().join("flow");
|
||||||
|
fs::create_dir_all(&external_output).unwrap();
|
||||||
|
fs::write(external_output.join("notes.md"), "external").unwrap();
|
||||||
|
symlink(&external_output, &output).unwrap();
|
||||||
|
|
||||||
|
let err = write_trace_bundle(&output, &sample_trace("s1", "replacement")).unwrap_err();
|
||||||
|
|
||||||
|
assert!(err.to_string().contains("symlink"));
|
||||||
|
assert_eq!(
|
||||||
|
fs::read_to_string(external_output.join("notes.md")).unwrap(),
|
||||||
|
"external"
|
||||||
|
);
|
||||||
|
assert!(!external_output.join("trace.json").exists());
|
||||||
|
assert!(!external_output.join("pages").exists());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -141,9 +141,9 @@ pub fn info_for(code: ErrorCode) -> RenderInfo {
|
|||||||
exit_code: 2,
|
exit_code: 2,
|
||||||
},
|
},
|
||||||
ErrorCode::VersionTooOld => RenderInfo {
|
ErrorCode::VersionTooOld => RenderInfo {
|
||||||
summary: "peer version is too old to communicate with this build",
|
summary: "the installed bsk CLI and BrowserSkill extension are incompatible",
|
||||||
hint: Some(
|
hint: Some(
|
||||||
"upgrade both bsk CLI and the browser-skill extension so both sides satisfy `min_compatible_protocol`",
|
"update bsk and the BrowserSkill extension, reload the extension, then run `bsk daemon restart`",
|
||||||
),
|
),
|
||||||
exit_code: 5,
|
exit_code: 5,
|
||||||
},
|
},
|
||||||
@@ -157,7 +157,7 @@ pub fn info_for(code: ErrorCode) -> RenderInfo {
|
|||||||
ErrorCode::NoBrowserConnected => RenderInfo {
|
ErrorCode::NoBrowserConnected => RenderInfo {
|
||||||
summary: "no browser is connected to the daemon",
|
summary: "no browser is connected to the daemon",
|
||||||
hint: Some(
|
hint: Some(
|
||||||
"open the browser-skill extension in your browser and wait for the popup to show \"connected\"",
|
"open the BrowserSkill extension and wait for it to show \"connected\"; if the extension is already open, update and reload it, then run `bsk daemon restart`",
|
||||||
),
|
),
|
||||||
exit_code: 1,
|
exit_code: 1,
|
||||||
},
|
},
|
||||||
@@ -310,6 +310,30 @@ mod tests {
|
|||||||
assert_eq!(exit_code_for(ErrorCode::VersionTooOld), 5);
|
assert_eq!(exit_code_for(ErrorCode::VersionTooOld), 5);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn version_error_names_the_components_and_gives_update_steps() {
|
||||||
|
let info = info_for(ErrorCode::VersionTooOld);
|
||||||
|
assert!(info.summary.contains("bsk CLI"));
|
||||||
|
assert!(info.summary.contains("BrowserSkill extension"));
|
||||||
|
let hint = info
|
||||||
|
.hint
|
||||||
|
.expect("version mismatch should have a repair hint");
|
||||||
|
assert!(hint.contains("update bsk"));
|
||||||
|
assert!(hint.contains("reload the extension"));
|
||||||
|
assert!(hint.contains("bsk daemon restart"));
|
||||||
|
assert!(!hint.contains("min_compatible_protocol"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn no_browser_hint_explains_version_mismatch_recovery() {
|
||||||
|
let hint = info_for(ErrorCode::NoBrowserConnected)
|
||||||
|
.hint
|
||||||
|
.expect("no browser should have a repair hint");
|
||||||
|
assert!(hint.contains("extension is already open"));
|
||||||
|
assert!(hint.contains("update"));
|
||||||
|
assert!(hint.contains("bsk daemon restart"));
|
||||||
|
}
|
||||||
|
|
||||||
/// Every code must carry a non-empty summary; an empty `summary`
|
/// Every code must carry a non-empty summary; an empty `summary`
|
||||||
/// would render `error: ` with nothing after it.
|
/// would render `error: ` with nothing after it.
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -786,7 +786,19 @@ async fn handle_session_stop(state: &Arc<DaemonState>, params: Value) -> Result<
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Marks a `session.stop` rejected because another teardown already owns the
|
||||||
|
/// session, so callers that race on purpose (`bsk record`) can tell it apart
|
||||||
|
/// from a browser that stopped responding.
|
||||||
|
pub const STOP_IN_PROGRESS_REASON: &str = "session_stop_in_progress";
|
||||||
|
|
||||||
fn map_stop_error(err: StopSessionError) -> RpcError {
|
fn map_stop_error(err: StopSessionError) -> RpcError {
|
||||||
|
if matches!(err, StopSessionError::Stopping) {
|
||||||
|
return RpcError {
|
||||||
|
code: ErrorCode::Timeout,
|
||||||
|
message: err.to_string(),
|
||||||
|
data: Some(serde_json::json!({ "reason": STOP_IN_PROGRESS_REASON })),
|
||||||
|
};
|
||||||
|
}
|
||||||
let code = match &err {
|
let code = match &err {
|
||||||
StopSessionError::NotFound => ErrorCode::NotFound,
|
StopSessionError::NotFound => ErrorCode::NotFound,
|
||||||
StopSessionError::Stopping => ErrorCode::Timeout,
|
StopSessionError::Stopping => ErrorCode::Timeout,
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ use crate::daemon::{
|
|||||||
browsers::{BROWSER_LIVENESS_TICK, BROWSER_LIVENESS_TIMEOUT, EXTENSION_CONNECT_WAIT},
|
browsers::{BROWSER_LIVENESS_TICK, BROWSER_LIVENESS_TIMEOUT, EXTENSION_CONNECT_WAIT},
|
||||||
info as daemon_info, ipc, lockfile, paths,
|
info as daemon_info, ipc, lockfile, paths,
|
||||||
sessions::{StopSessionError, forget_session, stop_session},
|
sessions::{StopSessionError, forget_session, stop_session},
|
||||||
state::DaemonState,
|
state::{DaemonState, PROTOCOL_VERSION},
|
||||||
ws,
|
ws,
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -299,7 +299,7 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
|
|||||||
ws_port,
|
ws_port,
|
||||||
sock_path: sock_path.clone(),
|
sock_path: sock_path.clone(),
|
||||||
daemon_version: env!("CARGO_PKG_VERSION"),
|
daemon_version: env!("CARGO_PKG_VERSION"),
|
||||||
protocol_version: "1.0",
|
protocol_version: PROTOCOL_VERSION,
|
||||||
};
|
};
|
||||||
let handler = ipc::full_handler(status, Arc::clone(&state));
|
let handler = ipc::full_handler(status, Arc::clone(&state));
|
||||||
|
|
||||||
|
|||||||
@@ -16,9 +16,9 @@ use super::start::DaemonConfig;
|
|||||||
use super::ws::WsHandle;
|
use super::ws::WsHandle;
|
||||||
|
|
||||||
pub const DAEMON_VERSION: &str = env!("CARGO_PKG_VERSION");
|
pub const DAEMON_VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||||
pub const PROTOCOL_VERSION: &str = "1.0";
|
pub const PROTOCOL_VERSION: &str = "1.1";
|
||||||
/// Lowest **protocol** version peers must speak (e.g. `"1.0"`).
|
/// Lowest **protocol** version peers must speak (e.g. `"1.1"`).
|
||||||
pub const MIN_COMPATIBLE_PROTOCOL: &str = "1.0";
|
pub const MIN_COMPATIBLE_PROTOCOL: &str = "1.1";
|
||||||
/// Legacy app-semver floor used only when `HandshakeResult.min_compatible_peer`
|
/// Legacy app-semver floor used only when `HandshakeResult.min_compatible_peer`
|
||||||
/// is emitted for old extensions. New code ignores this on read.
|
/// is emitted for old extensions. New code ignores this on read.
|
||||||
pub const LEGACY_MIN_COMPATIBLE_PEER: &str = "0.0.0";
|
pub const LEGACY_MIN_COMPATIBLE_PEER: &str = "0.0.0";
|
||||||
|
|||||||
@@ -112,14 +112,14 @@ async fn handshake_as_ext(
|
|||||||
let params = HandshakeParams {
|
let params = HandshakeParams {
|
||||||
client: "browser-skill-extension".into(),
|
client: "browser-skill-extension".into(),
|
||||||
version: "0.1.0-dev.0".parse().unwrap(),
|
version: "0.1.0-dev.0".parse().unwrap(),
|
||||||
protocol_version: "1.0".into(),
|
protocol_version: "1.1".into(),
|
||||||
instance_id: TEST_EXT_ID.into(),
|
instance_id: TEST_EXT_ID.into(),
|
||||||
browser: BrowserPeerInfo {
|
browser: BrowserPeerInfo {
|
||||||
name: "chrome".into(),
|
name: "chrome".into(),
|
||||||
version: "131.0".into(),
|
version: "131.0".into(),
|
||||||
},
|
},
|
||||||
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
||||||
min_compatible_protocol: Some("1.0".into()),
|
min_compatible_protocol: Some("1.1".into()),
|
||||||
label: "Test".into(),
|
label: "Test".into(),
|
||||||
};
|
};
|
||||||
let req = RequestFrame {
|
let req = RequestFrame {
|
||||||
|
|||||||
@@ -75,7 +75,7 @@ async fn handshake_as_ext(
|
|||||||
let params = HandshakeParams {
|
let params = HandshakeParams {
|
||||||
client: "browser-skill-extension".into(),
|
client: "browser-skill-extension".into(),
|
||||||
version: "0.1.0-dev.0".parse().unwrap(),
|
version: "0.1.0-dev.0".parse().unwrap(),
|
||||||
protocol_version: "1.0".into(),
|
protocol_version: "1.1".into(),
|
||||||
instance_id: TEST_EXT_ID.into(),
|
instance_id: TEST_EXT_ID.into(),
|
||||||
browser: BrowserPeerInfo {
|
browser: BrowserPeerInfo {
|
||||||
name: "chrome".into(),
|
name: "chrome".into(),
|
||||||
@@ -83,7 +83,7 @@ async fn handshake_as_ext(
|
|||||||
},
|
},
|
||||||
label: "Test".into(),
|
label: "Test".into(),
|
||||||
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
||||||
min_compatible_protocol: Some("1.0".into()),
|
min_compatible_protocol: Some("1.1".into()),
|
||||||
};
|
};
|
||||||
let req = RequestFrame {
|
let req = RequestFrame {
|
||||||
id: "hs".into(),
|
id: "hs".into(),
|
||||||
|
|||||||
@@ -108,12 +108,12 @@ async fn send_handshake_with_floors(
|
|||||||
async fn handshake_ok_when_protocol_matches() {
|
async fn handshake_ok_when_protocol_matches() {
|
||||||
let (handle, _sock) = spawn_daemon().await;
|
let (handle, _sock) = spawn_daemon().await;
|
||||||
let mut ws = open_ws(handle.ws_addr()).await;
|
let mut ws = open_ws(handle.ws_addr()).await;
|
||||||
let resp = send_handshake(&mut ws, "1.0", env!("CARGO_PKG_VERSION")).await;
|
let resp = send_handshake(&mut ws, "1.1", env!("CARGO_PKG_VERSION")).await;
|
||||||
let result: HandshakeResult = match resp.body {
|
let result: HandshakeResult = match resp.body {
|
||||||
ResponseBody::Ok(v) => serde_json::from_value(v).unwrap(),
|
ResponseBody::Ok(v) => serde_json::from_value(v).unwrap(),
|
||||||
ResponseBody::Err(e) => panic!("expected ok handshake, got {e:?}"),
|
ResponseBody::Err(e) => panic!("expected ok handshake, got {e:?}"),
|
||||||
};
|
};
|
||||||
assert_eq!(result.protocol_version, "1.0");
|
assert_eq!(result.protocol_version, "1.1");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
result
|
result
|
||||||
.min_compatible_peer
|
.min_compatible_peer
|
||||||
@@ -124,7 +124,7 @@ async fn handshake_ok_when_protocol_matches() {
|
|||||||
);
|
);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
result.min_compatible_protocol.as_deref(),
|
result.min_compatible_protocol.as_deref(),
|
||||||
Some("1.0"),
|
Some("1.1"),
|
||||||
"daemon must advertise protocol floor for new extensions"
|
"daemon must advertise protocol floor for new extensions"
|
||||||
);
|
);
|
||||||
handle.shutdown().await;
|
handle.shutdown().await;
|
||||||
@@ -135,7 +135,7 @@ async fn handshake_ok_when_app_versions_differ_but_protocol_matches() {
|
|||||||
let (handle, _sock) = spawn_daemon().await;
|
let (handle, _sock) = spawn_daemon().await;
|
||||||
let mut ws = open_ws(handle.ws_addr()).await;
|
let mut ws = open_ws(handle.ws_addr()).await;
|
||||||
let resp =
|
let resp =
|
||||||
send_handshake_with_floors(&mut ws, "1.0", "9.9.9", Some("0.0.0"), Some("1.0")).await;
|
send_handshake_with_floors(&mut ws, "1.1", "9.9.9", Some("0.0.0"), Some("1.1")).await;
|
||||||
match resp.body {
|
match resp.body {
|
||||||
ResponseBody::Ok(_) => {}
|
ResponseBody::Ok(_) => {}
|
||||||
other => panic!("expected ok when protocol matches, got {other:?}"),
|
other => panic!("expected ok when protocol matches, got {other:?}"),
|
||||||
@@ -149,10 +149,10 @@ async fn handshake_skew_when_protocol_minor_differs() {
|
|||||||
let mut ws = open_ws(handle.ws_addr()).await;
|
let mut ws = open_ws(handle.ws_addr()).await;
|
||||||
let resp = send_handshake_with_floors(
|
let resp = send_handshake_with_floors(
|
||||||
&mut ws,
|
&mut ws,
|
||||||
"1.1",
|
"1.2",
|
||||||
env!("CARGO_PKG_VERSION"),
|
env!("CARGO_PKG_VERSION"),
|
||||||
Some("0.0.0"),
|
Some("0.0.0"),
|
||||||
Some("1.0"),
|
Some("1.1"),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
match resp.body {
|
match resp.body {
|
||||||
@@ -168,6 +168,28 @@ async fn handshake_skew_when_protocol_minor_differs() {
|
|||||||
handle.shutdown().await;
|
handle.shutdown().await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn handshake_rejects_pre_trace_v3_extension() {
|
||||||
|
let (handle, _sock) = spawn_daemon().await;
|
||||||
|
let mut ws = open_ws(handle.ws_addr()).await;
|
||||||
|
let resp = send_handshake_with_floors(
|
||||||
|
&mut ws,
|
||||||
|
"1.0",
|
||||||
|
env!("CARGO_PKG_VERSION"),
|
||||||
|
Some("0.0.0"),
|
||||||
|
Some("1.0"),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
match resp.body {
|
||||||
|
ResponseBody::Err(e) => {
|
||||||
|
assert_eq!(e.code, ErrorCode::VersionTooOld);
|
||||||
|
assert!(e.message.contains("below local min_compatible_protocol"));
|
||||||
|
}
|
||||||
|
other => panic!("pre-Trace-v3 extension must be rejected, got {other:?}"),
|
||||||
|
}
|
||||||
|
handle.shutdown().await;
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn handshake_rejected_on_protocol_major_mismatch() {
|
async fn handshake_rejected_on_protocol_major_mismatch() {
|
||||||
let (handle, _sock) = spawn_daemon().await;
|
let (handle, _sock) = spawn_daemon().await;
|
||||||
@@ -210,7 +232,7 @@ async fn handshake_legacy_ext_without_protocol_floor_still_ok() {
|
|||||||
let mut ws = open_ws(handle.ws_addr()).await;
|
let mut ws = open_ws(handle.ws_addr()).await;
|
||||||
let resp = send_handshake_with_floors(
|
let resp = send_handshake_with_floors(
|
||||||
&mut ws,
|
&mut ws,
|
||||||
"1.0",
|
"1.1",
|
||||||
env!("CARGO_PKG_VERSION"),
|
env!("CARGO_PKG_VERSION"),
|
||||||
Some("0.1.0"),
|
Some("0.1.0"),
|
||||||
None,
|
None,
|
||||||
@@ -235,7 +257,7 @@ async fn status_surfaces_version_skew_for_skewed_browser() {
|
|||||||
browser_name: "chrome".into(),
|
browser_name: "chrome".into(),
|
||||||
browser_version: "131.0".into(),
|
browser_version: "131.0".into(),
|
||||||
extension_version: "9.9.9".into(),
|
extension_version: "9.9.9".into(),
|
||||||
extension_protocol_version: "1.1".into(),
|
extension_protocol_version: "1.2".into(),
|
||||||
label: "Older".into(),
|
label: "Older".into(),
|
||||||
sink: bsk::daemon::browsers::BrowserSink { tx },
|
sink: bsk::daemon::browsers::BrowserSink { tx },
|
||||||
pending: Mutex::new(bsk::daemon::browsers::Pending::default()),
|
pending: Mutex::new(bsk::daemon::browsers::Pending::default()),
|
||||||
@@ -263,8 +285,8 @@ async fn status_surfaces_version_skew_for_skewed_browser() {
|
|||||||
.iter()
|
.iter()
|
||||||
.find(|s| s.instance_id == "skew-only-test")
|
.find(|s| s.instance_id == "skew-only-test")
|
||||||
.expect("status must list our skew client");
|
.expect("status must list our skew client");
|
||||||
assert_eq!(skew.client_protocol_version, "1.1");
|
assert_eq!(skew.client_protocol_version, "1.2");
|
||||||
assert_eq!(skew.server_protocol_version, "1.0");
|
assert_eq!(skew.server_protocol_version, "1.1");
|
||||||
assert_eq!(skew.client_version, "9.9.9");
|
assert_eq!(skew.client_version, "9.9.9");
|
||||||
let entry = status
|
let entry = status
|
||||||
.browsers
|
.browsers
|
||||||
@@ -281,7 +303,7 @@ async fn handshake_rejects_when_local_below_peer_min_compatible_protocol() {
|
|||||||
let mut ws = open_ws(handle.ws_addr()).await;
|
let mut ws = open_ws(handle.ws_addr()).await;
|
||||||
let resp = send_handshake_with_floors(
|
let resp = send_handshake_with_floors(
|
||||||
&mut ws,
|
&mut ws,
|
||||||
"1.0",
|
"1.1",
|
||||||
env!("CARGO_PKG_VERSION"),
|
env!("CARGO_PKG_VERSION"),
|
||||||
Some("0.0.0"),
|
Some("0.0.0"),
|
||||||
Some("99.0.0"),
|
Some("99.0.0"),
|
||||||
|
|||||||
@@ -81,14 +81,14 @@ async fn handshake_as_ext(
|
|||||||
let params = HandshakeParams {
|
let params = HandshakeParams {
|
||||||
client: "browser-skill-extension".into(),
|
client: "browser-skill-extension".into(),
|
||||||
version: "0.1.0-dev.0".parse().unwrap(),
|
version: "0.1.0-dev.0".parse().unwrap(),
|
||||||
protocol_version: "1.0".into(),
|
protocol_version: "1.1".into(),
|
||||||
instance_id: TEST_EXT_ID.into(),
|
instance_id: TEST_EXT_ID.into(),
|
||||||
browser: BrowserPeerInfo {
|
browser: BrowserPeerInfo {
|
||||||
name: "chrome".into(),
|
name: "chrome".into(),
|
||||||
version: "131.0".into(),
|
version: "131.0".into(),
|
||||||
},
|
},
|
||||||
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
||||||
min_compatible_protocol: Some("1.0".into()),
|
min_compatible_protocol: Some("1.1".into()),
|
||||||
label: "Test".into(),
|
label: "Test".into(),
|
||||||
};
|
};
|
||||||
let req = RequestFrame {
|
let req = RequestFrame {
|
||||||
|
|||||||
@@ -0,0 +1,272 @@
|
|||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::sync::Arc;
|
||||||
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
use bsk::cli::record_state;
|
||||||
|
use bsk::daemon::info::DaemonInfo;
|
||||||
|
use bsk::daemon::paths::BSK_HOME_ENV;
|
||||||
|
use bsk::daemon::{self, DaemonConfig};
|
||||||
|
use bsk::ipc_client::IpcClient;
|
||||||
|
use bsk_protocol::system::{HandshakeParams, HandshakeResult};
|
||||||
|
use bsk_protocol::tools::{
|
||||||
|
RecordStopResult, RecorderInfo, SessionStartParams, SessionStartResult, StopReason, Trace,
|
||||||
|
TraceEntry, TraceState, VOM_FORMAT_VERSION,
|
||||||
|
};
|
||||||
|
use bsk_protocol::{
|
||||||
|
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
|
||||||
|
};
|
||||||
|
use futures_util::{SinkExt, StreamExt};
|
||||||
|
use serde_json::json;
|
||||||
|
use tokio::process::Command;
|
||||||
|
use tokio_tungstenite::tungstenite::handshake::client::generate_key;
|
||||||
|
use tokio_tungstenite::tungstenite::http::Request;
|
||||||
|
use tokio_tungstenite::tungstenite::protocol::Message;
|
||||||
|
|
||||||
|
const TEST_EXT_ID: &str = "abcdefghijklmnopabcdefghijklmnop";
|
||||||
|
|
||||||
|
type Ws =
|
||||||
|
tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>>;
|
||||||
|
|
||||||
|
fn bsk_bin() -> PathBuf {
|
||||||
|
PathBuf::from(env!("CARGO_BIN_EXE_bsk"))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn connect_extension(addr: std::net::SocketAddr) -> Ws {
|
||||||
|
let origin = format!("chrome-extension://{TEST_EXT_ID}");
|
||||||
|
let request = Request::builder()
|
||||||
|
.method("GET")
|
||||||
|
.uri(format!("ws://{addr}/"))
|
||||||
|
.header("Host", addr.to_string())
|
||||||
|
.header("Upgrade", "websocket")
|
||||||
|
.header("Connection", "Upgrade")
|
||||||
|
.header("Sec-WebSocket-Version", "13")
|
||||||
|
.header("Sec-WebSocket-Key", generate_key())
|
||||||
|
.header("Origin", origin)
|
||||||
|
.body(())
|
||||||
|
.unwrap();
|
||||||
|
tokio_tungstenite::connect_async(request).await.unwrap().0
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn handshake(ws: &mut Ws) {
|
||||||
|
let request = RequestFrame {
|
||||||
|
id: "handshake".into(),
|
||||||
|
method: Method::SystemHandshake,
|
||||||
|
params: Some(
|
||||||
|
serde_json::to_value(HandshakeParams {
|
||||||
|
client: "browser-skill-extension".into(),
|
||||||
|
version: "0.1.10".parse().unwrap(),
|
||||||
|
protocol_version: "1.1".into(),
|
||||||
|
instance_id: TEST_EXT_ID.into(),
|
||||||
|
browser: BrowserPeerInfo {
|
||||||
|
name: "chrome".into(),
|
||||||
|
version: "131.0".into(),
|
||||||
|
},
|
||||||
|
min_compatible_peer: Some("0.1.10".parse().unwrap()),
|
||||||
|
min_compatible_protocol: Some("1.1".into()),
|
||||||
|
label: "Record stop retry test".into(),
|
||||||
|
})
|
||||||
|
.unwrap(),
|
||||||
|
),
|
||||||
|
};
|
||||||
|
ws.send(Message::Text(serde_json::to_string(&request).unwrap()))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let Message::Text(response) = ws.next().await.unwrap().unwrap() else {
|
||||||
|
panic!("expected text handshake response");
|
||||||
|
};
|
||||||
|
let response: ResponseFrame = serde_json::from_str(&response).unwrap();
|
||||||
|
let ResponseBody::Ok(value) = response.body else {
|
||||||
|
panic!("handshake failed");
|
||||||
|
};
|
||||||
|
let _: HandshakeResult = serde_json::from_value(value).unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn sample_trace() -> Trace {
|
||||||
|
Trace {
|
||||||
|
version: 3,
|
||||||
|
purpose: Some("retry record stop".into()),
|
||||||
|
started_at: Some("2026-08-12T10:00:00Z".into()),
|
||||||
|
recorded_at: "2026-08-12T10:01:00Z".into(),
|
||||||
|
stopped_by: StopReason::CliStop,
|
||||||
|
entry: TraceEntry {
|
||||||
|
start_url: "https://example.com/".into(),
|
||||||
|
},
|
||||||
|
recorder: RecorderInfo {
|
||||||
|
bsk: "0.1.10".into(),
|
||||||
|
vom: VOM_FORMAT_VERSION,
|
||||||
|
},
|
||||||
|
frame_capture: None,
|
||||||
|
states: vec![TraceState {
|
||||||
|
id: "s1".into(),
|
||||||
|
url: "https://example.com/".into(),
|
||||||
|
title: Some("Example".into()),
|
||||||
|
body: Some("@vom 1\nL1 page".into()),
|
||||||
|
page: None,
|
||||||
|
truncated: false,
|
||||||
|
}],
|
||||||
|
steps: vec![],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run_extension(
|
||||||
|
mut ws: Ws,
|
||||||
|
record_stop_calls: Arc<AtomicUsize>,
|
||||||
|
session_stop_calls: Arc<AtomicUsize>,
|
||||||
|
) {
|
||||||
|
tokio::spawn(async move {
|
||||||
|
while let Some(Ok(Message::Text(text))) = ws.next().await {
|
||||||
|
let Ok(Frame::Request(request)) = serde_json::from_str::<Frame>(&text) else {
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
let body = match request.method {
|
||||||
|
Method::ToolSessionStart => {
|
||||||
|
let _: SessionStartParams =
|
||||||
|
serde_json::from_value(request.params.clone().unwrap()).unwrap();
|
||||||
|
ResponseBody::Ok(
|
||||||
|
serde_json::to_value(SessionStartResult {
|
||||||
|
agent_window_id: Some(100),
|
||||||
|
})
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
Method::ToolRecordStop => {
|
||||||
|
if record_stop_calls.fetch_add(1, Ordering::SeqCst) == 0 {
|
||||||
|
ResponseBody::Err(RpcError {
|
||||||
|
code: ErrorCode::ProtocolError,
|
||||||
|
message: "recording remains active; retry record stop".into(),
|
||||||
|
data: Some(json!({ "recording": true, "retryable": true })),
|
||||||
|
})
|
||||||
|
} else {
|
||||||
|
ResponseBody::Ok(
|
||||||
|
serde_json::to_value(RecordStopResult {
|
||||||
|
trace: sample_trace(),
|
||||||
|
})
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Method::ToolSessionStop => {
|
||||||
|
session_stop_calls.fetch_add(1, Ordering::SeqCst);
|
||||||
|
ResponseBody::Ok(json!({}))
|
||||||
|
}
|
||||||
|
_ => ResponseBody::Err(RpcError {
|
||||||
|
code: ErrorCode::ProtocolError,
|
||||||
|
message: format!("unexpected method {:?}", request.method),
|
||||||
|
data: None,
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
let response = ResponseFrame {
|
||||||
|
id: request.id,
|
||||||
|
body,
|
||||||
|
};
|
||||||
|
ws.send(Message::Text(serde_json::to_string(&response).unwrap()))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn start_session(sock: &Path) -> String {
|
||||||
|
#[derive(serde::Deserialize)]
|
||||||
|
struct StartResult {
|
||||||
|
session_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut client = IpcClient::connect(sock).await.unwrap();
|
||||||
|
client
|
||||||
|
.call::<(), StartResult>(
|
||||||
|
"session-start",
|
||||||
|
Method::SessionStart,
|
||||||
|
None,
|
||||||
|
Duration::from_secs(5),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.session_id
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn run_record_stop(home: &Path, output: &Path) -> std::process::Output {
|
||||||
|
Command::new(bsk_bin())
|
||||||
|
.arg("record")
|
||||||
|
.arg("stop")
|
||||||
|
.arg("--output")
|
||||||
|
.arg(output)
|
||||||
|
.env(BSK_HOME_ENV, home)
|
||||||
|
.output()
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn failed_record_stop_keeps_state_and_session_for_retry() {
|
||||||
|
let temp = tempfile::Builder::new()
|
||||||
|
.prefix(".record-stop-retry-")
|
||||||
|
.tempdir_in(env!("CARGO_MANIFEST_DIR"))
|
||||||
|
.unwrap();
|
||||||
|
let home = temp.path().join("bsk-home");
|
||||||
|
let sock = temp.path().join("daemon.sock");
|
||||||
|
let output = temp.path().join("trace");
|
||||||
|
|
||||||
|
let daemon = daemon::run(DaemonConfig::new(0), Some(sock.clone()))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
bsk::daemon::info::write_to_path(
|
||||||
|
&DaemonInfo::now(
|
||||||
|
std::process::id(),
|
||||||
|
sock.clone(),
|
||||||
|
daemon.ws_addr().port(),
|
||||||
|
env!("CARGO_PKG_VERSION"),
|
||||||
|
),
|
||||||
|
&home.join("daemon.json"),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
let mut extension = connect_extension(daemon.ws_addr()).await;
|
||||||
|
handshake(&mut extension).await;
|
||||||
|
let record_stop_calls = Arc::new(AtomicUsize::new(0));
|
||||||
|
let session_stop_calls = Arc::new(AtomicUsize::new(0));
|
||||||
|
run_extension(
|
||||||
|
extension,
|
||||||
|
Arc::clone(&record_stop_calls),
|
||||||
|
Arc::clone(&session_stop_calls),
|
||||||
|
);
|
||||||
|
|
||||||
|
let session_id = start_session(&sock).await;
|
||||||
|
unsafe {
|
||||||
|
std::env::set_var(BSK_HOME_ENV, &home);
|
||||||
|
}
|
||||||
|
record_state::write(&session_id).unwrap();
|
||||||
|
|
||||||
|
let first = run_record_stop(&home, &output).await;
|
||||||
|
assert!(
|
||||||
|
!first.status.success(),
|
||||||
|
"first stop unexpectedly succeeded: {}",
|
||||||
|
String::from_utf8_lossy(&first.stdout)
|
||||||
|
);
|
||||||
|
assert_eq!(record_state::read().unwrap().session_id, session_id);
|
||||||
|
assert_eq!(record_stop_calls.load(Ordering::SeqCst), 1);
|
||||||
|
assert_eq!(
|
||||||
|
session_stop_calls.load(Ordering::SeqCst),
|
||||||
|
0,
|
||||||
|
"failed record stop must leave the session active"
|
||||||
|
);
|
||||||
|
|
||||||
|
let second = run_record_stop(&home, &output).await;
|
||||||
|
assert!(
|
||||||
|
second.status.success(),
|
||||||
|
"retry failed: {}",
|
||||||
|
String::from_utf8_lossy(&second.stderr)
|
||||||
|
);
|
||||||
|
assert!(record_state::read().is_err());
|
||||||
|
assert_eq!(record_stop_calls.load(Ordering::SeqCst), 2);
|
||||||
|
assert_eq!(session_stop_calls.load(Ordering::SeqCst), 1);
|
||||||
|
assert!(output.join("trace.json").exists());
|
||||||
|
assert!(output.join("pages/s1.vom.txt").exists());
|
||||||
|
|
||||||
|
unsafe {
|
||||||
|
std::env::remove_var(BSK_HOME_ENV);
|
||||||
|
}
|
||||||
|
daemon.shutdown().await;
|
||||||
|
}
|
||||||
@@ -81,7 +81,7 @@ async fn handshake_as_ext(
|
|||||||
let params = HandshakeParams {
|
let params = HandshakeParams {
|
||||||
client: "browser-skill-extension".into(),
|
client: "browser-skill-extension".into(),
|
||||||
version: "0.1.0-dev.0".parse().unwrap(),
|
version: "0.1.0-dev.0".parse().unwrap(),
|
||||||
protocol_version: "1.0".into(),
|
protocol_version: "1.1".into(),
|
||||||
instance_id: TEST_EXT_ID.into(),
|
instance_id: TEST_EXT_ID.into(),
|
||||||
browser: BrowserPeerInfo {
|
browser: BrowserPeerInfo {
|
||||||
name: "chrome".into(),
|
name: "chrome".into(),
|
||||||
@@ -89,7 +89,7 @@ async fn handshake_as_ext(
|
|||||||
},
|
},
|
||||||
label: "Test".into(),
|
label: "Test".into(),
|
||||||
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
||||||
min_compatible_protocol: Some("1.0".into()),
|
min_compatible_protocol: Some("1.1".into()),
|
||||||
};
|
};
|
||||||
let req = RequestFrame {
|
let req = RequestFrame {
|
||||||
id: "hs".into(),
|
id: "hs".into(),
|
||||||
|
|||||||
@@ -84,14 +84,14 @@ async fn handshake_as_ext(
|
|||||||
let params = HandshakeParams {
|
let params = HandshakeParams {
|
||||||
client: "browser-skill-extension".into(),
|
client: "browser-skill-extension".into(),
|
||||||
version: "0.1.0-dev.0".parse().unwrap(),
|
version: "0.1.0-dev.0".parse().unwrap(),
|
||||||
protocol_version: "1.0".into(),
|
protocol_version: "1.1".into(),
|
||||||
instance_id: TEST_EXT_ID.into(),
|
instance_id: TEST_EXT_ID.into(),
|
||||||
browser: BrowserPeerInfo {
|
browser: BrowserPeerInfo {
|
||||||
name: "chrome".into(),
|
name: "chrome".into(),
|
||||||
version: "131.0".into(),
|
version: "131.0".into(),
|
||||||
},
|
},
|
||||||
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
||||||
min_compatible_protocol: Some("1.0".into()),
|
min_compatible_protocol: Some("1.1".into()),
|
||||||
label: "Test".into(),
|
label: "Test".into(),
|
||||||
};
|
};
|
||||||
let req = RequestFrame {
|
let req = RequestFrame {
|
||||||
@@ -400,14 +400,14 @@ async fn connect_second_ext(
|
|||||||
let params = HandshakeParams {
|
let params = HandshakeParams {
|
||||||
client: "browser-skill-extension".into(),
|
client: "browser-skill-extension".into(),
|
||||||
version: "0.1.0-dev.0".parse().unwrap(),
|
version: "0.1.0-dev.0".parse().unwrap(),
|
||||||
protocol_version: "1.0".into(),
|
protocol_version: "1.1".into(),
|
||||||
instance_id: instance_id.into(),
|
instance_id: instance_id.into(),
|
||||||
browser: BrowserPeerInfo {
|
browser: BrowserPeerInfo {
|
||||||
name: "edge".into(),
|
name: "edge".into(),
|
||||||
version: "130".into(),
|
version: "130".into(),
|
||||||
},
|
},
|
||||||
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
||||||
min_compatible_protocol: Some("1.0".into()),
|
min_compatible_protocol: Some("1.1".into()),
|
||||||
label: label.into(),
|
label: label.into(),
|
||||||
};
|
};
|
||||||
let hs = RequestFrame {
|
let hs = RequestFrame {
|
||||||
|
|||||||
@@ -61,7 +61,7 @@ fn bsk_status_json_returns_structured_payload() {
|
|||||||
|
|
||||||
assert!(parsed["pid"].as_u64().unwrap() > 0);
|
assert!(parsed["pid"].as_u64().unwrap() > 0);
|
||||||
assert!(!parsed["daemon_version"].as_str().unwrap().is_empty());
|
assert!(!parsed["daemon_version"].as_str().unwrap().is_empty());
|
||||||
assert_eq!(parsed["protocol_version"], "1.0");
|
assert_eq!(parsed["protocol_version"], "1.1");
|
||||||
assert!(parsed["sock_path"].as_str().is_some());
|
assert!(parsed["sock_path"].as_str().is_some());
|
||||||
assert_eq!(parsed["browsers"], serde_json::json!([]));
|
assert_eq!(parsed["browsers"], serde_json::json!([]));
|
||||||
assert_eq!(parsed["sessions"], serde_json::json!([]));
|
assert_eq!(parsed["sessions"], serde_json::json!([]));
|
||||||
|
|||||||
@@ -73,14 +73,14 @@ async fn do_handshake(ws: &mut Ws) -> HandshakeResult {
|
|||||||
let params = HandshakeParams {
|
let params = HandshakeParams {
|
||||||
client: "browser-skill-extension".into(),
|
client: "browser-skill-extension".into(),
|
||||||
version: "0.1.0-dev.0".parse().unwrap(),
|
version: "0.1.0-dev.0".parse().unwrap(),
|
||||||
protocol_version: "1.0".into(),
|
protocol_version: "1.1".into(),
|
||||||
instance_id: TEST_EXT_ID.into(),
|
instance_id: TEST_EXT_ID.into(),
|
||||||
browser: BrowserPeerInfo {
|
browser: BrowserPeerInfo {
|
||||||
name: "chrome".into(),
|
name: "chrome".into(),
|
||||||
version: "131.0".into(),
|
version: "131.0".into(),
|
||||||
},
|
},
|
||||||
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
||||||
min_compatible_protocol: Some("1.0".into()),
|
min_compatible_protocol: Some("1.1".into()),
|
||||||
label: "Test".into(),
|
label: "Test".into(),
|
||||||
};
|
};
|
||||||
let req = RequestFrame {
|
let req = RequestFrame {
|
||||||
|
|||||||
@@ -76,14 +76,14 @@ async fn do_handshake(ws: &mut Ws) -> HandshakeResult {
|
|||||||
let params = HandshakeParams {
|
let params = HandshakeParams {
|
||||||
client: "browser-skill-extension".into(),
|
client: "browser-skill-extension".into(),
|
||||||
version: "0.1.0-dev.0".parse().unwrap(),
|
version: "0.1.0-dev.0".parse().unwrap(),
|
||||||
protocol_version: "1.0".into(),
|
protocol_version: "1.1".into(),
|
||||||
instance_id: TEST_EXT_ID.into(),
|
instance_id: TEST_EXT_ID.into(),
|
||||||
browser: BrowserPeerInfo {
|
browser: BrowserPeerInfo {
|
||||||
name: "chrome".into(),
|
name: "chrome".into(),
|
||||||
version: "131.0".into(),
|
version: "131.0".into(),
|
||||||
},
|
},
|
||||||
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
||||||
min_compatible_protocol: Some("1.0".into()),
|
min_compatible_protocol: Some("1.1".into()),
|
||||||
label: "Test".into(),
|
label: "Test".into(),
|
||||||
};
|
};
|
||||||
let req = RequestFrame {
|
let req = RequestFrame {
|
||||||
|
|||||||
@@ -75,14 +75,14 @@ async fn do_handshake(ws: &mut Ws) -> HandshakeResult {
|
|||||||
let params = HandshakeParams {
|
let params = HandshakeParams {
|
||||||
client: "browser-skill-extension".into(),
|
client: "browser-skill-extension".into(),
|
||||||
version: "0.1.0-dev.0".parse().unwrap(),
|
version: "0.1.0-dev.0".parse().unwrap(),
|
||||||
protocol_version: "1.0".into(),
|
protocol_version: "1.1".into(),
|
||||||
instance_id: TEST_EXT_ID.into(),
|
instance_id: TEST_EXT_ID.into(),
|
||||||
browser: BrowserPeerInfo {
|
browser: BrowserPeerInfo {
|
||||||
name: "chrome".into(),
|
name: "chrome".into(),
|
||||||
version: "131.0".into(),
|
version: "131.0".into(),
|
||||||
},
|
},
|
||||||
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
||||||
min_compatible_protocol: Some("1.0".into()),
|
min_compatible_protocol: Some("1.1".into()),
|
||||||
label: "Test".into(),
|
label: "Test".into(),
|
||||||
};
|
};
|
||||||
let req = RequestFrame {
|
let req = RequestFrame {
|
||||||
|
|||||||
@@ -85,14 +85,14 @@ async fn do_handshake(ws: &mut Ws) -> HandshakeResult {
|
|||||||
let params = HandshakeParams {
|
let params = HandshakeParams {
|
||||||
client: "browser-skill-extension".into(),
|
client: "browser-skill-extension".into(),
|
||||||
version: "0.1.0-dev.0".parse().unwrap(),
|
version: "0.1.0-dev.0".parse().unwrap(),
|
||||||
protocol_version: "1.0".into(),
|
protocol_version: "1.1".into(),
|
||||||
instance_id: TEST_EXT_ID.into(),
|
instance_id: TEST_EXT_ID.into(),
|
||||||
browser: BrowserPeerInfo {
|
browser: BrowserPeerInfo {
|
||||||
name: "chrome".into(),
|
name: "chrome".into(),
|
||||||
version: "131.0".into(),
|
version: "131.0".into(),
|
||||||
},
|
},
|
||||||
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
||||||
min_compatible_protocol: Some("1.0".into()),
|
min_compatible_protocol: Some("1.1".into()),
|
||||||
label: "Test".into(),
|
label: "Test".into(),
|
||||||
};
|
};
|
||||||
let req = RequestFrame {
|
let req = RequestFrame {
|
||||||
|
|||||||
@@ -53,14 +53,14 @@ pub async fn send_handshake(
|
|||||||
let params = HandshakeParams {
|
let params = HandshakeParams {
|
||||||
client: "browser-skill-extension".into(),
|
client: "browser-skill-extension".into(),
|
||||||
version: "0.1.0-dev.0".parse().unwrap(),
|
version: "0.1.0-dev.0".parse().unwrap(),
|
||||||
protocol_version: "1.0".into(),
|
protocol_version: "1.1".into(),
|
||||||
instance_id: instance_id.into(),
|
instance_id: instance_id.into(),
|
||||||
browser: BrowserPeerInfo {
|
browser: BrowserPeerInfo {
|
||||||
name: "chrome".into(),
|
name: "chrome".into(),
|
||||||
version: "131.0".into(),
|
version: "131.0".into(),
|
||||||
},
|
},
|
||||||
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
||||||
min_compatible_protocol: Some("1.0".into()),
|
min_compatible_protocol: Some("1.1".into()),
|
||||||
label: "Test Chrome".into(),
|
label: "Test Chrome".into(),
|
||||||
};
|
};
|
||||||
let req = RequestFrame {
|
let req = RequestFrame {
|
||||||
@@ -91,7 +91,7 @@ async fn ws_handshake_registers_browser_in_state() {
|
|||||||
let mut ws = connect_ext(handle.ws_addr(), &origin).await;
|
let mut ws = connect_ext(handle.ws_addr(), &origin).await;
|
||||||
let result = send_handshake(&mut ws, TEST_EXT_ID).await;
|
let result = send_handshake(&mut ws, TEST_EXT_ID).await;
|
||||||
assert_eq!(result.server, "browser-skill-daemon");
|
assert_eq!(result.server, "browser-skill-daemon");
|
||||||
assert_eq!(result.protocol_version, "1.0");
|
assert_eq!(result.protocol_version, "1.1");
|
||||||
|
|
||||||
let state = handle.state();
|
let state = handle.state();
|
||||||
let browsers = state.browsers.snapshot();
|
let browsers = state.browsers.snapshot();
|
||||||
|
|||||||
@@ -11,6 +11,14 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"definitions": {
|
"definitions": {
|
||||||
|
"FillCommit": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"enter",
|
||||||
|
"suggestion",
|
||||||
|
"blur"
|
||||||
|
]
|
||||||
|
},
|
||||||
"KeyModifier": {
|
"KeyModifier": {
|
||||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||||
"type": "string",
|
"type": "string",
|
||||||
@@ -21,25 +29,32 @@
|
|||||||
"shift"
|
"shift"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"PageRef": {
|
"NavigationCause": {
|
||||||
"description": "Page context dictionary entry — referenced by steps via `page` id.",
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"user_typed",
|
||||||
|
"link",
|
||||||
|
"form_submit",
|
||||||
|
"reload",
|
||||||
|
"history",
|
||||||
|
"script",
|
||||||
|
"browser"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"RecorderInfo": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
"required": [
|
||||||
"id",
|
"bsk",
|
||||||
"url"
|
"vom"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"id": {
|
"bsk": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"title": {
|
"vom": {
|
||||||
"type": [
|
"type": "integer",
|
||||||
"string",
|
"format": "uint32",
|
||||||
"null"
|
"minimum": 0.0
|
||||||
]
|
|
||||||
},
|
|
||||||
"url": {
|
|
||||||
"type": "string"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -68,21 +83,16 @@
|
|||||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
"required": [
|
||||||
|
"cause",
|
||||||
"id",
|
"id",
|
||||||
"op",
|
"op",
|
||||||
"page",
|
"result",
|
||||||
|
"state",
|
||||||
"to"
|
"to"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"effect": {
|
"cause": {
|
||||||
"anyOf": [
|
"$ref": "#/definitions/NavigationCause"
|
||||||
{
|
|
||||||
"$ref": "#/definitions/StepEffect"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "null"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
},
|
||||||
"id": {
|
"id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
@@ -95,8 +105,11 @@
|
|||||||
"navigate"
|
"navigate"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"page": {
|
"result": {
|
||||||
"description": "Reference into `pages[]`.",
|
"$ref": "#/definitions/StepResult"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"to": {
|
"to": {
|
||||||
@@ -110,20 +123,11 @@
|
|||||||
"required": [
|
"required": [
|
||||||
"id",
|
"id",
|
||||||
"op",
|
"op",
|
||||||
"page",
|
"result",
|
||||||
|
"state",
|
||||||
"target"
|
"target"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"effect": {
|
|
||||||
"anyOf": [
|
|
||||||
{
|
|
||||||
"$ref": "#/definitions/StepEffect"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "null"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"id": {
|
"id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
"format": "uint32",
|
"format": "uint32",
|
||||||
@@ -135,8 +139,11 @@
|
|||||||
"click"
|
"click"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"page": {
|
"result": {
|
||||||
"description": "Reference into `pages[]`.",
|
"$ref": "#/definitions/StepResult"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"target": {
|
"target": {
|
||||||
@@ -150,20 +157,11 @@
|
|||||||
"required": [
|
"required": [
|
||||||
"id",
|
"id",
|
||||||
"op",
|
"op",
|
||||||
"page",
|
"result",
|
||||||
|
"state",
|
||||||
"target"
|
"target"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"effect": {
|
|
||||||
"anyOf": [
|
|
||||||
{
|
|
||||||
"$ref": "#/definitions/StepEffect"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "null"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"id": {
|
"id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
"format": "uint32",
|
"format": "uint32",
|
||||||
@@ -175,8 +173,11 @@
|
|||||||
"hover"
|
"hover"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"page": {
|
"result": {
|
||||||
"description": "Reference into `pages[]`.",
|
"$ref": "#/definitions/StepResult"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"target": {
|
"target": {
|
||||||
@@ -188,22 +189,17 @@
|
|||||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
"required": [
|
||||||
|
"commit",
|
||||||
"id",
|
"id",
|
||||||
"op",
|
"op",
|
||||||
"page",
|
"result",
|
||||||
|
"state",
|
||||||
"target",
|
"target",
|
||||||
"value"
|
"value"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"effect": {
|
"commit": {
|
||||||
"anyOf": [
|
"$ref": "#/definitions/FillCommit"
|
||||||
{
|
|
||||||
"$ref": "#/definitions/StepEffect"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "null"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
},
|
||||||
"id": {
|
"id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
@@ -216,15 +212,15 @@
|
|||||||
"fill"
|
"fill"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"page": {
|
|
||||||
"description": "Reference into `pages[]`.",
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"redacted": {
|
"redacted": {
|
||||||
"type": [
|
"type": "boolean"
|
||||||
"boolean",
|
},
|
||||||
"null"
|
"result": {
|
||||||
]
|
"$ref": "#/definitions/StepResult"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
|
"type": "string"
|
||||||
},
|
},
|
||||||
"target": {
|
"target": {
|
||||||
"$ref": "#/definitions/TargetDescriptor"
|
"$ref": "#/definitions/TargetDescriptor"
|
||||||
@@ -240,21 +236,11 @@
|
|||||||
"required": [
|
"required": [
|
||||||
"id",
|
"id",
|
||||||
"op",
|
"op",
|
||||||
"page",
|
"result",
|
||||||
"selection",
|
"state",
|
||||||
"target"
|
"target"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"effect": {
|
|
||||||
"anyOf": [
|
|
||||||
{
|
|
||||||
"$ref": "#/definitions/StepEffect"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "null"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"id": {
|
"id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
"format": "uint32",
|
"format": "uint32",
|
||||||
@@ -266,9 +252,8 @@
|
|||||||
"select"
|
"select"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"page": {
|
"result": {
|
||||||
"description": "Reference into `pages[]`.",
|
"$ref": "#/definitions/StepResult"
|
||||||
"type": "string"
|
|
||||||
},
|
},
|
||||||
"selection": {
|
"selection": {
|
||||||
"type": "array",
|
"type": "array",
|
||||||
@@ -276,6 +261,10 @@
|
|||||||
"$ref": "#/definitions/SelectedOption"
|
"$ref": "#/definitions/SelectedOption"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
"target": {
|
"target": {
|
||||||
"$ref": "#/definitions/TargetDescriptor"
|
"$ref": "#/definitions/TargetDescriptor"
|
||||||
}
|
}
|
||||||
@@ -288,19 +277,10 @@
|
|||||||
"id",
|
"id",
|
||||||
"key",
|
"key",
|
||||||
"op",
|
"op",
|
||||||
"page"
|
"result",
|
||||||
|
"state"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"effect": {
|
|
||||||
"anyOf": [
|
|
||||||
{
|
|
||||||
"$ref": "#/definitions/StepEffect"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "null"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"id": {
|
"id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
"format": "uint32",
|
"format": "uint32",
|
||||||
@@ -324,8 +304,11 @@
|
|||||||
"press"
|
"press"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"page": {
|
"result": {
|
||||||
"description": "Reference into `pages[]`.",
|
"$ref": "#/definitions/StepResult"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"target": {
|
"target": {
|
||||||
@@ -339,48 +322,74 @@
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"id",
|
||||||
|
"op",
|
||||||
|
"result",
|
||||||
|
"state"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"id": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "uint32",
|
||||||
|
"minimum": 0.0
|
||||||
|
},
|
||||||
|
"op": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"scroll"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"result": {
|
||||||
|
"$ref": "#/definitions/StepResult"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"StepEffect": {
|
"StepResult": {
|
||||||
"description": "Observed navigation after a step (objective fact only).",
|
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
"required": [
|
||||||
"navigated_to"
|
"state"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"navigated_to": {
|
"state": {
|
||||||
"description": "Reference into `pages[]` for the destination page.",
|
|
||||||
"type": "string"
|
"type": "string"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"StopReason": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"user_finish",
|
||||||
|
"cli_stop"
|
||||||
|
]
|
||||||
|
},
|
||||||
"TargetDescriptor": {
|
"TargetDescriptor": {
|
||||||
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
|
"description": "Stable semantic handle for an interacted element within a page observation.",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
|
||||||
"tag"
|
|
||||||
],
|
|
||||||
"properties": {
|
"properties": {
|
||||||
|
"ctx": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
"name": {
|
"name": {
|
||||||
"type": [
|
"type": [
|
||||||
"string",
|
"string",
|
||||||
"null"
|
"null"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"name_attr": {
|
"ref": {
|
||||||
"type": [
|
|
||||||
"string",
|
|
||||||
"null"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"nearby_label": {
|
|
||||||
"type": [
|
|
||||||
"string",
|
|
||||||
"null"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"placeholder": {
|
|
||||||
"type": [
|
"type": [
|
||||||
"string",
|
"string",
|
||||||
"null"
|
"null"
|
||||||
@@ -392,8 +401,8 @@
|
|||||||
"null"
|
"null"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"tag": {
|
"unmatched": {
|
||||||
"type": "string"
|
"type": "boolean"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -402,42 +411,54 @@
|
|||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
"required": [
|
||||||
"entry",
|
"entry",
|
||||||
"pages",
|
|
||||||
"recorded_at",
|
"recorded_at",
|
||||||
"steps"
|
"recorder",
|
||||||
|
"states",
|
||||||
|
"steps",
|
||||||
|
"stopped_by",
|
||||||
|
"version"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"entry": {
|
"entry": {
|
||||||
"$ref": "#/definitions/TraceEntry"
|
"$ref": "#/definitions/TraceEntry"
|
||||||
},
|
},
|
||||||
"pages": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {
|
|
||||||
"$ref": "#/definitions/PageRef"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"purpose": {
|
"purpose": {
|
||||||
"description": "Optional user-provided goal from `--purpose` (metadata only).",
|
|
||||||
"type": [
|
"type": [
|
||||||
"string",
|
"string",
|
||||||
"null"
|
"null"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"recorded_at": {
|
"recorded_at": {
|
||||||
"description": "RFC 3339 timestamp when recording stopped.",
|
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
|
"recorder": {
|
||||||
|
"$ref": "#/definitions/RecorderInfo"
|
||||||
|
},
|
||||||
"started_at": {
|
"started_at": {
|
||||||
"type": [
|
"type": [
|
||||||
"string",
|
"string",
|
||||||
"null"
|
"null"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
"states": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/TraceState"
|
||||||
|
}
|
||||||
|
},
|
||||||
"steps": {
|
"steps": {
|
||||||
"type": "array",
|
"type": "array",
|
||||||
"items": {
|
"items": {
|
||||||
"$ref": "#/definitions/Step"
|
"$ref": "#/definitions/Step"
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
"stopped_by": {
|
||||||
|
"$ref": "#/definitions/StopReason"
|
||||||
|
},
|
||||||
|
"version": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "uint32",
|
||||||
|
"minimum": 0.0
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -452,6 +473,45 @@
|
|||||||
"type": "string"
|
"type": "string"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
"TraceState": {
|
||||||
|
"description": "Page observation dictionary entry — referenced by steps via `state` / `result.state`.",
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"id",
|
||||||
|
"url"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"body": {
|
||||||
|
"description": "Wire-only: full page observation (front matter + VOM body + annotations).",
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"id": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"page": {
|
||||||
|
"description": "Disk-only: filename under the bundle `pages/` directory.",
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"title": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"truncated": {
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
|
"url": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,12 +6,26 @@
|
|||||||
"session_id"
|
"session_id"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
|
"max_page_tokens": {
|
||||||
|
"type": [
|
||||||
|
"integer",
|
||||||
|
"null"
|
||||||
|
],
|
||||||
|
"format": "uint32",
|
||||||
|
"minimum": 0.0
|
||||||
|
},
|
||||||
"purpose": {
|
"purpose": {
|
||||||
"type": [
|
"type": [
|
||||||
"string",
|
"string",
|
||||||
"null"
|
"null"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
"redact_values": {
|
||||||
|
"type": [
|
||||||
|
"boolean",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
"session_id": {
|
"session_id": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -11,6 +11,14 @@
|
|||||||
}
|
}
|
||||||
},
|
},
|
||||||
"definitions": {
|
"definitions": {
|
||||||
|
"FillCommit": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"enter",
|
||||||
|
"suggestion",
|
||||||
|
"blur"
|
||||||
|
]
|
||||||
|
},
|
||||||
"KeyModifier": {
|
"KeyModifier": {
|
||||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||||
"type": "string",
|
"type": "string",
|
||||||
@@ -21,25 +29,32 @@
|
|||||||
"shift"
|
"shift"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"PageRef": {
|
"NavigationCause": {
|
||||||
"description": "Page context dictionary entry — referenced by steps via `page` id.",
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"user_typed",
|
||||||
|
"link",
|
||||||
|
"form_submit",
|
||||||
|
"reload",
|
||||||
|
"history",
|
||||||
|
"script",
|
||||||
|
"browser"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"RecorderInfo": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
"required": [
|
||||||
"id",
|
"bsk",
|
||||||
"url"
|
"vom"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"id": {
|
"bsk": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"title": {
|
"vom": {
|
||||||
"type": [
|
"type": "integer",
|
||||||
"string",
|
"format": "uint32",
|
||||||
"null"
|
"minimum": 0.0
|
||||||
]
|
|
||||||
},
|
|
||||||
"url": {
|
|
||||||
"type": "string"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -68,21 +83,16 @@
|
|||||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
"required": [
|
||||||
|
"cause",
|
||||||
"id",
|
"id",
|
||||||
"op",
|
"op",
|
||||||
"page",
|
"result",
|
||||||
|
"state",
|
||||||
"to"
|
"to"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"effect": {
|
"cause": {
|
||||||
"anyOf": [
|
"$ref": "#/definitions/NavigationCause"
|
||||||
{
|
|
||||||
"$ref": "#/definitions/StepEffect"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "null"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
},
|
||||||
"id": {
|
"id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
@@ -95,8 +105,11 @@
|
|||||||
"navigate"
|
"navigate"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"page": {
|
"result": {
|
||||||
"description": "Reference into `pages[]`.",
|
"$ref": "#/definitions/StepResult"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"to": {
|
"to": {
|
||||||
@@ -110,20 +123,11 @@
|
|||||||
"required": [
|
"required": [
|
||||||
"id",
|
"id",
|
||||||
"op",
|
"op",
|
||||||
"page",
|
"result",
|
||||||
|
"state",
|
||||||
"target"
|
"target"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"effect": {
|
|
||||||
"anyOf": [
|
|
||||||
{
|
|
||||||
"$ref": "#/definitions/StepEffect"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "null"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"id": {
|
"id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
"format": "uint32",
|
"format": "uint32",
|
||||||
@@ -135,8 +139,11 @@
|
|||||||
"click"
|
"click"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"page": {
|
"result": {
|
||||||
"description": "Reference into `pages[]`.",
|
"$ref": "#/definitions/StepResult"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"target": {
|
"target": {
|
||||||
@@ -150,20 +157,11 @@
|
|||||||
"required": [
|
"required": [
|
||||||
"id",
|
"id",
|
||||||
"op",
|
"op",
|
||||||
"page",
|
"result",
|
||||||
|
"state",
|
||||||
"target"
|
"target"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"effect": {
|
|
||||||
"anyOf": [
|
|
||||||
{
|
|
||||||
"$ref": "#/definitions/StepEffect"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "null"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"id": {
|
"id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
"format": "uint32",
|
"format": "uint32",
|
||||||
@@ -175,8 +173,11 @@
|
|||||||
"hover"
|
"hover"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"page": {
|
"result": {
|
||||||
"description": "Reference into `pages[]`.",
|
"$ref": "#/definitions/StepResult"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"target": {
|
"target": {
|
||||||
@@ -188,22 +189,17 @@
|
|||||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
"required": [
|
||||||
|
"commit",
|
||||||
"id",
|
"id",
|
||||||
"op",
|
"op",
|
||||||
"page",
|
"result",
|
||||||
|
"state",
|
||||||
"target",
|
"target",
|
||||||
"value"
|
"value"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"effect": {
|
"commit": {
|
||||||
"anyOf": [
|
"$ref": "#/definitions/FillCommit"
|
||||||
{
|
|
||||||
"$ref": "#/definitions/StepEffect"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "null"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
},
|
||||||
"id": {
|
"id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
@@ -216,15 +212,15 @@
|
|||||||
"fill"
|
"fill"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"page": {
|
|
||||||
"description": "Reference into `pages[]`.",
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"redacted": {
|
"redacted": {
|
||||||
"type": [
|
"type": "boolean"
|
||||||
"boolean",
|
},
|
||||||
"null"
|
"result": {
|
||||||
]
|
"$ref": "#/definitions/StepResult"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
|
"type": "string"
|
||||||
},
|
},
|
||||||
"target": {
|
"target": {
|
||||||
"$ref": "#/definitions/TargetDescriptor"
|
"$ref": "#/definitions/TargetDescriptor"
|
||||||
@@ -240,21 +236,11 @@
|
|||||||
"required": [
|
"required": [
|
||||||
"id",
|
"id",
|
||||||
"op",
|
"op",
|
||||||
"page",
|
"result",
|
||||||
"selection",
|
"state",
|
||||||
"target"
|
"target"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"effect": {
|
|
||||||
"anyOf": [
|
|
||||||
{
|
|
||||||
"$ref": "#/definitions/StepEffect"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "null"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"id": {
|
"id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
"format": "uint32",
|
"format": "uint32",
|
||||||
@@ -266,9 +252,8 @@
|
|||||||
"select"
|
"select"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"page": {
|
"result": {
|
||||||
"description": "Reference into `pages[]`.",
|
"$ref": "#/definitions/StepResult"
|
||||||
"type": "string"
|
|
||||||
},
|
},
|
||||||
"selection": {
|
"selection": {
|
||||||
"type": "array",
|
"type": "array",
|
||||||
@@ -276,6 +261,10 @@
|
|||||||
"$ref": "#/definitions/SelectedOption"
|
"$ref": "#/definitions/SelectedOption"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
"target": {
|
"target": {
|
||||||
"$ref": "#/definitions/TargetDescriptor"
|
"$ref": "#/definitions/TargetDescriptor"
|
||||||
}
|
}
|
||||||
@@ -288,19 +277,10 @@
|
|||||||
"id",
|
"id",
|
||||||
"key",
|
"key",
|
||||||
"op",
|
"op",
|
||||||
"page"
|
"result",
|
||||||
|
"state"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"effect": {
|
|
||||||
"anyOf": [
|
|
||||||
{
|
|
||||||
"$ref": "#/definitions/StepEffect"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "null"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"id": {
|
"id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
"format": "uint32",
|
"format": "uint32",
|
||||||
@@ -324,8 +304,11 @@
|
|||||||
"press"
|
"press"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"page": {
|
"result": {
|
||||||
"description": "Reference into `pages[]`.",
|
"$ref": "#/definitions/StepResult"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"target": {
|
"target": {
|
||||||
@@ -339,48 +322,74 @@
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"id",
|
||||||
|
"op",
|
||||||
|
"result",
|
||||||
|
"state"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"id": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "uint32",
|
||||||
|
"minimum": 0.0
|
||||||
|
},
|
||||||
|
"op": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"scroll"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"result": {
|
||||||
|
"$ref": "#/definitions/StepResult"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"StepEffect": {
|
"StepResult": {
|
||||||
"description": "Observed navigation after a step (objective fact only).",
|
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
"required": [
|
||||||
"navigated_to"
|
"state"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"navigated_to": {
|
"state": {
|
||||||
"description": "Reference into `pages[]` for the destination page.",
|
|
||||||
"type": "string"
|
"type": "string"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"StopReason": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"user_finish",
|
||||||
|
"cli_stop"
|
||||||
|
]
|
||||||
|
},
|
||||||
"TargetDescriptor": {
|
"TargetDescriptor": {
|
||||||
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
|
"description": "Stable semantic handle for an interacted element within a page observation.",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
|
||||||
"tag"
|
|
||||||
],
|
|
||||||
"properties": {
|
"properties": {
|
||||||
|
"ctx": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
"name": {
|
"name": {
|
||||||
"type": [
|
"type": [
|
||||||
"string",
|
"string",
|
||||||
"null"
|
"null"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"name_attr": {
|
"ref": {
|
||||||
"type": [
|
|
||||||
"string",
|
|
||||||
"null"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"nearby_label": {
|
|
||||||
"type": [
|
|
||||||
"string",
|
|
||||||
"null"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"placeholder": {
|
|
||||||
"type": [
|
"type": [
|
||||||
"string",
|
"string",
|
||||||
"null"
|
"null"
|
||||||
@@ -392,8 +401,8 @@
|
|||||||
"null"
|
"null"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"tag": {
|
"unmatched": {
|
||||||
"type": "string"
|
"type": "boolean"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -402,42 +411,54 @@
|
|||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
"required": [
|
||||||
"entry",
|
"entry",
|
||||||
"pages",
|
|
||||||
"recorded_at",
|
"recorded_at",
|
||||||
"steps"
|
"recorder",
|
||||||
|
"states",
|
||||||
|
"steps",
|
||||||
|
"stopped_by",
|
||||||
|
"version"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"entry": {
|
"entry": {
|
||||||
"$ref": "#/definitions/TraceEntry"
|
"$ref": "#/definitions/TraceEntry"
|
||||||
},
|
},
|
||||||
"pages": {
|
|
||||||
"type": "array",
|
|
||||||
"items": {
|
|
||||||
"$ref": "#/definitions/PageRef"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"purpose": {
|
"purpose": {
|
||||||
"description": "Optional user-provided goal from `--purpose` (metadata only).",
|
|
||||||
"type": [
|
"type": [
|
||||||
"string",
|
"string",
|
||||||
"null"
|
"null"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"recorded_at": {
|
"recorded_at": {
|
||||||
"description": "RFC 3339 timestamp when recording stopped.",
|
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
|
"recorder": {
|
||||||
|
"$ref": "#/definitions/RecorderInfo"
|
||||||
|
},
|
||||||
"started_at": {
|
"started_at": {
|
||||||
"type": [
|
"type": [
|
||||||
"string",
|
"string",
|
||||||
"null"
|
"null"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
"states": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/TraceState"
|
||||||
|
}
|
||||||
|
},
|
||||||
"steps": {
|
"steps": {
|
||||||
"type": "array",
|
"type": "array",
|
||||||
"items": {
|
"items": {
|
||||||
"$ref": "#/definitions/Step"
|
"$ref": "#/definitions/Step"
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
"stopped_by": {
|
||||||
|
"$ref": "#/definitions/StopReason"
|
||||||
|
},
|
||||||
|
"version": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "uint32",
|
||||||
|
"minimum": 0.0
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -452,6 +473,45 @@
|
|||||||
"type": "string"
|
"type": "string"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
"TraceState": {
|
||||||
|
"description": "Page observation dictionary entry — referenced by steps via `state` / `result.state`.",
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"id",
|
||||||
|
"url"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"body": {
|
||||||
|
"description": "Wire-only: full page observation (front matter + VOM body + annotations).",
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"id": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"page": {
|
||||||
|
"description": "Disk-only: filename under the bundle `pages/` directory.",
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"title": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"truncated": {
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
|
"url": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,45 +5,146 @@
|
|||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
"required": [
|
||||||
"entry",
|
"entry",
|
||||||
"pages",
|
|
||||||
"recorded_at",
|
"recorded_at",
|
||||||
"steps"
|
"recorder",
|
||||||
|
"states",
|
||||||
|
"steps",
|
||||||
|
"stopped_by",
|
||||||
|
"version"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"entry": {
|
"entry": {
|
||||||
"$ref": "#/definitions/TraceEntry"
|
"$ref": "#/definitions/TraceEntry"
|
||||||
},
|
},
|
||||||
"pages": {
|
"frame_capture": {
|
||||||
"type": "array",
|
"anyOf": [
|
||||||
"items": {
|
{
|
||||||
"$ref": "#/definitions/PageRef"
|
"$ref": "#/definitions/FrameCaptureInfo"
|
||||||
}
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
},
|
},
|
||||||
"purpose": {
|
"purpose": {
|
||||||
"description": "Optional user-provided goal from `--purpose` (metadata only).",
|
|
||||||
"type": [
|
"type": [
|
||||||
"string",
|
"string",
|
||||||
"null"
|
"null"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"recorded_at": {
|
"recorded_at": {
|
||||||
"description": "RFC 3339 timestamp when recording stopped.",
|
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
|
"recorder": {
|
||||||
|
"$ref": "#/definitions/RecorderInfo"
|
||||||
|
},
|
||||||
"started_at": {
|
"started_at": {
|
||||||
"type": [
|
"type": [
|
||||||
"string",
|
"string",
|
||||||
"null"
|
"null"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
"states": {
|
||||||
|
"type": "array",
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/TraceState"
|
||||||
|
}
|
||||||
|
},
|
||||||
"steps": {
|
"steps": {
|
||||||
"type": "array",
|
"type": "array",
|
||||||
"items": {
|
"items": {
|
||||||
"$ref": "#/definitions/Step"
|
"$ref": "#/definitions/Step"
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
"stopped_by": {
|
||||||
|
"$ref": "#/definitions/StopReason"
|
||||||
|
},
|
||||||
|
"version": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "uint32",
|
||||||
|
"minimum": 0.0
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"definitions": {
|
"definitions": {
|
||||||
|
"FillCommit": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"enter",
|
||||||
|
"suggestion",
|
||||||
|
"blur"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"FrameCaptureFailure": {
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"frame_id",
|
||||||
|
"reason"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"detail": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"document_id": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"frame_id": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "int64"
|
||||||
|
},
|
||||||
|
"reason": {
|
||||||
|
"$ref": "#/definitions/FrameCaptureFailureReason"
|
||||||
|
},
|
||||||
|
"url": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"FrameCaptureFailureReason": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"not_injectable",
|
||||||
|
"arm_failed",
|
||||||
|
"rearm_failed",
|
||||||
|
"drain_failed",
|
||||||
|
"flush_failed"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"FrameCaptureInfo": {
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"status"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"failures": {
|
||||||
|
"type": [
|
||||||
|
"array",
|
||||||
|
"null"
|
||||||
|
],
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/definitions/FrameCaptureFailure"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"status": {
|
||||||
|
"$ref": "#/definitions/FrameCaptureStatus"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"FrameCaptureStatus": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"complete",
|
||||||
|
"partial"
|
||||||
|
]
|
||||||
|
},
|
||||||
"KeyModifier": {
|
"KeyModifier": {
|
||||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||||
"type": "string",
|
"type": "string",
|
||||||
@@ -54,25 +155,32 @@
|
|||||||
"shift"
|
"shift"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"PageRef": {
|
"NavigationCause": {
|
||||||
"description": "Page context dictionary entry — referenced by steps via `page` id.",
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"user_typed",
|
||||||
|
"link",
|
||||||
|
"form_submit",
|
||||||
|
"reload",
|
||||||
|
"history",
|
||||||
|
"script",
|
||||||
|
"browser"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"RecorderInfo": {
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
"required": [
|
||||||
"id",
|
"bsk",
|
||||||
"url"
|
"vom"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"id": {
|
"bsk": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"title": {
|
"vom": {
|
||||||
"type": [
|
"type": "integer",
|
||||||
"string",
|
"format": "uint32",
|
||||||
"null"
|
"minimum": 0.0
|
||||||
]
|
|
||||||
},
|
|
||||||
"url": {
|
|
||||||
"type": "string"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -101,21 +209,16 @@
|
|||||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
"required": [
|
||||||
|
"cause",
|
||||||
"id",
|
"id",
|
||||||
"op",
|
"op",
|
||||||
"page",
|
"result",
|
||||||
|
"state",
|
||||||
"to"
|
"to"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"effect": {
|
"cause": {
|
||||||
"anyOf": [
|
"$ref": "#/definitions/NavigationCause"
|
||||||
{
|
|
||||||
"$ref": "#/definitions/StepEffect"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "null"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
},
|
||||||
"id": {
|
"id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
@@ -128,8 +231,11 @@
|
|||||||
"navigate"
|
"navigate"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"page": {
|
"result": {
|
||||||
"description": "Reference into `pages[]`.",
|
"$ref": "#/definitions/StepResult"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"to": {
|
"to": {
|
||||||
@@ -143,20 +249,11 @@
|
|||||||
"required": [
|
"required": [
|
||||||
"id",
|
"id",
|
||||||
"op",
|
"op",
|
||||||
"page",
|
"result",
|
||||||
|
"state",
|
||||||
"target"
|
"target"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"effect": {
|
|
||||||
"anyOf": [
|
|
||||||
{
|
|
||||||
"$ref": "#/definitions/StepEffect"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "null"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"id": {
|
"id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
"format": "uint32",
|
"format": "uint32",
|
||||||
@@ -168,8 +265,11 @@
|
|||||||
"click"
|
"click"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"page": {
|
"result": {
|
||||||
"description": "Reference into `pages[]`.",
|
"$ref": "#/definitions/StepResult"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"target": {
|
"target": {
|
||||||
@@ -183,20 +283,11 @@
|
|||||||
"required": [
|
"required": [
|
||||||
"id",
|
"id",
|
||||||
"op",
|
"op",
|
||||||
"page",
|
"result",
|
||||||
|
"state",
|
||||||
"target"
|
"target"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"effect": {
|
|
||||||
"anyOf": [
|
|
||||||
{
|
|
||||||
"$ref": "#/definitions/StepEffect"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "null"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"id": {
|
"id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
"format": "uint32",
|
"format": "uint32",
|
||||||
@@ -208,8 +299,11 @@
|
|||||||
"hover"
|
"hover"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"page": {
|
"result": {
|
||||||
"description": "Reference into `pages[]`.",
|
"$ref": "#/definitions/StepResult"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"target": {
|
"target": {
|
||||||
@@ -221,22 +315,17 @@
|
|||||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
"required": [
|
||||||
|
"commit",
|
||||||
"id",
|
"id",
|
||||||
"op",
|
"op",
|
||||||
"page",
|
"result",
|
||||||
|
"state",
|
||||||
"target",
|
"target",
|
||||||
"value"
|
"value"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"effect": {
|
"commit": {
|
||||||
"anyOf": [
|
"$ref": "#/definitions/FillCommit"
|
||||||
{
|
|
||||||
"$ref": "#/definitions/StepEffect"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "null"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
},
|
||||||
"id": {
|
"id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
@@ -249,15 +338,15 @@
|
|||||||
"fill"
|
"fill"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"page": {
|
|
||||||
"description": "Reference into `pages[]`.",
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"redacted": {
|
"redacted": {
|
||||||
"type": [
|
"type": "boolean"
|
||||||
"boolean",
|
},
|
||||||
"null"
|
"result": {
|
||||||
]
|
"$ref": "#/definitions/StepResult"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
|
"type": "string"
|
||||||
},
|
},
|
||||||
"target": {
|
"target": {
|
||||||
"$ref": "#/definitions/TargetDescriptor"
|
"$ref": "#/definitions/TargetDescriptor"
|
||||||
@@ -273,21 +362,11 @@
|
|||||||
"required": [
|
"required": [
|
||||||
"id",
|
"id",
|
||||||
"op",
|
"op",
|
||||||
"page",
|
"result",
|
||||||
"selection",
|
"state",
|
||||||
"target"
|
"target"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"effect": {
|
|
||||||
"anyOf": [
|
|
||||||
{
|
|
||||||
"$ref": "#/definitions/StepEffect"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "null"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"id": {
|
"id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
"format": "uint32",
|
"format": "uint32",
|
||||||
@@ -299,9 +378,8 @@
|
|||||||
"select"
|
"select"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"page": {
|
"result": {
|
||||||
"description": "Reference into `pages[]`.",
|
"$ref": "#/definitions/StepResult"
|
||||||
"type": "string"
|
|
||||||
},
|
},
|
||||||
"selection": {
|
"selection": {
|
||||||
"type": "array",
|
"type": "array",
|
||||||
@@ -309,6 +387,10 @@
|
|||||||
"$ref": "#/definitions/SelectedOption"
|
"$ref": "#/definitions/SelectedOption"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
"target": {
|
"target": {
|
||||||
"$ref": "#/definitions/TargetDescriptor"
|
"$ref": "#/definitions/TargetDescriptor"
|
||||||
}
|
}
|
||||||
@@ -321,19 +403,10 @@
|
|||||||
"id",
|
"id",
|
||||||
"key",
|
"key",
|
||||||
"op",
|
"op",
|
||||||
"page"
|
"result",
|
||||||
|
"state"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"effect": {
|
|
||||||
"anyOf": [
|
|
||||||
{
|
|
||||||
"$ref": "#/definitions/StepEffect"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "null"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"id": {
|
"id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
"format": "uint32",
|
"format": "uint32",
|
||||||
@@ -357,8 +430,11 @@
|
|||||||
"press"
|
"press"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"page": {
|
"result": {
|
||||||
"description": "Reference into `pages[]`.",
|
"$ref": "#/definitions/StepResult"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"target": {
|
"target": {
|
||||||
@@ -372,48 +448,74 @@
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"id",
|
||||||
|
"op",
|
||||||
|
"result",
|
||||||
|
"state"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"id": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "uint32",
|
||||||
|
"minimum": 0.0
|
||||||
|
},
|
||||||
|
"op": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"scroll"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"result": {
|
||||||
|
"$ref": "#/definitions/StepResult"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"StepEffect": {
|
"StepResult": {
|
||||||
"description": "Observed navigation after a step (objective fact only).",
|
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
"required": [
|
||||||
"navigated_to"
|
"state"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"navigated_to": {
|
"state": {
|
||||||
"description": "Reference into `pages[]` for the destination page.",
|
|
||||||
"type": "string"
|
"type": "string"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"StopReason": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"user_finish",
|
||||||
|
"cli_stop"
|
||||||
|
]
|
||||||
|
},
|
||||||
"TargetDescriptor": {
|
"TargetDescriptor": {
|
||||||
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
|
"description": "Stable semantic handle for an interacted element within a page observation.",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
|
||||||
"tag"
|
|
||||||
],
|
|
||||||
"properties": {
|
"properties": {
|
||||||
|
"ctx": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
"name": {
|
"name": {
|
||||||
"type": [
|
"type": [
|
||||||
"string",
|
"string",
|
||||||
"null"
|
"null"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"name_attr": {
|
"ref": {
|
||||||
"type": [
|
|
||||||
"string",
|
|
||||||
"null"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"nearby_label": {
|
|
||||||
"type": [
|
|
||||||
"string",
|
|
||||||
"null"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"placeholder": {
|
|
||||||
"type": [
|
"type": [
|
||||||
"string",
|
"string",
|
||||||
"null"
|
"null"
|
||||||
@@ -425,8 +527,8 @@
|
|||||||
"null"
|
"null"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"tag": {
|
"unmatched": {
|
||||||
"type": "string"
|
"type": "boolean"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
@@ -441,6 +543,45 @@
|
|||||||
"type": "string"
|
"type": "string"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
"TraceState": {
|
||||||
|
"description": "Page observation dictionary entry — referenced by steps via `state` / `result.state`.",
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"id",
|
||||||
|
"url"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"body": {
|
||||||
|
"description": "Wire-only: full page observation (front matter + VOM body + annotations).",
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"id": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"page": {
|
||||||
|
"description": "Disk-only: filename under the bundle `pages/` directory.",
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"title": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"truncated": {
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
|
"url": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,21 +7,16 @@
|
|||||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
"required": [
|
||||||
|
"cause",
|
||||||
"id",
|
"id",
|
||||||
"op",
|
"op",
|
||||||
"page",
|
"result",
|
||||||
|
"state",
|
||||||
"to"
|
"to"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"effect": {
|
"cause": {
|
||||||
"anyOf": [
|
"$ref": "#/definitions/NavigationCause"
|
||||||
{
|
|
||||||
"$ref": "#/definitions/StepEffect"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "null"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
},
|
||||||
"id": {
|
"id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
@@ -34,8 +29,11 @@
|
|||||||
"navigate"
|
"navigate"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"page": {
|
"result": {
|
||||||
"description": "Reference into `pages[]`.",
|
"$ref": "#/definitions/StepResult"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"to": {
|
"to": {
|
||||||
@@ -49,20 +47,11 @@
|
|||||||
"required": [
|
"required": [
|
||||||
"id",
|
"id",
|
||||||
"op",
|
"op",
|
||||||
"page",
|
"result",
|
||||||
|
"state",
|
||||||
"target"
|
"target"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"effect": {
|
|
||||||
"anyOf": [
|
|
||||||
{
|
|
||||||
"$ref": "#/definitions/StepEffect"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "null"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"id": {
|
"id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
"format": "uint32",
|
"format": "uint32",
|
||||||
@@ -74,8 +63,11 @@
|
|||||||
"click"
|
"click"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"page": {
|
"result": {
|
||||||
"description": "Reference into `pages[]`.",
|
"$ref": "#/definitions/StepResult"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"target": {
|
"target": {
|
||||||
@@ -89,20 +81,11 @@
|
|||||||
"required": [
|
"required": [
|
||||||
"id",
|
"id",
|
||||||
"op",
|
"op",
|
||||||
"page",
|
"result",
|
||||||
|
"state",
|
||||||
"target"
|
"target"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"effect": {
|
|
||||||
"anyOf": [
|
|
||||||
{
|
|
||||||
"$ref": "#/definitions/StepEffect"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "null"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"id": {
|
"id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
"format": "uint32",
|
"format": "uint32",
|
||||||
@@ -114,8 +97,11 @@
|
|||||||
"hover"
|
"hover"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"page": {
|
"result": {
|
||||||
"description": "Reference into `pages[]`.",
|
"$ref": "#/definitions/StepResult"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"target": {
|
"target": {
|
||||||
@@ -127,22 +113,17 @@
|
|||||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
"required": [
|
||||||
|
"commit",
|
||||||
"id",
|
"id",
|
||||||
"op",
|
"op",
|
||||||
"page",
|
"result",
|
||||||
|
"state",
|
||||||
"target",
|
"target",
|
||||||
"value"
|
"value"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"effect": {
|
"commit": {
|
||||||
"anyOf": [
|
"$ref": "#/definitions/FillCommit"
|
||||||
{
|
|
||||||
"$ref": "#/definitions/StepEffect"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "null"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
},
|
||||||
"id": {
|
"id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
@@ -155,15 +136,15 @@
|
|||||||
"fill"
|
"fill"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"page": {
|
|
||||||
"description": "Reference into `pages[]`.",
|
|
||||||
"type": "string"
|
|
||||||
},
|
|
||||||
"redacted": {
|
"redacted": {
|
||||||
"type": [
|
"type": "boolean"
|
||||||
"boolean",
|
},
|
||||||
"null"
|
"result": {
|
||||||
]
|
"$ref": "#/definitions/StepResult"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
|
"type": "string"
|
||||||
},
|
},
|
||||||
"target": {
|
"target": {
|
||||||
"$ref": "#/definitions/TargetDescriptor"
|
"$ref": "#/definitions/TargetDescriptor"
|
||||||
@@ -179,21 +160,11 @@
|
|||||||
"required": [
|
"required": [
|
||||||
"id",
|
"id",
|
||||||
"op",
|
"op",
|
||||||
"page",
|
"result",
|
||||||
"selection",
|
"state",
|
||||||
"target"
|
"target"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"effect": {
|
|
||||||
"anyOf": [
|
|
||||||
{
|
|
||||||
"$ref": "#/definitions/StepEffect"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "null"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"id": {
|
"id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
"format": "uint32",
|
"format": "uint32",
|
||||||
@@ -205,9 +176,8 @@
|
|||||||
"select"
|
"select"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"page": {
|
"result": {
|
||||||
"description": "Reference into `pages[]`.",
|
"$ref": "#/definitions/StepResult"
|
||||||
"type": "string"
|
|
||||||
},
|
},
|
||||||
"selection": {
|
"selection": {
|
||||||
"type": "array",
|
"type": "array",
|
||||||
@@ -215,6 +185,10 @@
|
|||||||
"$ref": "#/definitions/SelectedOption"
|
"$ref": "#/definitions/SelectedOption"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
"target": {
|
"target": {
|
||||||
"$ref": "#/definitions/TargetDescriptor"
|
"$ref": "#/definitions/TargetDescriptor"
|
||||||
}
|
}
|
||||||
@@ -227,19 +201,10 @@
|
|||||||
"id",
|
"id",
|
||||||
"key",
|
"key",
|
||||||
"op",
|
"op",
|
||||||
"page"
|
"result",
|
||||||
|
"state"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"effect": {
|
|
||||||
"anyOf": [
|
|
||||||
{
|
|
||||||
"$ref": "#/definitions/StepEffect"
|
|
||||||
},
|
|
||||||
{
|
|
||||||
"type": "null"
|
|
||||||
}
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"id": {
|
"id": {
|
||||||
"type": "integer",
|
"type": "integer",
|
||||||
"format": "uint32",
|
"format": "uint32",
|
||||||
@@ -263,8 +228,11 @@
|
|||||||
"press"
|
"press"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"page": {
|
"result": {
|
||||||
"description": "Reference into `pages[]`.",
|
"$ref": "#/definitions/StepResult"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
"target": {
|
"target": {
|
||||||
@@ -278,9 +246,47 @@
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||||
|
"type": "object",
|
||||||
|
"required": [
|
||||||
|
"id",
|
||||||
|
"op",
|
||||||
|
"result",
|
||||||
|
"state"
|
||||||
|
],
|
||||||
|
"properties": {
|
||||||
|
"id": {
|
||||||
|
"type": "integer",
|
||||||
|
"format": "uint32",
|
||||||
|
"minimum": 0.0
|
||||||
|
},
|
||||||
|
"op": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"scroll"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"result": {
|
||||||
|
"$ref": "#/definitions/StepResult"
|
||||||
|
},
|
||||||
|
"state": {
|
||||||
|
"description": "Observation id immediately before this action.",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"definitions": {
|
"definitions": {
|
||||||
|
"FillCommit": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"enter",
|
||||||
|
"suggestion",
|
||||||
|
"blur"
|
||||||
|
]
|
||||||
|
},
|
||||||
"KeyModifier": {
|
"KeyModifier": {
|
||||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||||
"type": "string",
|
"type": "string",
|
||||||
@@ -291,6 +297,18 @@
|
|||||||
"shift"
|
"shift"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
"NavigationCause": {
|
||||||
|
"type": "string",
|
||||||
|
"enum": [
|
||||||
|
"user_typed",
|
||||||
|
"link",
|
||||||
|
"form_submit",
|
||||||
|
"reload",
|
||||||
|
"history",
|
||||||
|
"script",
|
||||||
|
"browser"
|
||||||
|
]
|
||||||
|
},
|
||||||
"SelectedOption": {
|
"SelectedOption": {
|
||||||
"description": "One selected option (`select` op).",
|
"description": "One selected option (`select` op).",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
@@ -309,45 +327,34 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"StepEffect": {
|
"StepResult": {
|
||||||
"description": "Observed navigation after a step (objective fact only).",
|
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
"required": [
|
||||||
"navigated_to"
|
"state"
|
||||||
],
|
],
|
||||||
"properties": {
|
"properties": {
|
||||||
"navigated_to": {
|
"state": {
|
||||||
"description": "Reference into `pages[]` for the destination page.",
|
|
||||||
"type": "string"
|
"type": "string"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"TargetDescriptor": {
|
"TargetDescriptor": {
|
||||||
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
|
"description": "Stable semantic handle for an interacted element within a page observation.",
|
||||||
"type": "object",
|
"type": "object",
|
||||||
"required": [
|
|
||||||
"tag"
|
|
||||||
],
|
|
||||||
"properties": {
|
"properties": {
|
||||||
|
"ctx": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
"name": {
|
"name": {
|
||||||
"type": [
|
"type": [
|
||||||
"string",
|
"string",
|
||||||
"null"
|
"null"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"name_attr": {
|
"ref": {
|
||||||
"type": [
|
|
||||||
"string",
|
|
||||||
"null"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"nearby_label": {
|
|
||||||
"type": [
|
|
||||||
"string",
|
|
||||||
"null"
|
|
||||||
]
|
|
||||||
},
|
|
||||||
"placeholder": {
|
|
||||||
"type": [
|
"type": [
|
||||||
"string",
|
"string",
|
||||||
"null"
|
"null"
|
||||||
@@ -359,8 +366,8 @@
|
|||||||
"null"
|
"null"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"tag": {
|
"unmatched": {
|
||||||
"type": "string"
|
"type": "boolean"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,35 +1,33 @@
|
|||||||
//! Semantic user-action recording (`tool.record_start` / `stop` / `await`).
|
//! Semantic user-action recording (`tool.record_start` / `stop` / `await`).
|
||||||
//!
|
//!
|
||||||
//! Trace v2 is a **record-only** log of user actions: what was clicked, filled,
|
//! Trace v3 is a **state-action-state** chain: each step binds to page
|
||||||
//! selected, and where navigation occurred. No LLM runs during recording, so
|
//! observations (VOM) captured before and after the action.
|
||||||
//! variable-vs-constant classification is **not** stored — executing agents
|
|
||||||
//! infer that at run time from raw values and control names.
|
|
||||||
|
|
||||||
use schemars::JsonSchema;
|
use schemars::JsonSchema;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
use super::interaction::KeyModifier;
|
use super::interaction::KeyModifier;
|
||||||
|
|
||||||
|
pub const TRACE_VERSION: u32 = 3;
|
||||||
|
pub const VOM_FORMAT_VERSION: u32 = 1;
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
// Target
|
// Target
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
/// Stable semantic handle for an interacted element.
|
/// Stable semantic handle for an interacted element within a page observation.
|
||||||
///
|
|
||||||
/// `name` and `nearby_label` are **untrusted page text**.
|
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
pub struct TargetDescriptor {
|
pub struct TargetDescriptor {
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none", rename = "ref")]
|
||||||
|
pub element_ref: Option<String>,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub role: Option<String>,
|
pub role: Option<String>,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub name: Option<String>,
|
pub name: Option<String>,
|
||||||
pub tag: String,
|
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub name_attr: Option<String>,
|
pub ctx: Option<String>,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||||
pub placeholder: Option<String>,
|
pub unmatched: bool,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
||||||
pub nearby_label: Option<String>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -42,15 +40,54 @@ pub struct TraceEntry {
|
|||||||
pub start_url: String,
|
pub start_url: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Page context dictionary entry — referenced by steps via `page` id.
|
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
pub struct PageRef {
|
pub struct RecorderInfo {
|
||||||
|
pub bsk: String,
|
||||||
|
pub vom: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum StopReason {
|
||||||
|
UserFinish,
|
||||||
|
CliStop,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Page observation dictionary entry — referenced by steps via `state` / `result.state`.
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct TraceState {
|
||||||
pub id: String,
|
pub id: String,
|
||||||
pub url: String,
|
pub url: String,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub title: Option<String>,
|
pub title: Option<String>,
|
||||||
|
/// Wire-only: full page observation (front matter + VOM body + annotations).
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub body: Option<String>,
|
||||||
|
/// Disk-only: filename under the bundle `pages/` directory.
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub page: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||||
|
pub truncated: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct StepResult {
|
||||||
|
pub state: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Fields shared by every step variant (flattened in JSON).
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct StepCommon {
|
||||||
|
pub id: u32,
|
||||||
|
/// Observation id immediately before this action.
|
||||||
|
pub state: String,
|
||||||
|
pub result: StepResult,
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Step op-specific payloads
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
/// One selected option (`select` op).
|
/// One selected option (`select` op).
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
pub struct SelectedOption {
|
pub struct SelectedOption {
|
||||||
@@ -59,27 +96,26 @@ pub struct SelectedOption {
|
|||||||
pub label: Option<String>,
|
pub label: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Observed navigation after a step (objective fact only).
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
#[serde(rename_all = "snake_case")]
|
||||||
pub struct StepEffect {
|
pub enum NavigationCause {
|
||||||
/// Reference into `pages[]` for the destination page.
|
UserTyped,
|
||||||
pub navigated_to: String,
|
Link,
|
||||||
|
FormSubmit,
|
||||||
|
Reload,
|
||||||
|
History,
|
||||||
|
Script,
|
||||||
|
Browser,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Fields shared by every step variant (flattened in JSON).
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
#[serde(rename_all = "snake_case")]
|
||||||
pub struct StepCommon {
|
pub enum FillCommit {
|
||||||
pub id: u32,
|
Enter,
|
||||||
/// Reference into `pages[]`.
|
Suggestion,
|
||||||
pub page: String,
|
Blur,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
||||||
pub effect: Option<StepEffect>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
// Step op-specific payloads
|
|
||||||
// ---------------------------------------------------------------------------
|
|
||||||
|
|
||||||
/// One recorded user action — discriminated union by `op`.
|
/// One recorded user action — discriminated union by `op`.
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
#[serde(tag = "op", rename_all = "snake_case")]
|
#[serde(tag = "op", rename_all = "snake_case")]
|
||||||
@@ -88,6 +124,7 @@ pub enum Step {
|
|||||||
#[serde(flatten)]
|
#[serde(flatten)]
|
||||||
common: StepCommon,
|
common: StepCommon,
|
||||||
to: String,
|
to: String,
|
||||||
|
cause: NavigationCause,
|
||||||
},
|
},
|
||||||
Click {
|
Click {
|
||||||
#[serde(flatten)]
|
#[serde(flatten)]
|
||||||
@@ -104,13 +141,15 @@ pub enum Step {
|
|||||||
common: StepCommon,
|
common: StepCommon,
|
||||||
target: TargetDescriptor,
|
target: TargetDescriptor,
|
||||||
value: String,
|
value: String,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
commit: FillCommit,
|
||||||
redacted: Option<bool>,
|
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||||
|
redacted: bool,
|
||||||
},
|
},
|
||||||
Select {
|
Select {
|
||||||
#[serde(flatten)]
|
#[serde(flatten)]
|
||||||
common: StepCommon,
|
common: StepCommon,
|
||||||
target: TargetDescriptor,
|
target: TargetDescriptor,
|
||||||
|
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||||
selection: Vec<SelectedOption>,
|
selection: Vec<SelectedOption>,
|
||||||
},
|
},
|
||||||
Press {
|
Press {
|
||||||
@@ -122,6 +161,50 @@ pub enum Step {
|
|||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
target: Option<TargetDescriptor>,
|
target: Option<TargetDescriptor>,
|
||||||
},
|
},
|
||||||
|
Scroll {
|
||||||
|
#[serde(flatten)]
|
||||||
|
common: StepCommon,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
// Frame capture metadata
|
||||||
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum FrameCaptureStatus {
|
||||||
|
Complete,
|
||||||
|
Partial,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum FrameCaptureFailureReason {
|
||||||
|
NotInjectable,
|
||||||
|
ArmFailed,
|
||||||
|
RearmFailed,
|
||||||
|
DrainFailed,
|
||||||
|
FlushFailed,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct FrameCaptureFailure {
|
||||||
|
pub reason: FrameCaptureFailureReason,
|
||||||
|
pub frame_id: i64,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub document_id: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub url: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub detail: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
|
pub struct FrameCaptureInfo {
|
||||||
|
pub status: FrameCaptureStatus,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub failures: Option<Vec<FrameCaptureFailure>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
@@ -131,15 +214,18 @@ pub enum Step {
|
|||||||
/// Persisted user-action trace exported by `tool.record_stop` / `await`.
|
/// Persisted user-action trace exported by `tool.record_stop` / `await`.
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
pub struct Trace {
|
pub struct Trace {
|
||||||
/// RFC 3339 timestamp when recording stopped.
|
pub version: u32,
|
||||||
pub recorded_at: String,
|
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
|
||||||
pub started_at: Option<String>,
|
|
||||||
/// Optional user-provided goal from `--purpose` (metadata only).
|
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub purpose: Option<String>,
|
pub purpose: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub started_at: Option<String>,
|
||||||
|
pub recorded_at: String,
|
||||||
|
pub stopped_by: StopReason,
|
||||||
pub entry: TraceEntry,
|
pub entry: TraceEntry,
|
||||||
pub pages: Vec<PageRef>,
|
pub recorder: RecorderInfo,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub frame_capture: Option<FrameCaptureInfo>,
|
||||||
|
pub states: Vec<TraceState>,
|
||||||
pub steps: Vec<Step>,
|
pub steps: Vec<Step>,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -156,6 +242,10 @@ pub struct RecordStartParams {
|
|||||||
pub url: Option<String>,
|
pub url: Option<String>,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
pub purpose: Option<String>,
|
pub purpose: Option<String>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub max_page_tokens: Option<u32>,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub redact_values: Option<bool>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||||
@@ -195,81 +285,80 @@ mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
use serde_json::json;
|
use serde_json::json;
|
||||||
|
|
||||||
fn sample_common(id: u32) -> StepCommon {
|
fn sample_common(id: u32, state: &str, result_state: &str) -> StepCommon {
|
||||||
StepCommon {
|
StepCommon {
|
||||||
id,
|
id,
|
||||||
page: "p1".into(),
|
state: state.into(),
|
||||||
effect: None,
|
result: StepResult {
|
||||||
|
state: result_state.into(),
|
||||||
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sample_target() -> TargetDescriptor {
|
fn sample_target() -> TargetDescriptor {
|
||||||
TargetDescriptor {
|
TargetDescriptor {
|
||||||
|
element_ref: Some("e21".into()),
|
||||||
role: Some("button".into()),
|
role: Some("button".into()),
|
||||||
name: Some("发布".into()),
|
name: Some("发布".into()),
|
||||||
tag: "button".into(),
|
ctx: Some("金桔柠檬 6 号".into()),
|
||||||
name_attr: None,
|
unmatched: false,
|
||||||
placeholder: None,
|
|
||||||
nearby_label: None,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn sample_trace() -> Trace {
|
fn sample_trace() -> Trace {
|
||||||
Trace {
|
Trace {
|
||||||
recorded_at: "2026-07-17T09:01:10Z".into(),
|
version: TRACE_VERSION,
|
||||||
started_at: Some("2026-07-17T09:00:00Z".into()),
|
purpose: Some("把草稿商品发布上架".into()),
|
||||||
purpose: Some("发布一篇文章".into()),
|
started_at: Some("2026-08-10T02:10:41.080Z".into()),
|
||||||
|
recorded_at: "2026-08-10T02:12:55.360Z".into(),
|
||||||
|
stopped_by: StopReason::UserFinish,
|
||||||
entry: TraceEntry {
|
entry: TraceEntry {
|
||||||
start_url: "https://x.com/editor".into(),
|
start_url: "https://example.com/".into(),
|
||||||
},
|
},
|
||||||
pages: vec![
|
recorder: RecorderInfo {
|
||||||
PageRef {
|
bsk: "0.1.10".into(),
|
||||||
id: "p1".into(),
|
vom: VOM_FORMAT_VERSION,
|
||||||
url: "https://x.com/editor".into(),
|
},
|
||||||
title: Some("写文章".into()),
|
frame_capture: None,
|
||||||
|
states: vec![
|
||||||
|
TraceState {
|
||||||
|
id: "s1".into(),
|
||||||
|
url: "https://example.com/".into(),
|
||||||
|
title: Some("Example Domain".into()),
|
||||||
|
body: None,
|
||||||
|
page: Some("s1.vom.txt".into()),
|
||||||
|
truncated: false,
|
||||||
},
|
},
|
||||||
PageRef {
|
TraceState {
|
||||||
id: "p2".into(),
|
id: "s2".into(),
|
||||||
url: "https://x.com/p/99".into(),
|
url: "https://shop.example.com/products?status=draft".into(),
|
||||||
title: None,
|
title: Some("商品管理".into()),
|
||||||
|
body: None,
|
||||||
|
page: Some("s2.vom.txt".into()),
|
||||||
|
truncated: false,
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
steps: vec![
|
steps: vec![
|
||||||
Step::Fill {
|
Step::Navigate {
|
||||||
common: sample_common(1),
|
common: sample_common(1, "s1", "s2"),
|
||||||
target: TargetDescriptor {
|
to: "https://shop.example.com/products?status=draft".into(),
|
||||||
role: Some("textbox".into()),
|
cause: NavigationCause::UserTyped,
|
||||||
name: Some("标题".into()),
|
|
||||||
tag: "input".into(),
|
|
||||||
name_attr: None,
|
|
||||||
placeholder: None,
|
|
||||||
nearby_label: None,
|
|
||||||
},
|
|
||||||
value: "我的第一篇文章".into(),
|
|
||||||
redacted: None,
|
|
||||||
},
|
},
|
||||||
Step::Select {
|
Step::Fill {
|
||||||
common: sample_common(3),
|
common: sample_common(2, "s2", "s2"),
|
||||||
target: TargetDescriptor {
|
target: TargetDescriptor {
|
||||||
role: Some("combobox".into()),
|
element_ref: Some("e12".into()),
|
||||||
name: Some("分类".into()),
|
role: Some("textbox".into()),
|
||||||
tag: "select".into(),
|
name: Some("搜索商品".into()),
|
||||||
name_attr: None,
|
ctx: None,
|
||||||
placeholder: None,
|
unmatched: false,
|
||||||
nearby_label: None,
|
|
||||||
},
|
},
|
||||||
selection: vec![SelectedOption {
|
value: "金桔柠檬".into(),
|
||||||
value: "tech".into(),
|
commit: FillCommit::Enter,
|
||||||
label: Some("技术分享".into()),
|
redacted: false,
|
||||||
}],
|
|
||||||
},
|
},
|
||||||
Step::Click {
|
Step::Click {
|
||||||
common: StepCommon {
|
common: sample_common(3, "s2", "s2"),
|
||||||
effect: Some(StepEffect {
|
|
||||||
navigated_to: "p2".into(),
|
|
||||||
}),
|
|
||||||
..sample_common(4)
|
|
||||||
},
|
|
||||||
target: sample_target(),
|
target: sample_target(),
|
||||||
},
|
},
|
||||||
],
|
],
|
||||||
@@ -279,35 +368,37 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn step_click_round_trips() {
|
fn step_click_round_trips() {
|
||||||
let step = Step::Click {
|
let step = Step::Click {
|
||||||
common: sample_common(1),
|
common: sample_common(1, "s1", "s2"),
|
||||||
target: sample_target(),
|
target: sample_target(),
|
||||||
};
|
};
|
||||||
let v = serde_json::to_value(&step).unwrap();
|
let v = serde_json::to_value(&step).unwrap();
|
||||||
assert_eq!(v.get("op").and_then(|v| v.as_str()), Some("click"));
|
assert_eq!(v.get("op").and_then(|v| v.as_str()), Some("click"));
|
||||||
assert_eq!(v.get("page").and_then(|v| v.as_str()), Some("p1"));
|
assert_eq!(v.get("state").and_then(|v| v.as_str()), Some("s1"));
|
||||||
assert!(v.get("key").is_none());
|
assert_eq!(v["result"]["state"], "s2");
|
||||||
|
assert_eq!(v["target"]["ref"], "e21");
|
||||||
let round: Step = serde_json::from_value(v).unwrap();
|
let round: Step = serde_json::from_value(v).unwrap();
|
||||||
assert_eq!(round, step);
|
assert_eq!(round, step);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn step_fill_with_raw_value_round_trips() {
|
fn step_fill_with_commit_round_trips() {
|
||||||
let step = Step::Fill {
|
let step = Step::Fill {
|
||||||
common: sample_common(2),
|
common: sample_common(2, "s2", "s3"),
|
||||||
target: TargetDescriptor {
|
target: TargetDescriptor {
|
||||||
|
element_ref: Some("e12".into()),
|
||||||
role: Some("textbox".into()),
|
role: Some("textbox".into()),
|
||||||
name: Some("服务名称".into()),
|
name: Some("搜索商品".into()),
|
||||||
tag: "input".into(),
|
ctx: None,
|
||||||
name_attr: Some("serviceName".into()),
|
unmatched: false,
|
||||||
placeholder: None,
|
|
||||||
nearby_label: None,
|
|
||||||
},
|
},
|
||||||
value: "my-svc".into(),
|
value: "browser skill".into(),
|
||||||
redacted: None,
|
commit: FillCommit::Enter,
|
||||||
|
redacted: false,
|
||||||
};
|
};
|
||||||
let v = serde_json::to_value(&step).unwrap();
|
let v = serde_json::to_value(&step).unwrap();
|
||||||
assert_eq!(v["op"], "fill");
|
assert_eq!(v["op"], "fill");
|
||||||
assert_eq!(v["value"], "my-svc");
|
assert_eq!(v["commit"], "enter");
|
||||||
|
assert!(v.get("redacted").is_none());
|
||||||
let round: Step = serde_json::from_value(v).unwrap();
|
let round: Step = serde_json::from_value(v).unwrap();
|
||||||
assert_eq!(round, step);
|
assert_eq!(round, step);
|
||||||
}
|
}
|
||||||
@@ -315,17 +406,17 @@ mod tests {
|
|||||||
#[test]
|
#[test]
|
||||||
fn step_fill_password_is_redacted() {
|
fn step_fill_password_is_redacted() {
|
||||||
let step = Step::Fill {
|
let step = Step::Fill {
|
||||||
common: sample_common(1),
|
common: sample_common(1, "s1", "s1"),
|
||||||
target: TargetDescriptor {
|
target: TargetDescriptor {
|
||||||
|
element_ref: Some("e3".into()),
|
||||||
role: Some("textbox".into()),
|
role: Some("textbox".into()),
|
||||||
name: Some("密码".into()),
|
name: Some("密码".into()),
|
||||||
tag: "input".into(),
|
ctx: None,
|
||||||
name_attr: None,
|
unmatched: false,
|
||||||
placeholder: None,
|
|
||||||
nearby_label: None,
|
|
||||||
},
|
},
|
||||||
value: "***".into(),
|
value: "***".into(),
|
||||||
redacted: Some(true),
|
commit: FillCommit::Blur,
|
||||||
|
redacted: true,
|
||||||
};
|
};
|
||||||
let v = serde_json::to_value(&step).unwrap();
|
let v = serde_json::to_value(&step).unwrap();
|
||||||
assert_eq!(v["value"], "***");
|
assert_eq!(v["value"], "***");
|
||||||
@@ -333,131 +424,137 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn step_select_uses_object_array() {
|
fn step_navigate_with_cause_round_trips() {
|
||||||
let step = Step::Select {
|
|
||||||
common: sample_common(3),
|
|
||||||
target: TargetDescriptor {
|
|
||||||
role: Some("combobox".into()),
|
|
||||||
name: Some("分类".into()),
|
|
||||||
tag: "select".into(),
|
|
||||||
name_attr: None,
|
|
||||||
placeholder: None,
|
|
||||||
nearby_label: None,
|
|
||||||
},
|
|
||||||
selection: vec![SelectedOption {
|
|
||||||
value: "tech".into(),
|
|
||||||
label: Some("技术分享".into()),
|
|
||||||
}],
|
|
||||||
};
|
|
||||||
let v = serde_json::to_value(&step).unwrap();
|
|
||||||
assert_eq!(v["selection"][0]["value"], "tech");
|
|
||||||
assert_eq!(v["selection"][0]["label"], "技术分享");
|
|
||||||
let round: Step = serde_json::from_value(v).unwrap();
|
|
||||||
assert_eq!(round, step);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn step_navigate_round_trips() {
|
|
||||||
let step = Step::Navigate {
|
let step = Step::Navigate {
|
||||||
common: sample_common(1),
|
common: sample_common(1, "s1", "s2"),
|
||||||
to: "https://example.com".into(),
|
to: "https://example.com".into(),
|
||||||
|
cause: NavigationCause::UserTyped,
|
||||||
};
|
};
|
||||||
let v = serde_json::to_value(&step).unwrap();
|
let v = serde_json::to_value(&step).unwrap();
|
||||||
assert_eq!(v["op"], "navigate");
|
assert_eq!(v["op"], "navigate");
|
||||||
assert_eq!(v["to"], "https://example.com");
|
assert_eq!(v["cause"], "user_typed");
|
||||||
let round: Step = serde_json::from_value(v).unwrap();
|
let round: Step = serde_json::from_value(v).unwrap();
|
||||||
assert_eq!(round, step);
|
assert_eq!(round, step);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn step_press_with_modifiers_round_trips() {
|
fn trace_v3_round_trips() {
|
||||||
let step = Step::Press {
|
|
||||||
common: StepCommon {
|
|
||||||
effect: Some(StepEffect {
|
|
||||||
navigated_to: "p2".into(),
|
|
||||||
}),
|
|
||||||
..sample_common(2)
|
|
||||||
},
|
|
||||||
key: "Enter".into(),
|
|
||||||
modifiers: Some(vec![KeyModifier::Ctrl, KeyModifier::Shift]),
|
|
||||||
target: Some(sample_target()),
|
|
||||||
};
|
|
||||||
let v = serde_json::to_value(&step).unwrap();
|
|
||||||
assert_eq!(v["key"], "Enter");
|
|
||||||
assert_eq!(v["modifiers"], json!(["ctrl", "shift"]));
|
|
||||||
assert_eq!(v["effect"]["navigated_to"], "p2");
|
|
||||||
let round: Step = serde_json::from_value(v).unwrap();
|
|
||||||
assert_eq!(round, step);
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn trace_record_only_round_trips() {
|
|
||||||
let trace = sample_trace();
|
let trace = sample_trace();
|
||||||
let v = serde_json::to_value(&trace).unwrap();
|
let v = serde_json::to_value(&trace).unwrap();
|
||||||
assert!(v.get("version").is_none());
|
assert_eq!(v.get("version").and_then(|v| v.as_u64()), Some(3));
|
||||||
assert_eq!(
|
assert!(v.get("pages").is_none());
|
||||||
v.get("purpose").and_then(|v| v.as_str()),
|
assert_eq!(v["recorder"]["vom"], 1);
|
||||||
Some("发布一篇文章")
|
assert_eq!(v["states"].as_array().unwrap().len(), 2);
|
||||||
);
|
|
||||||
assert!(v.get("parameters").is_none());
|
|
||||||
assert!(v.get("site").is_none());
|
|
||||||
assert!(v.get("goal").is_none());
|
|
||||||
assert_eq!(
|
|
||||||
v.get("entry")
|
|
||||||
.and_then(|e| e.get("start_url"))
|
|
||||||
.and_then(|u| u.as_str()),
|
|
||||||
Some("https://x.com/editor")
|
|
||||||
);
|
|
||||||
let round: Trace = serde_json::from_value(v).unwrap();
|
let round: Trace = serde_json::from_value(v).unwrap();
|
||||||
assert_eq!(round, trace);
|
assert_eq!(round, trace);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn discriminated_union_click_ignores_extra_key_field() {
|
fn default_fields_are_omitted() {
|
||||||
let bad = json!({
|
let step = Step::Click {
|
||||||
"op": "click",
|
common: sample_common(1, "s1", "s1"),
|
||||||
"id": 1,
|
target: TargetDescriptor {
|
||||||
"page": "p1",
|
element_ref: Some("e1".into()),
|
||||||
"target": { "tag": "button", "name": "OK" },
|
role: Some("button".into()),
|
||||||
"key": "Enter"
|
name: Some("OK".into()),
|
||||||
});
|
ctx: None,
|
||||||
let step: Step = serde_json::from_value(bad).unwrap();
|
unmatched: false,
|
||||||
assert!(matches!(step, Step::Click { .. }));
|
},
|
||||||
|
};
|
||||||
let v = serde_json::to_value(&step).unwrap();
|
let v = serde_json::to_value(&step).unwrap();
|
||||||
assert!(v.get("key").is_none());
|
assert!(v.get("unmatched").is_none());
|
||||||
|
assert!(v["target"].get("ctx").is_none());
|
||||||
|
assert!(v["target"].get("unmatched").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unmatched_target_serializes_flag() {
|
||||||
|
let step = Step::Click {
|
||||||
|
common: sample_common(1, "s1", "s2"),
|
||||||
|
target: TargetDescriptor {
|
||||||
|
element_ref: None,
|
||||||
|
role: Some("button".into()),
|
||||||
|
name: Some("发布".into()),
|
||||||
|
ctx: None,
|
||||||
|
unmatched: true,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
let v = serde_json::to_value(&step).unwrap();
|
||||||
|
assert_eq!(v["target"]["unmatched"], true);
|
||||||
|
assert!(v["target"].get("ref").is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn extension_trace_deserializes() {
|
fn extension_trace_deserializes() {
|
||||||
let v = json!({
|
let v = json!({
|
||||||
|
"version": 3,
|
||||||
"recorded_at": "2026-07-21T08:00:00Z",
|
"recorded_at": "2026-07-21T08:00:00Z",
|
||||||
"started_at": "2026-07-21T07:59:00Z",
|
"started_at": "2026-07-21T07:59:00Z",
|
||||||
"purpose": "demo",
|
"purpose": "demo",
|
||||||
|
"stopped_by": "user_finish",
|
||||||
"entry": { "start_url": "https://example.com/editor" },
|
"entry": { "start_url": "https://example.com/editor" },
|
||||||
"pages": [
|
"recorder": { "bsk": "0.1.10", "vom": 1 },
|
||||||
{ "id": "p1", "url": "https://example.com/editor" },
|
"states": [
|
||||||
{ "id": "p2", "url": "https://example.com/p/99" }
|
{ "id": "s1", "url": "https://example.com/editor", "page": "s1.vom.txt" },
|
||||||
|
{ "id": "s2", "url": "https://example.com/p/99", "page": "s2.vom.txt" }
|
||||||
],
|
],
|
||||||
"steps": [
|
"steps": [
|
||||||
{
|
{
|
||||||
"op": "fill",
|
"op": "fill",
|
||||||
"id": 1,
|
"id": 1,
|
||||||
"page": "p1",
|
"state": "s1",
|
||||||
"target": { "tag": "input", "role": "textbox", "name": "标题" },
|
"result": { "state": "s1" },
|
||||||
"value": "hello"
|
"target": { "ref": "e1", "role": "textbox", "name": "标题" },
|
||||||
|
"value": "hello",
|
||||||
|
"commit": "blur"
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
"op": "click",
|
"op": "click",
|
||||||
"id": 2,
|
"id": 2,
|
||||||
"page": "p1",
|
"state": "s1",
|
||||||
"target": { "tag": "button", "role": "button", "name": "发布" },
|
"result": { "state": "s2" },
|
||||||
"effect": { "navigated_to": "p2" }
|
"target": { "ref": "e2", "role": "button", "name": "发布" }
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
});
|
});
|
||||||
let trace: Trace = serde_json::from_value(v).unwrap();
|
let trace: Trace = serde_json::from_value(v).unwrap();
|
||||||
assert_eq!(trace.entry.start_url, "https://example.com/editor");
|
assert_eq!(trace.version, 3);
|
||||||
assert_eq!(trace.pages.len(), 2);
|
assert_eq!(trace.states.len(), 2);
|
||||||
assert_eq!(trace.steps.len(), 2);
|
assert_eq!(trace.steps.len(), 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn trace_deserializes_optional_frame_capture_metadata() {
|
||||||
|
let v = json!({
|
||||||
|
"version": 3,
|
||||||
|
"recorded_at": "2026-07-21T08:00:00Z",
|
||||||
|
"stopped_by": "cli_stop",
|
||||||
|
"entry": { "start_url": "https://example.com/editor" },
|
||||||
|
"recorder": { "bsk": "0.1.10", "vom": 1 },
|
||||||
|
"frame_capture": {
|
||||||
|
"status": "partial",
|
||||||
|
"failures": [
|
||||||
|
{
|
||||||
|
"reason": "not_injectable",
|
||||||
|
"frame_id": 2,
|
||||||
|
"url": "chrome://settings",
|
||||||
|
"detail": "restricted frame url"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"states": [],
|
||||||
|
"steps": []
|
||||||
|
});
|
||||||
|
let trace: Trace = serde_json::from_value(v).unwrap();
|
||||||
|
let frame_capture = trace.frame_capture.expect("frame_capture");
|
||||||
|
assert_eq!(frame_capture.status, FrameCaptureStatus::Partial);
|
||||||
|
let failures = frame_capture.failures.expect("failures");
|
||||||
|
assert_eq!(failures.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
failures[0].reason,
|
||||||
|
FrameCaptureFailureReason::NotInjectable
|
||||||
|
);
|
||||||
|
assert_eq!(failures[0].frame_id, 2);
|
||||||
|
assert_eq!(failures[0].url.as_deref(), Some("chrome://settings"));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,20 +6,20 @@
|
|||||||
"disconnected": "Disconnected",
|
"disconnected": "Disconnected",
|
||||||
"connecting": "Connecting…",
|
"connecting": "Connecting…",
|
||||||
"connected": "Connected",
|
"connected": "Connected",
|
||||||
"version_skew": "Protocol mismatch",
|
"version_skew": "Version mismatch",
|
||||||
"disabled": "Connection off"
|
"disabled": "Connection off"
|
||||||
},
|
},
|
||||||
"stateDetail": {
|
"stateDetail": {
|
||||||
"disconnected": "Open BrowserSkill to connect.",
|
"disconnected": "Open BrowserSkill to connect.",
|
||||||
"connecting": "Connecting to BrowserSkill…",
|
"connecting": "Connecting to BrowserSkill…",
|
||||||
"connected": "Ready for automation tasks.",
|
"connected": "Ready for automation tasks.",
|
||||||
"version_skew": "Some features may be unavailable."
|
"version_skew": "bsk and the browser extension differ in version. You can continue using them, but updating is recommended."
|
||||||
},
|
},
|
||||||
"stateBadge": {
|
"stateBadge": {
|
||||||
"disconnected": "Standby",
|
"disconnected": "Standby",
|
||||||
"connecting": "Igniting",
|
"connecting": "Igniting",
|
||||||
"connected": "Ready",
|
"connected": "Ready",
|
||||||
"version_skew": "Action needed",
|
"version_skew": "Update recommended",
|
||||||
"disabled": "Off"
|
"disabled": "Off"
|
||||||
},
|
},
|
||||||
"connectionToggleTitle": "BrowserSkill connection",
|
"connectionToggleTitle": "BrowserSkill connection",
|
||||||
@@ -27,6 +27,7 @@
|
|||||||
"controlHintsToggleHint": "Show the status pill and orange glow while the Agent controls a page.",
|
"controlHintsToggleHint": "Show the status pill and orange glow while the Agent controls a page.",
|
||||||
"controlHintsInfoLabel": "About control hints",
|
"controlHintsInfoLabel": "About control hints",
|
||||||
"versionSkewWarning": "Extension protocol v{{extensionProtocol}}, daemon v{{daemonProtocol}}.",
|
"versionSkewWarning": "Extension protocol v{{extensionProtocol}}, daemon v{{daemonProtocol}}.",
|
||||||
|
"versionIncompatibleError": "bsk and the BrowserSkill extension are incompatible. Update both, reload the extension, then run bsk daemon restart.",
|
||||||
"upgradeAvailable": "Upgradable",
|
"upgradeAvailable": "Upgradable",
|
||||||
"launcher": {
|
"launcher": {
|
||||||
"title": "Quick actions"
|
"title": "Quick actions"
|
||||||
@@ -48,7 +49,7 @@
|
|||||||
"record": {
|
"record": {
|
||||||
"sectionTitle": "Action recording",
|
"sectionTitle": "Action recording",
|
||||||
"cardDesc": "Record your actions for Agent reference",
|
"cardDesc": "Record your actions for Agent reference",
|
||||||
"sectionHint": "Generates a recording instruction — send it to your Agent to start recording your actions; the Agent can then use the recording to automate similar tasks.",
|
"sectionHint": "Generates a recording instruction — send it to your Agent to start recording your actions; the Agent can then use the resulting trace bundle to automate similar tasks.",
|
||||||
"topicLabel": "Topic (optional)",
|
"topicLabel": "Topic (optional)",
|
||||||
"topicPlaceholder": "e.g. Publish an article",
|
"topicPlaceholder": "e.g. Publish an article",
|
||||||
"startUrlLabel": "Starting URL (optional)",
|
"startUrlLabel": "Starting URL (optional)",
|
||||||
@@ -56,7 +57,7 @@
|
|||||||
"copyButton": "Copy recording instructions",
|
"copyButton": "Copy recording instructions",
|
||||||
"copied": "Copied",
|
"copied": "Copied",
|
||||||
"hintDisconnected": "Available when connected",
|
"hintDisconnected": "Available when connected",
|
||||||
"promptTemplate": "Use BrowserSkill's `bsk` CLI to record the browser actions I'm about to demonstrate.\n\nSteps:\n1. Run: {{command}}\n2. Once you confirm it's started, I'll operate in the opened window myself.\n3. After I click Finish, read the returned trace.json and summarize the actions (pages + steps); don't re-run or \"verify\"."
|
"promptTemplate": "Use BrowserSkill's `bsk` CLI to record the browser actions I'm about to demonstrate.\n\nSteps:\n1. Run: {{command}}\n2. Once you confirm it's started, I'll operate in the opened window myself.\n3. After I click Finish, read the written `./trace` bundle (`trace.json` + `pages/`) and summarize what I did."
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"controlOverlay": {
|
"controlOverlay": {
|
||||||
@@ -89,6 +90,7 @@
|
|||||||
},
|
},
|
||||||
"recordOverlay": {
|
"recordOverlay": {
|
||||||
"recording": "Recording your actions",
|
"recording": "Recording your actions",
|
||||||
"finish": "Finish"
|
"finish": "Finish",
|
||||||
|
"partialCapture": "Some embedded frames may not be recorded."
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,20 +6,20 @@
|
|||||||
"disconnected": "未连接",
|
"disconnected": "未连接",
|
||||||
"connecting": "连接中…",
|
"connecting": "连接中…",
|
||||||
"connected": "已连接",
|
"connected": "已连接",
|
||||||
"version_skew": "协议不一致",
|
"version_skew": "版本不一致",
|
||||||
"disabled": "连接已关闭"
|
"disabled": "连接已关闭"
|
||||||
},
|
},
|
||||||
"stateDetail": {
|
"stateDetail": {
|
||||||
"disconnected": "请先打开 BrowserSkill。",
|
"disconnected": "请先打开 BrowserSkill。",
|
||||||
"connecting": "正在连接 BrowserSkill…",
|
"connecting": "正在连接 BrowserSkill…",
|
||||||
"connected": "可接收自动化任务。",
|
"connected": "可接收自动化任务。",
|
||||||
"version_skew": "部分能力可能不可用。"
|
"version_skew": "bsk 与浏览器扩展版本不同,当前仍可使用,建议更新至最新版本。"
|
||||||
},
|
},
|
||||||
"stateBadge": {
|
"stateBadge": {
|
||||||
"disconnected": "Standby",
|
"disconnected": "Standby",
|
||||||
"connecting": "Igniting",
|
"connecting": "Igniting",
|
||||||
"connected": "Ready",
|
"connected": "Ready",
|
||||||
"version_skew": "Action needed",
|
"version_skew": "建议更新",
|
||||||
"disabled": "Off"
|
"disabled": "Off"
|
||||||
},
|
},
|
||||||
"connectionToggleTitle": "BrowserSkill 连接",
|
"connectionToggleTitle": "BrowserSkill 连接",
|
||||||
@@ -27,6 +27,7 @@
|
|||||||
"controlHintsToggleHint": "Agent 控制页面时显示提示条和橙色闪光。",
|
"controlHintsToggleHint": "Agent 控制页面时显示提示条和橙色闪光。",
|
||||||
"controlHintsInfoLabel": "控制提示说明",
|
"controlHintsInfoLabel": "控制提示说明",
|
||||||
"versionSkewWarning": "扩展协议 v{{extensionProtocol}},daemon 协议 v{{daemonProtocol}}。",
|
"versionSkewWarning": "扩展协议 v{{extensionProtocol}},daemon 协议 v{{daemonProtocol}}。",
|
||||||
|
"versionIncompatibleError": "bsk 与 BrowserSkill 扩展版本不兼容。请将两者更新至最新版本,重新加载扩展后,再运行 bsk daemon restart。",
|
||||||
"upgradeAvailable": "可升级",
|
"upgradeAvailable": "可升级",
|
||||||
"launcher": {
|
"launcher": {
|
||||||
"title": "快捷功能"
|
"title": "快捷功能"
|
||||||
@@ -48,7 +49,7 @@
|
|||||||
"record": {
|
"record": {
|
||||||
"sectionTitle": "操作录制",
|
"sectionTitle": "操作录制",
|
||||||
"cardDesc": "录制你的操作,供 Agent 参考",
|
"cardDesc": "录制你的操作,供 Agent 参考",
|
||||||
"sectionHint": "生成一段录制指令,发给 Agent 后可以启动插件录制用户操作,之后 Agent 可以参考录制结果自动完成同类任务。",
|
"sectionHint": "生成一段录制指令,发给 Agent 后可以启动插件录制用户操作,之后 Agent 可以参考写出的 trace 目录包自动完成同类任务。",
|
||||||
"topicLabel": "操作主题(可选)",
|
"topicLabel": "操作主题(可选)",
|
||||||
"topicPlaceholder": "例如:发布一篇文章",
|
"topicPlaceholder": "例如:发布一篇文章",
|
||||||
"startUrlLabel": "起始网址(可选)",
|
"startUrlLabel": "起始网址(可选)",
|
||||||
@@ -56,7 +57,7 @@
|
|||||||
"copyButton": "复制录制指令",
|
"copyButton": "复制录制指令",
|
||||||
"copied": "已复制",
|
"copied": "已复制",
|
||||||
"hintDisconnected": "连接后可用",
|
"hintDisconnected": "连接后可用",
|
||||||
"promptTemplate": "用 BrowserSkill 的 `bsk` CLI 录制我接下来的浏览器操作。\n\n步骤:\n1. 执行:{{command}}\n2. 你确认已开始后,我会在打开的窗口里自己操作。\n3. 我点“结束”后,读取返回的 trace.json 并总结操作(pages + steps),不要重跑或“验证”。"
|
"promptTemplate": "用 BrowserSkill 的 `bsk` CLI 录制我接下来的浏览器操作。\n\n步骤:\n1. 执行:{{command}}\n2. 你确认已开始后,我会在打开的窗口里自己操作。\n3. 我点“结束”后,读取写出的 `./trace` 目录包(`trace.json` + `pages/`),总结我做了什么。"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"controlOverlay": {
|
"controlOverlay": {
|
||||||
@@ -89,6 +90,7 @@
|
|||||||
},
|
},
|
||||||
"recordOverlay": {
|
"recordOverlay": {
|
||||||
"recording": "正在录制你的操作",
|
"recording": "正在录制你的操作",
|
||||||
"finish": "结束"
|
"finish": "结束",
|
||||||
|
"partialCapture": "部分嵌入页面可能未被录制。"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -24,6 +24,11 @@ function scene(nodes: VomNode[]): VomScene {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Compare refs without recording-metadata fields (role/name/ctx/line). */
|
||||||
|
function coreRefs(refs: ReturnType<typeof renderVom>["refs"]) {
|
||||||
|
return refs.map(({ ref: id, backendNodeId }) => ({ ref: id, backendNodeId }));
|
||||||
|
}
|
||||||
|
|
||||||
describe("renderVom single-layer page", () => {
|
describe("renderVom single-layer page", () => {
|
||||||
it("always emits a complete @vom single-layer document when there is no blocker", () => {
|
it("always emits a complete @vom single-layer document when there is no blocker", () => {
|
||||||
const out = renderVom(
|
const out = renderVom(
|
||||||
@@ -51,7 +56,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
' @e2 button "加入购物车"',
|
' @e2 button "加入购物车"',
|
||||||
].join("\n"),
|
].join("\n"),
|
||||||
);
|
);
|
||||||
expect(out.refs).toEqual([
|
expect(coreRefs(out.refs)).toEqual([
|
||||||
{ ref: "e1", backendNodeId: 3 },
|
{ ref: "e1", backendNodeId: 3 },
|
||||||
{ ref: "e2", backendNodeId: 6 },
|
{ ref: "e2", backendNodeId: 6 },
|
||||||
]);
|
]);
|
||||||
@@ -72,7 +77,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
expect(out.text).toContain(' heading "Title"');
|
expect(out.text).toContain(' heading "Title"');
|
||||||
expect(out.text).toContain(' img "Logo"');
|
expect(out.text).toContain(' img "Logo"');
|
||||||
expect(out.text).toContain(' @e1 button "Continue"');
|
expect(out.text).toContain(' @e1 button "Continue"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("assigns refs to listbox controls", () => {
|
it("assigns refs to listbox controls", () => {
|
||||||
@@ -84,7 +89,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
);
|
);
|
||||||
|
|
||||||
expect(out.text).toContain(' @e1 listbox "Choices"');
|
expect(out.text).toContain(' @e1 listbox "Choices"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("annotates external links regardless of role casing", () => {
|
it("annotates external links regardless of role casing", () => {
|
||||||
@@ -121,7 +126,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
expect(out.text).toContain(" alertdialog");
|
expect(out.text).toContain(" alertdialog");
|
||||||
expect(out.text).toContain(" section");
|
expect(out.text).toContain(" section");
|
||||||
expect(out.text).toContain(' @e1 button "Close"');
|
expect(out.text).toContain(' @e1 button "Close"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 5 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 5 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("skips generic nodes without adding an extra indentation level", () => {
|
it("skips generic nodes without adding an extra indentation level", () => {
|
||||||
@@ -192,7 +197,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
expect(out.text).toContain(" section");
|
expect(out.text).toContain(" section");
|
||||||
expect(out.text).not.toContain("Too deep");
|
expect(out.text).not.toContain("Too deep");
|
||||||
expect(out.text).toContain(' @e1 button "Later"');
|
expect(out.text).toContain(' @e1 button "Later"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("truncates when maxTokens is exceeded while keeping refs in sync with rendered text", () => {
|
it("truncates when maxTokens is exceeded while keeping refs in sync with rendered text", () => {
|
||||||
@@ -227,7 +232,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
);
|
);
|
||||||
|
|
||||||
expect(out.text).toContain('@e1 button "Open settings"');
|
expect(out.text).toContain('@e1 button "Open settings"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("keeps the deepest custom control when wrapper and child both look clickable", () => {
|
it("keeps the deepest custom control when wrapper and child both look clickable", () => {
|
||||||
@@ -257,7 +262,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
|
|
||||||
expect(out.text).not.toContain('@e1 button "Card"');
|
expect(out.text).not.toContain('@e1 button "Card"');
|
||||||
expect(out.text).toContain('@e1 button "Details"');
|
expect(out.text).toContain('@e1 button "Details"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not recover clickable wrappers around native controls", () => {
|
it("does not recover clickable wrappers around native controls", () => {
|
||||||
@@ -280,7 +285,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
|
|
||||||
expect(out.text).not.toContain('@e1 button "Checkout"');
|
expect(out.text).not.toContain('@e1 button "Checkout"');
|
||||||
expect(out.text).toContain('@e1 button "Pay"');
|
expect(out.text).toContain('@e1 button "Pay"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("adds context to duplicate weak action labels", () => {
|
it("adds context to duplicate weak action labels", () => {
|
||||||
@@ -343,7 +348,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
|
|
||||||
expect(out.text).toContain("@layers 1 focus=L1");
|
expect(out.text).toContain("@layers 1 focus=L1");
|
||||||
expect(out.text).toContain('@e1 button "Background"');
|
expect(out.text).toContain('@e1 button "Background"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("can filter refs blocked by active positioned regions without changing layer output", () => {
|
it("can filter refs blocked by active positioned regions without changing layer output", () => {
|
||||||
@@ -386,7 +391,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
expect(out.text).not.toContain("active-region");
|
expect(out.text).not.toContain("active-region");
|
||||||
expect(out.text).not.toContain("Background");
|
expect(out.text).not.toContain("Background");
|
||||||
expect(out.text).toContain('@e1 button "Foreground"');
|
expect(out.text).toContain('@e1 button "Foreground"');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("renders conditional surface items inline on the trigger", () => {
|
it("renders conditional surface items inline on the trigger", () => {
|
||||||
@@ -405,7 +410,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
expect(out.text).toContain('@e1 button "Products" [hover first: Shoes | Bags | Accessories]');
|
expect(out.text).toContain('@e1 button "Products" [hover first: Shoes | Bags | Accessories]');
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("injects active scope blocks immediately after their trigger without refs", () => {
|
it("injects active scope blocks immediately after their trigger without refs", () => {
|
||||||
@@ -431,7 +436,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
" Bob - great sound",
|
" Bob - great sound",
|
||||||
].join("\n"),
|
].join("\n"),
|
||||||
);
|
);
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not render redundant children inside named ref controls", () => {
|
it("does not render redundant children inside named ref controls", () => {
|
||||||
@@ -445,7 +450,7 @@ describe("renderVom single-layer page", () => {
|
|||||||
|
|
||||||
expect(out.text).toContain('@e1 button "Save"');
|
expect(out.text).toContain('@e1 button "Save"');
|
||||||
expect(out.text).not.toContain("disk icon");
|
expect(out.text).not.toContain("disk icon");
|
||||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("does not render redundant children inside native text inputs", () => {
|
it("does not render redundant children inside native text inputs", () => {
|
||||||
@@ -612,7 +617,7 @@ describe("renderVom double-layer page", () => {
|
|||||||
].join("\n"),
|
].join("\n"),
|
||||||
);
|
);
|
||||||
expect(out.text).not.toContain("底层按钮");
|
expect(out.text).not.toContain("底层按钮");
|
||||||
expect(out.refs).toEqual([
|
expect(coreRefs(out.refs)).toEqual([
|
||||||
{ ref: "e1", backendNodeId: 4 },
|
{ ref: "e1", backendNodeId: 4 },
|
||||||
{ ref: "e2", backendNodeId: 5 },
|
{ ref: "e2", backendNodeId: 5 },
|
||||||
]);
|
]);
|
||||||
@@ -664,4 +669,24 @@ describe("renderVom double-layer page", () => {
|
|||||||
expect(out.text).toContain(' @e2 textbox "密码" [filled] ="•••"');
|
expect(out.text).toContain(' @e2 textbox "密码" [filled] ="•••"');
|
||||||
expect(out.refs.map((r) => r.backendNodeId)).toEqual([101, 102]);
|
expect(out.refs.map((r) => r.backendNodeId)).toEqual([101, 102]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("redactValues omits literal form values", () => {
|
||||||
|
const out = renderVom(
|
||||||
|
scene([
|
||||||
|
node({ id: 1, role: "RootWebArea", name: "Form" }),
|
||||||
|
node({
|
||||||
|
id: 2,
|
||||||
|
parentId: 1,
|
||||||
|
role: "textbox",
|
||||||
|
name: "邮箱",
|
||||||
|
value: "user@example.com",
|
||||||
|
inputState: "filled",
|
||||||
|
tag: "input",
|
||||||
|
}),
|
||||||
|
]),
|
||||||
|
{ redactValues: true },
|
||||||
|
);
|
||||||
|
expect(out.text).toContain("[filled]");
|
||||||
|
expect(out.text).not.toContain("user@example.com");
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ export type {
|
|||||||
CondSurface,
|
CondSurface,
|
||||||
LayerKind,
|
LayerKind,
|
||||||
Rect,
|
Rect,
|
||||||
|
RenderedRef,
|
||||||
Viewport,
|
Viewport,
|
||||||
VomNode,
|
VomNode,
|
||||||
VomOptions,
|
VomOptions,
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ import type {
|
|||||||
BlockingLayer,
|
BlockingLayer,
|
||||||
CondSurface,
|
CondSurface,
|
||||||
Rect,
|
Rect,
|
||||||
|
RenderedRef,
|
||||||
VomNode,
|
VomNode,
|
||||||
VomOptions,
|
VomOptions,
|
||||||
VomResult,
|
VomResult,
|
||||||
@@ -606,6 +607,7 @@ function renderNodeLine(
|
|||||||
ref: string | undefined,
|
ref: string | undefined,
|
||||||
context: string[] = [],
|
context: string[] = [],
|
||||||
surface: CondSurface | undefined = undefined,
|
surface: CondSurface | undefined = undefined,
|
||||||
|
redactValues = false,
|
||||||
): string {
|
): string {
|
||||||
let line = `${" ".repeat(depth)}${ref ? `@${ref} ` : ""}${node.role}`;
|
let line = `${" ".repeat(depth)}${ref ? `@${ref} ` : ""}${node.role}`;
|
||||||
|
|
||||||
@@ -627,11 +629,12 @@ function renderNodeLine(
|
|||||||
}
|
}
|
||||||
const placeholder = cleaned(node.placeholder);
|
const placeholder = cleaned(node.placeholder);
|
||||||
if (placeholder) line += ` placeholder=${JSON.stringify(placeholder)}`;
|
if (placeholder) line += ` placeholder=${JSON.stringify(placeholder)}`;
|
||||||
const value = node.sensitive
|
const value =
|
||||||
? rawValue !== undefined
|
node.sensitive || redactValues
|
||||||
? SENSITIVE_MASK
|
? rawValue !== undefined
|
||||||
: undefined
|
? SENSITIVE_MASK
|
||||||
: rawValue?.slice(0, MAX_VALUE_LENGTH);
|
: undefined
|
||||||
|
: rawValue?.slice(0, MAX_VALUE_LENGTH);
|
||||||
if (value !== undefined) line += ` =${JSON.stringify(value)}`;
|
if (value !== undefined) line += ` =${JSON.stringify(value)}`;
|
||||||
|
|
||||||
if (surface && surface.subItems.length > 0) {
|
if (surface && surface.subItems.length > 0) {
|
||||||
@@ -684,11 +687,12 @@ function shouldSkipRedundantChildren(node: VomNode, state: RenderState): boolean
|
|||||||
|
|
||||||
interface RenderState {
|
interface RenderState {
|
||||||
lines: string[];
|
lines: string[];
|
||||||
refs: Array<{ ref: string; backendNodeId: number }>;
|
refs: RenderedRef[];
|
||||||
nextRef: number;
|
nextRef: number;
|
||||||
tokens: number;
|
tokens: number;
|
||||||
maxDepth: number;
|
maxDepth: number;
|
||||||
maxTokens: number;
|
maxTokens: number;
|
||||||
|
redactValues: boolean;
|
||||||
truncated: boolean;
|
truncated: boolean;
|
||||||
stopped: boolean;
|
stopped: boolean;
|
||||||
children: Map<number | null, VomNode[]>;
|
children: Map<number | null, VomNode[]>;
|
||||||
@@ -753,7 +757,7 @@ function renderTree(
|
|||||||
const ref = isVomReferenceNode(node) ? `e${state.nextRef}` : undefined;
|
const ref = isVomReferenceNode(node) ? `e${state.nextRef}` : undefined;
|
||||||
const context = ref ? handleContext(node, state) : [];
|
const context = ref ? handleContext(node, state) : [];
|
||||||
const surface = state.surfaceMap.get(node.id);
|
const surface = state.surfaceMap.get(node.id);
|
||||||
const line = renderNodeLine(node, depth, ref, context, surface);
|
const line = renderNodeLine(node, depth, ref, context, surface, state.redactValues);
|
||||||
const nextTokens = state.tokens + estimateTokens(line);
|
const nextTokens = state.tokens + estimateTokens(line);
|
||||||
if (nextTokens > state.maxTokens) {
|
if (nextTokens > state.maxTokens) {
|
||||||
state.truncated = true;
|
state.truncated = true;
|
||||||
@@ -764,7 +768,16 @@ function renderTree(
|
|||||||
state.lines.push(line);
|
state.lines.push(line);
|
||||||
state.tokens = nextTokens;
|
state.tokens = nextTokens;
|
||||||
if (ref) {
|
if (ref) {
|
||||||
state.refs.push({ ref, backendNodeId: node.id });
|
const lineIndex = state.lines.length - 1;
|
||||||
|
const refName = cleaned(node.name);
|
||||||
|
state.refs.push({
|
||||||
|
ref,
|
||||||
|
backendNodeId: node.id,
|
||||||
|
role: node.role,
|
||||||
|
...(refName ? { name: refName } : {}),
|
||||||
|
...(context.length > 0 ? { ctx: context.join(" > ") } : {}),
|
||||||
|
line: lineIndex,
|
||||||
|
});
|
||||||
state.nextRef += 1;
|
state.nextRef += 1;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -793,6 +806,7 @@ function renderNodes(
|
|||||||
tokens: estimateTokens(initialLines.join("\n")),
|
tokens: estimateTokens(initialLines.join("\n")),
|
||||||
maxDepth: options.maxDepth ?? DEFAULT_MAX_DEPTH,
|
maxDepth: options.maxDepth ?? DEFAULT_MAX_DEPTH,
|
||||||
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
|
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
|
||||||
|
redactValues: options.redactValues ?? false,
|
||||||
truncated: false,
|
truncated: false,
|
||||||
stopped: false,
|
stopped: false,
|
||||||
children,
|
children,
|
||||||
|
|||||||
@@ -59,6 +59,10 @@ export interface VomScene {
|
|||||||
export interface VomOptions {
|
export interface VomOptions {
|
||||||
maxDepth?: number;
|
maxDepth?: number;
|
||||||
maxTokens?: number;
|
maxTokens?: number;
|
||||||
|
/**
|
||||||
|
* When true, all form values render as [filled]/[empty] without literal values.
|
||||||
|
*/
|
||||||
|
redactValues?: boolean;
|
||||||
/**
|
/**
|
||||||
* Experimental: filter refs that are geometrically blocked by foreground
|
* Experimental: filter refs that are geometrically blocked by foreground
|
||||||
* fixed/absolute/sticky regions. Disabled by default and does not alter the
|
* fixed/absolute/sticky regions. Disabled by default and does not alter the
|
||||||
@@ -79,9 +83,19 @@ export interface ActiveScopeBlock {
|
|||||||
lines: string[];
|
lines: string[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface RenderedRef {
|
||||||
|
ref: string;
|
||||||
|
backendNodeId: number;
|
||||||
|
role?: string;
|
||||||
|
name?: string;
|
||||||
|
ctx?: string;
|
||||||
|
/** Zero-based line index in `VomResult.text`. */
|
||||||
|
line: number;
|
||||||
|
}
|
||||||
|
|
||||||
export interface VomResult {
|
export interface VomResult {
|
||||||
text: string;
|
text: string;
|
||||||
refs: Array<{ ref: string; backendNodeId: number }>;
|
refs: RenderedRef[];
|
||||||
truncated: boolean;
|
truncated: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+14
-6
@@ -265,19 +265,27 @@ retry — complete the task autonomously or stop gracefully.
|
|||||||
|
|
||||||
### Recording — `bsk record`
|
### Recording — `bsk record`
|
||||||
|
|
||||||
Capture the user's own actions in the Agent Window to a `trace.json`, for later LLM-driven automation:
|
Capture the user's own actions in the Agent Window to a **trace v3 bundle**, for later LLM-driven automation:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
bsk record start --browser <instance-id-or-label> [--url https://…] [--purpose "publish a wiki doc"] [--output trace.json]
|
bsk record start --browser <instance-id-or-label> \
|
||||||
|
[--url https://…] [--purpose "publish a wiki doc"] \
|
||||||
|
[--max-page-tokens 3000] [--redact-values] \
|
||||||
|
[--output trace]
|
||||||
# `--url` is optional; default https://example.com/ when omitted (must be http(s)).
|
# `--url` is optional; default https://example.com/ when omitted (must be http(s)).
|
||||||
# Blocks until the user clicks Finish in the recording panel, then writes ./trace.json and closes the window.
|
# Blocks until the user clicks Finish in the recording panel, then writes:
|
||||||
|
# trace/trace.json — action chain + state index (version 3)
|
||||||
|
# trace/pages/ — one `.vom.txt` observe snapshot per settled page state
|
||||||
|
|
||||||
bsk record stop [--output trace.json] # terminal fallback if the browser panel is unavailable
|
bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable
|
||||||
```
|
```
|
||||||
|
|
||||||
- The trace is a **record-only action log** (a `pages[]` dictionary + `navigate`/`click`/`fill`/`select`/`press` steps with `target` descriptors). It records *what the user did*; deciding which inputs are variable is left to the executing agent.
|
- **Bundle layout:** `--output` is a directory (default `./trace`) containing `trace.json` and `pages/`. `trace.json` lists `states[]` (page observation ids) and `steps[]` (each step binds `state` = observe snapshot *before* the action and `result.state` = snapshot *after* settle). Page bodies live in `pages/sN.vom.txt` using the same VOM format as `bsk observe`.
|
||||||
|
- Each `states[]` entry is one **settled page observation** (captured after recording start, navigation landing, or action settle — not on a timer).
|
||||||
|
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
|
||||||
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
||||||
- There is **no** `bsk replay` — to redo a flow, read the trace and reuse the existing `session` / `snapshot` / `@eN` / `click` / `fill` tools. Follow **Stop when the goal is met**.
|
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
|
||||||
|
- Recording captures interactions in the **top frame and injectable child frames** (same-origin and cross-origin where the extension can attach). Embedded frames that cannot be injected are omitted; the trace may include `frame_capture.status: "partial"` with failure details.
|
||||||
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
||||||
|
|
||||||
## Error handling
|
## Error handling
|
||||||
|
|||||||
Reference in New Issue
Block a user