mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat: add bsk record for semantic user-action traces
Capture user actions in the Agent Window and export a record-only trace (pages[] + navigate/click/fill/select/press steps with target descriptors). Adds the record overlay UI aligned with the control overlay, and wires record start/stop/await through CLI, daemon, protocol, and extension.
This commit is contained in:
@@ -58,6 +58,7 @@ export function ControlOverlay({
|
||||
/>
|
||||
|
||||
<div
|
||||
data-slot="control-overlay-blocker"
|
||||
onPointerDown={(event) => {
|
||||
if (automationBypass) return;
|
||||
event.preventDefault();
|
||||
@@ -80,13 +81,16 @@ export function ControlOverlay({
|
||||
/>
|
||||
|
||||
<div
|
||||
data-slot="control-overlay-pill"
|
||||
style={{
|
||||
position: "fixed",
|
||||
bottom: 32,
|
||||
left: "50%",
|
||||
transform: "translateX(-50%)",
|
||||
zIndex: 2147483647,
|
||||
pointerEvents,
|
||||
// Always receive clicks so Interrupt works even while the page
|
||||
// blocker is in automation-bypass mode (CDP pass-through).
|
||||
pointerEvents: "auto",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 12,
|
||||
@@ -122,7 +126,7 @@ export function ControlOverlay({
|
||||
disabled={interrupting}
|
||||
onClick={onInterrupt}
|
||||
style={{
|
||||
pointerEvents,
|
||||
pointerEvents: "auto",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 6,
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
import { useTranslation } from "@browser-skill/i18n/react";
|
||||
import { RiCheckboxCircleLine } from "@remixicon/react";
|
||||
import { useEffect, useState } from "react";
|
||||
|
||||
export interface RecordRequestData {
|
||||
id: string;
|
||||
onFinish: () => void;
|
||||
}
|
||||
|
||||
type Props = {
|
||||
request: RecordRequestData | null;
|
||||
};
|
||||
|
||||
export function RecordOverlay({ request }: Props) {
|
||||
const { t } = useTranslation("extension");
|
||||
const [show, setShow] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (request) {
|
||||
const raf = requestAnimationFrame(() => setShow(true));
|
||||
return () => cancelAnimationFrame(raf);
|
||||
}
|
||||
setShow(false);
|
||||
}, [request]);
|
||||
|
||||
if (!request) return null;
|
||||
|
||||
return (
|
||||
<>
|
||||
<style>{`
|
||||
@keyframes bsk-rec-pulse {
|
||||
0%, 100% { opacity: 1; transform: scale(1); }
|
||||
50% { opacity: 0.35; transform: scale(0.82); }
|
||||
}
|
||||
`}</style>
|
||||
|
||||
<div
|
||||
data-slot="record-overlay-pill"
|
||||
style={{
|
||||
position: "fixed",
|
||||
bottom: 32,
|
||||
left: "50%",
|
||||
zIndex: 2147483647,
|
||||
pointerEvents: "auto",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 10,
|
||||
backgroundColor: "#fff",
|
||||
borderRadius: 9999,
|
||||
padding: "10px 10px 10px 20px",
|
||||
boxShadow: "0 8px 32px rgba(15,23,42,0.16), 0 2px 8px rgba(0,0,0,0.1)",
|
||||
opacity: show ? 1 : 0,
|
||||
transform: show ? "translateX(-50%) translateY(0)" : "translateX(-50%) translateY(8px)",
|
||||
transition: "opacity 300ms ease-out, transform 300ms ease-out",
|
||||
fontFamily:
|
||||
'-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif',
|
||||
maxWidth: "min(420px, calc(100vw - 32px))",
|
||||
}}
|
||||
>
|
||||
<span
|
||||
data-slot="record-overlay-indicator"
|
||||
style={{
|
||||
width: 9,
|
||||
height: 9,
|
||||
borderRadius: "50%",
|
||||
backgroundColor: "#ef4444",
|
||||
flexShrink: 0,
|
||||
animation: "bsk-rec-pulse 1.4s ease-in-out infinite",
|
||||
}}
|
||||
/>
|
||||
<span
|
||||
style={{
|
||||
flex: 1,
|
||||
fontSize: 16,
|
||||
fontWeight: 500,
|
||||
color: "#333",
|
||||
whiteSpace: "nowrap",
|
||||
userSelect: "none",
|
||||
overflow: "hidden",
|
||||
textOverflow: "ellipsis",
|
||||
}}
|
||||
>
|
||||
{t("recordOverlay.recording")}
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
data-slot="record-overlay-finish"
|
||||
onClick={request.onFinish}
|
||||
style={{
|
||||
pointerEvents: "auto",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 6,
|
||||
border: "none",
|
||||
borderRadius: 9999,
|
||||
padding: "8px 20px 8px 16px",
|
||||
fontSize: 15,
|
||||
fontWeight: 600,
|
||||
color: "#fff",
|
||||
backgroundColor: "#f97316",
|
||||
cursor: "pointer",
|
||||
transition: "background-color 150ms ease-out, opacity 150ms ease-out",
|
||||
whiteSpace: "nowrap",
|
||||
lineHeight: 1,
|
||||
flexShrink: 0,
|
||||
}}
|
||||
>
|
||||
<RiCheckboxCircleLine size={18} color="#fff" />
|
||||
{t("recordOverlay.finish")}
|
||||
</button>
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -7,7 +7,7 @@ describe("ControlOverlay", () => {
|
||||
cleanup();
|
||||
});
|
||||
|
||||
it("sets pointer-events none on blocker and pill when automationBypass is true", () => {
|
||||
it("keeps page blocker none under automationBypass but Interrupt stays clickable", () => {
|
||||
const { container } = render(
|
||||
<ControlOverlay
|
||||
visible={true}
|
||||
@@ -17,13 +17,17 @@ describe("ControlOverlay", () => {
|
||||
/>,
|
||||
);
|
||||
|
||||
const blocker = container.querySelector("[data-slot='control-overlay']")?.nextElementSibling;
|
||||
const blocker = container.querySelector("[data-slot='control-overlay-blocker']");
|
||||
expect(blocker).toBeTruthy();
|
||||
expect((blocker as HTMLElement).style.pointerEvents).toBe("none");
|
||||
|
||||
const pill = container.querySelector("[data-slot='control-overlay-pill']");
|
||||
expect(pill).toBeTruthy();
|
||||
expect((pill as HTMLElement).style.pointerEvents).toBe("auto");
|
||||
|
||||
const stopBtn = container.querySelector("[data-slot='control-overlay-stop-all']");
|
||||
expect(stopBtn).toBeTruthy();
|
||||
expect((stopBtn as HTMLElement).style.pointerEvents).toBe("none");
|
||||
expect((stopBtn as HTMLElement).style.pointerEvents).toBe("auto");
|
||||
});
|
||||
|
||||
it("uses pointer-events auto on blocker when automationBypass is false", () => {
|
||||
@@ -36,7 +40,7 @@ describe("ControlOverlay", () => {
|
||||
/>,
|
||||
);
|
||||
|
||||
const blocker = container.querySelector("[data-slot='control-overlay']")?.nextElementSibling;
|
||||
const blocker = container.querySelector("[data-slot='control-overlay-blocker']");
|
||||
expect(blocker).toBeTruthy();
|
||||
expect((blocker as HTMLElement).style.pointerEvents).toBe("auto");
|
||||
});
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
import { cleanup, fireEvent, render } from "@testing-library/react";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { RecordOverlay } from "../RecordOverlay";
|
||||
|
||||
describe("RecordOverlay", () => {
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
});
|
||||
|
||||
it("renders the bottom pill with orange finish CTA and no logo", () => {
|
||||
const onFinish = vi.fn();
|
||||
const { container } = render(<RecordOverlay request={{ id: "rec-1", onFinish }} />);
|
||||
|
||||
const pill = container.querySelector("[data-slot='record-overlay-pill']");
|
||||
expect(pill).toBeTruthy();
|
||||
expect(pill?.querySelector("img")).toBeNull();
|
||||
expect((pill as HTMLElement).style.borderRadius).toBe("9999px");
|
||||
expect((pill as HTMLElement).style.backgroundColor).toBe("#fff");
|
||||
|
||||
const finish = container.querySelector("[data-slot='record-overlay-finish']");
|
||||
expect(finish).toBeTruthy();
|
||||
expect((finish as HTMLElement).style.backgroundColor).toBe("#f97316");
|
||||
expect((finish as HTMLElement).style.borderRadius).toBe("9999px");
|
||||
|
||||
fireEvent.click(finish!);
|
||||
expect(onFinish).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("shows a pulsing recording indicator and no full-screen glow layer", () => {
|
||||
const { container } = render(<RecordOverlay request={{ id: "rec-1", onFinish: vi.fn() }} />);
|
||||
expect(container.querySelector("[data-slot='record-overlay']")).toBeNull();
|
||||
|
||||
const indicator = container.querySelector("[data-slot='record-overlay-indicator']");
|
||||
expect(indicator).toBeTruthy();
|
||||
expect((indicator as HTMLElement).style.animation).toContain("bsk-rec-pulse");
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { OverlayController } from "../overlay-controller";
|
||||
import { OverlayController, shouldShowAgentControlOverlay } from "../overlay-controller";
|
||||
|
||||
describe("OverlayController", () => {
|
||||
it("resets agent overlays without clearing user-tab borrow requests", () => {
|
||||
@@ -31,6 +31,7 @@ describe("OverlayController", () => {
|
||||
expect(state.controlVisible).toBe(false);
|
||||
expect(state.activeSessionId).toBeNull();
|
||||
expect(state.activeHelp).toBeNull();
|
||||
expect(state.activeRecord).toBeNull();
|
||||
expect(state.automationBypassCount).toBe(0);
|
||||
});
|
||||
|
||||
@@ -69,4 +70,50 @@ describe("OverlayController", () => {
|
||||
expect(previous?.id).toBe("help-1");
|
||||
expect(controller.snapshot().activeHelp?.id).toBe("help-2");
|
||||
});
|
||||
|
||||
it("tracks active record request state", () => {
|
||||
const controller = new OverlayController();
|
||||
|
||||
controller.activateAgentSession("sess-1");
|
||||
controller.setAgentRecordRequest({
|
||||
id: "rec-1",
|
||||
onFinish: vi.fn(),
|
||||
});
|
||||
|
||||
expect(controller.snapshot().activeRecord?.id).toBe("rec-1");
|
||||
});
|
||||
|
||||
it("hides agent control overlay while recording is active", () => {
|
||||
const controller = new OverlayController();
|
||||
controller.activateAgentSession("sess-1");
|
||||
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(true);
|
||||
|
||||
controller.setAgentRecordRequest({
|
||||
id: "rec-1",
|
||||
onFinish: vi.fn(),
|
||||
});
|
||||
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(false);
|
||||
expect(controller.snapshot().controlVisible).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps control hidden after record ends until session overlays reset", () => {
|
||||
const controller = new OverlayController();
|
||||
controller.activateAgentSession("sess-1");
|
||||
controller.setAgentRecordRequest({
|
||||
id: "rec-1",
|
||||
onFinish: vi.fn(),
|
||||
});
|
||||
controller.setAutomationBypass(true);
|
||||
controller.setAutomationBypass(true);
|
||||
|
||||
controller.clearAgentRecordRequest("rec-1");
|
||||
expect(controller.snapshot().activeRecord).toBeNull();
|
||||
expect(controller.snapshot().suppressControlAfterRecord).toBe(true);
|
||||
expect(controller.snapshot().automationBypassCount).toBe(0);
|
||||
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(false);
|
||||
|
||||
controller.resetAgentOverlays("sess-1");
|
||||
expect(controller.snapshot().suppressControlAfterRecord).toBe(false);
|
||||
expect(controller.snapshot().controlVisible).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { handleRecordContentMessage, startRecordCapture } from "../record-capture";
|
||||
import { RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
|
||||
|
||||
vi.stubGlobal("chrome", {
|
||||
runtime: {
|
||||
sendMessage: vi.fn(() => Promise.resolve()),
|
||||
},
|
||||
});
|
||||
|
||||
describe("handleRecordContentMessage stop/cancel", () => {
|
||||
it("ignores STOP when no recording is active", () => {
|
||||
const dispose = vi.fn();
|
||||
const onStop = vi.fn();
|
||||
const setActiveRequestId = vi.fn();
|
||||
const setCapture = vi.fn();
|
||||
const sendResponse = vi.fn();
|
||||
|
||||
const needsAsync = handleRecordContentMessage(
|
||||
{ type: RECORD_STOP, requestId: "rec-stale" },
|
||||
{
|
||||
activeRequestId: null,
|
||||
capture: { dispose },
|
||||
setActiveRequestId,
|
||||
setCapture,
|
||||
onStart: vi.fn(),
|
||||
onStop,
|
||||
},
|
||||
sendResponse,
|
||||
);
|
||||
|
||||
expect(needsAsync).toBe(false);
|
||||
expect(dispose).not.toHaveBeenCalled();
|
||||
expect(onStop).not.toHaveBeenCalled();
|
||||
expect(setActiveRequestId).not.toHaveBeenCalled();
|
||||
expect(setCapture).not.toHaveBeenCalled();
|
||||
expect(sendResponse).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("ignores STOP for a mismatched requestId", () => {
|
||||
const dispose = vi.fn();
|
||||
const onStop = vi.fn();
|
||||
|
||||
const needsAsync = handleRecordContentMessage(
|
||||
{ type: RECORD_STOP, requestId: "rec-other" },
|
||||
{
|
||||
activeRequestId: "rec-1",
|
||||
capture: { dispose },
|
||||
setActiveRequestId: vi.fn(),
|
||||
setCapture: vi.fn(),
|
||||
onStart: vi.fn(),
|
||||
onStop,
|
||||
},
|
||||
vi.fn(),
|
||||
);
|
||||
|
||||
expect(needsAsync).toBe(false);
|
||||
expect(dispose).not.toHaveBeenCalled();
|
||||
expect(onStop).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("record-capture semantic", () => {
|
||||
let steps: RecordStepPayload[];
|
||||
|
||||
beforeEach(() => {
|
||||
steps = [];
|
||||
document.body.innerHTML = `
|
||||
<label for="q">查询</label>
|
||||
<input id="q" name="q" />
|
||||
<button type="button" aria-label="搜索">搜索</button>
|
||||
<div role="listbox" id="sug">
|
||||
<div role="option">建议项</div>
|
||||
</div>
|
||||
`;
|
||||
});
|
||||
|
||||
it("commits final fill value and records semantic click", () => {
|
||||
const capture = startRecordCapture("rec-1", (step) => steps.push(step));
|
||||
const input = document.querySelector("input")!;
|
||||
const button = document.querySelector("button")!;
|
||||
|
||||
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||
input.value = "hello";
|
||||
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
input.dispatchEvent(new FocusEvent("focusout", { bubbles: true }));
|
||||
|
||||
button.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
||||
|
||||
capture.dispose();
|
||||
|
||||
expect(steps).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
op: "fill",
|
||||
value: "hello",
|
||||
target: expect.objectContaining({ name: "查询", tag: "input" }),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
op: "click",
|
||||
target: expect.objectContaining({ name: "搜索", role: "button" }),
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(JSON.stringify(steps)).not.toMatch(/@e\d+/);
|
||||
expect(steps.some((s) => "selector" in s)).toBe(false);
|
||||
});
|
||||
|
||||
it("ignores autocomplete suggestion clicks while a fill session is open", () => {
|
||||
const capture = startRecordCapture("rec-2", (step) => steps.push(step));
|
||||
const input = document.querySelector("input")!;
|
||||
const option = document.querySelector('[role="option"]')!;
|
||||
|
||||
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||
input.value = "hel";
|
||||
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
option.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
||||
input.value = "hello";
|
||||
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
input.dispatchEvent(new FocusEvent("focusout", { bubbles: true }));
|
||||
capture.dispose();
|
||||
|
||||
expect(steps.filter((s) => s.op === "click")).toHaveLength(0);
|
||||
expect(steps.some((s) => s.op === "fill" && s.value === "hello")).toBe(true);
|
||||
});
|
||||
|
||||
it("records Enter press but not bare typing keys", () => {
|
||||
const capture = startRecordCapture("rec-press", (step) => steps.push(step));
|
||||
const input = document.querySelector("input")!;
|
||||
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||
input.dispatchEvent(
|
||||
new KeyboardEvent("keydown", { key: "a", bubbles: true }),
|
||||
);
|
||||
input.dispatchEvent(
|
||||
new KeyboardEvent("keydown", { key: "Enter", bubbles: true }),
|
||||
);
|
||||
capture.dispose();
|
||||
expect(steps.filter((s) => s.op === "press")).toEqual([
|
||||
expect.objectContaining({ op: "press", key: "Enter" }),
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not record clicks on anonymous layout divs", () => {
|
||||
document.body.innerHTML = `
|
||||
<div id="chrome">page chrome</div>
|
||||
<button type="button" aria-label="下一步">下一步</button>
|
||||
`;
|
||||
const capture = startRecordCapture("rec-3", (step) => steps.push(step));
|
||||
document
|
||||
.querySelector("#chrome")!
|
||||
.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
||||
document
|
||||
.querySelector("button")!
|
||||
.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 }));
|
||||
capture.dispose();
|
||||
|
||||
expect(steps).toHaveLength(1);
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "click",
|
||||
target: { name: "下一步", role: "button" },
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,13 +1,21 @@
|
||||
import type { BorrowRequestData } from "./BorrowConfirmationOverlay";
|
||||
import type { HelpRequestData } from "./HelpRequestOverlay";
|
||||
import type { RecordRequestData } from "./RecordOverlay";
|
||||
|
||||
export interface OverlayState {
|
||||
borrowRequests: BorrowRequestData[];
|
||||
activeHelp: HelpRequestData | null;
|
||||
activeRecord: RecordRequestData | null;
|
||||
controlVisible: boolean;
|
||||
interrupting: boolean;
|
||||
activeSessionId: string | null;
|
||||
automationBypassCount: number;
|
||||
/**
|
||||
* After record Finish/Stop clears `activeRecord`, the session is still
|
||||
* alive until CLI `session.stop`. Suppress the control mask for that gap
|
||||
* so 「Agent 正在控制」does not flash between RecordOverlay and teardown.
|
||||
*/
|
||||
suppressControlAfterRecord: boolean;
|
||||
}
|
||||
|
||||
type MutableOverlayState = Omit<OverlayState, "controlVisible">;
|
||||
@@ -20,9 +28,11 @@ export class OverlayController {
|
||||
private state: MutableOverlayState = {
|
||||
borrowRequests: [],
|
||||
activeHelp: null,
|
||||
activeRecord: null,
|
||||
interrupting: false,
|
||||
activeSessionId: null,
|
||||
automationBypassCount: 0,
|
||||
suppressControlAfterRecord: false,
|
||||
};
|
||||
|
||||
snapshot(): OverlayState {
|
||||
@@ -70,6 +80,25 @@ export class OverlayController {
|
||||
this.state.activeHelp = null;
|
||||
}
|
||||
|
||||
setAgentRecordRequest(request: RecordRequestData): RecordRequestData | null {
|
||||
const previous = this.state.activeRecord;
|
||||
this.state.activeRecord = request;
|
||||
this.state.suppressControlAfterRecord = false;
|
||||
return previous;
|
||||
}
|
||||
|
||||
clearAgentRecordRequest(requestId?: string): void {
|
||||
if (!this.state.activeRecord) return;
|
||||
if (requestId && this.state.activeRecord.id !== requestId) return;
|
||||
this.state.activeRecord = null;
|
||||
// Hide control until session teardown — see OverlayState.suppressControlAfterRecord.
|
||||
this.state.suppressControlAfterRecord = true;
|
||||
// Record start/rearm may have stacked automation-bypass refs; drop them so a
|
||||
// stray ControlOverlay cannot sit with pointer-events:none (page usable,
|
||||
// Interrupt unclickable).
|
||||
this.state.automationBypassCount = 0;
|
||||
}
|
||||
|
||||
setAutomationBypass(enabled: boolean): void {
|
||||
if (enabled) {
|
||||
this.state.automationBypassCount += 1;
|
||||
@@ -86,10 +115,22 @@ export class OverlayController {
|
||||
this.state = {
|
||||
...this.state,
|
||||
activeHelp: null,
|
||||
activeRecord: null,
|
||||
interrupting: false,
|
||||
activeSessionId: null,
|
||||
automationBypassCount: 0,
|
||||
suppressControlAfterRecord: false,
|
||||
};
|
||||
return previousHelp;
|
||||
}
|
||||
}
|
||||
|
||||
/** Control mask ("Agent 正在控制") must hide while help/record overlays own the chrome. */
|
||||
export function shouldShowAgentControlOverlay(state: OverlayState): boolean {
|
||||
return (
|
||||
state.controlVisible &&
|
||||
!state.suppressControlAfterRecord &&
|
||||
state.activeHelp === null &&
|
||||
state.activeRecord === null
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,541 @@
|
||||
import {
|
||||
describeEventTarget,
|
||||
describeTarget,
|
||||
type TargetDescriptor,
|
||||
} from "@/lib/describe-target";
|
||||
import {
|
||||
isRecordCancelMessage,
|
||||
isRecordStartMessage,
|
||||
isRecordStopMessage,
|
||||
RECORD_CANCEL,
|
||||
RECORD_START,
|
||||
RECORD_STEP,
|
||||
RECORD_STOP,
|
||||
type RecordCancelMessage,
|
||||
type RecordStartAck,
|
||||
type RecordStartMessage,
|
||||
type RecordStepPayload,
|
||||
type RecordStopAck,
|
||||
type RecordStopMessage,
|
||||
} from "@/lib/record-bridge";
|
||||
import { shouldRecordPress } from "@/lib/trace-reducer";
|
||||
|
||||
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
|
||||
const failedStepDeliveries = new Set<string>();
|
||||
const knownRecordRequests = new Set<string>();
|
||||
|
||||
function sendRecordStep(requestId: string, step: RecordStepPayload): void {
|
||||
const payload = { type: RECORD_STEP, requestId, step };
|
||||
const pending = Promise.resolve(chrome.runtime.sendMessage(payload)).then(
|
||||
() => true,
|
||||
() => {
|
||||
failedStepDeliveries.add(requestId);
|
||||
return false;
|
||||
},
|
||||
);
|
||||
const sends = pendingStepSends.get(requestId) ?? new Set<Promise<boolean>>();
|
||||
sends.add(pending);
|
||||
pendingStepSends.set(requestId, sends);
|
||||
void pending.then(() => {
|
||||
sends.delete(pending);
|
||||
if (sends.size === 0) pendingStepSends.delete(requestId);
|
||||
});
|
||||
}
|
||||
|
||||
export interface RecordCaptureController {
|
||||
dispose(): void;
|
||||
}
|
||||
|
||||
interface FillSession {
|
||||
element: FillableElement;
|
||||
target: TargetDescriptor;
|
||||
baselineValue: string;
|
||||
lastValue: string;
|
||||
}
|
||||
|
||||
type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement;
|
||||
|
||||
function eventTarget(event: Event): EventTarget | null {
|
||||
return event.composedPath()[0] ?? event.target;
|
||||
}
|
||||
|
||||
function isOverlayTarget(target: EventTarget | null): boolean {
|
||||
if (!(target instanceof Node)) return false;
|
||||
const root = document.documentElement;
|
||||
let node: Node | null = target;
|
||||
while (node && node !== root) {
|
||||
if (node instanceof Element && node.hasAttribute("data-bsk-overlay")) {
|
||||
return true;
|
||||
}
|
||||
const rootNode: Node | Document | ShadowRoot = node.getRootNode();
|
||||
if (rootNode instanceof ShadowRoot) {
|
||||
const host: Element = rootNode.host;
|
||||
if (host.hasAttribute("data-bsk-overlay")) {
|
||||
return true;
|
||||
}
|
||||
node = host;
|
||||
} else {
|
||||
node = node.parentNode;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function isTextFillable(el: Element): el is FillableElement {
|
||||
if (el instanceof HTMLElement && (el.isContentEditable || el.contentEditable === "true")) {
|
||||
return true;
|
||||
}
|
||||
if (el instanceof HTMLTextAreaElement) return true;
|
||||
if (!(el instanceof HTMLInputElement)) return false;
|
||||
const type = el.type.toLowerCase();
|
||||
return type !== "checkbox" && type !== "radio" && type !== "file" && type !== "button";
|
||||
}
|
||||
|
||||
function fillableFromTarget(target: EventTarget | null): FillableElement | null {
|
||||
if (!(target instanceof Element)) return null;
|
||||
if (isTextFillable(target)) return target;
|
||||
const el = target.closest('input,textarea,[contenteditable]:not([contenteditable="false"])');
|
||||
if (el && isTextFillable(el)) return el;
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Clicks on search chrome that only focus the nearby input should not become steps. */
|
||||
function nearbyFillableFromSearchChrome(target: Element): FillableElement | null {
|
||||
const container = target.closest(
|
||||
'[id*="chat-input"], [id*="search"], [class*="search"], form, [role="search"]',
|
||||
);
|
||||
if (!container) return null;
|
||||
const fillable = container.querySelector(
|
||||
'textarea, input[type="search"], input[name="q"], #chat-textarea',
|
||||
);
|
||||
if (
|
||||
fillable instanceof HTMLElement &&
|
||||
isTextFillable(fillable) &&
|
||||
fillable !== target &&
|
||||
!fillable.contains(target)
|
||||
) {
|
||||
return fillable;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function fillableValue(el: FillableElement): string {
|
||||
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
|
||||
return el.value;
|
||||
}
|
||||
return el.textContent ?? "";
|
||||
}
|
||||
|
||||
/** Clicks that only pick an autocomplete/suggestion value — not a semantic submit action. */
|
||||
function isInputCompletionClick(target: EventTarget | null, session: FillSession | null): boolean {
|
||||
if (!session || !(target instanceof Element)) return false;
|
||||
if (session.element.contains(target)) return false;
|
||||
|
||||
const option = target.closest('[role="option"]');
|
||||
if (option?.closest('[role="listbox"]')) return true;
|
||||
|
||||
const suggestionRoot = target.closest(
|
||||
[
|
||||
'[id*="Sug"]',
|
||||
'[id*="sug"]',
|
||||
'[class*="suggest"]',
|
||||
'[class*="autocomplete"]',
|
||||
'[class*="typeahead"]',
|
||||
"[data-autocomplete]",
|
||||
].join(", "),
|
||||
);
|
||||
if (suggestionRoot && !suggestionRoot.contains(session.element)) return true;
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
function scheduleInputCompletionCommit(
|
||||
sessionElement: FillableElement,
|
||||
syncFillSessionValue: (el: FillableElement) => void,
|
||||
commitFillSession: () => void,
|
||||
hasSessionFor: (el: FillableElement) => boolean,
|
||||
): void {
|
||||
const syncAndCommit = () => {
|
||||
if (!hasSessionFor(sessionElement)) return;
|
||||
syncFillSessionValue(sessionElement);
|
||||
commitFillSession();
|
||||
};
|
||||
syncFillSessionValue(sessionElement);
|
||||
queueMicrotask(syncAndCommit);
|
||||
setTimeout(syncAndCommit, 0);
|
||||
}
|
||||
|
||||
export function startRecordCapture(
|
||||
_requestId: string,
|
||||
sendStep: (step: RecordStepPayload) => void,
|
||||
): RecordCaptureController {
|
||||
const emitStep = (step: RecordStepPayload) => {
|
||||
sendStep({ page_url: location.href, ...step });
|
||||
};
|
||||
let fillSession: FillSession | null = null;
|
||||
let composing = false;
|
||||
let lastUrl = location.href;
|
||||
let keyboardActivation: { target: EventTarget | null; recordedAt: number } | undefined;
|
||||
let generatedControlClick: Element | null = null;
|
||||
let navigationActionPending = false;
|
||||
let navigationActionVersion = 0;
|
||||
const committedValues = new WeakMap<FillableElement, string>();
|
||||
|
||||
const markNavigationAction = () => {
|
||||
navigationActionPending = true;
|
||||
const version = ++navigationActionVersion;
|
||||
setTimeout(() => {
|
||||
if (navigationActionVersion === version) {
|
||||
navigationActionPending = false;
|
||||
}
|
||||
}, 0);
|
||||
};
|
||||
|
||||
const emitFill = (session: FillSession) => {
|
||||
if (session.lastValue === session.baselineValue) return;
|
||||
const isPassword =
|
||||
session.element instanceof HTMLInputElement && session.element.type === "password";
|
||||
const value = isPassword ? "***" : session.lastValue;
|
||||
emitStep({
|
||||
op: "fill",
|
||||
target: session.target,
|
||||
value,
|
||||
...(isPassword ? { redacted: true } : {}),
|
||||
});
|
||||
};
|
||||
|
||||
const commitFillSession = () => {
|
||||
if (!fillSession || composing) return;
|
||||
const session = fillSession;
|
||||
fillSession = null;
|
||||
emitFill(session);
|
||||
committedValues.set(session.element, session.lastValue);
|
||||
};
|
||||
|
||||
const syncFillSessionValue = (el: FillableElement) => {
|
||||
if (!fillSession || fillSession.element !== el) return;
|
||||
fillSession.lastValue = fillableValue(el);
|
||||
};
|
||||
|
||||
const ensureFillSession = (el: FillableElement) => {
|
||||
if (fillSession?.element === el) return;
|
||||
if (fillSession) commitFillSession();
|
||||
const currentValue = fillableValue(el);
|
||||
const baselineValue = committedValues.get(el) ?? currentValue;
|
||||
fillSession = {
|
||||
element: el,
|
||||
target: describeTarget(el),
|
||||
baselineValue,
|
||||
lastValue: currentValue,
|
||||
};
|
||||
};
|
||||
|
||||
const emitNavigateIfChanged = (causedByAction?: boolean) => {
|
||||
if (location.href === lastUrl) return;
|
||||
commitFillSession();
|
||||
lastUrl = location.href;
|
||||
emitStep({
|
||||
op: "navigate",
|
||||
url: location.href,
|
||||
...(causedByAction !== undefined ? { navigation_caused_by_action: causedByAction } : {}),
|
||||
});
|
||||
};
|
||||
|
||||
const emitClick = (event: MouseEvent) => {
|
||||
// Only record clicks an LLM can re-identify (named interactive controls).
|
||||
const target = describeEventTarget(eventTarget(event));
|
||||
if (!target) return;
|
||||
markNavigationAction();
|
||||
emitStep({
|
||||
op: "click",
|
||||
target,
|
||||
expects_navigation: true,
|
||||
});
|
||||
};
|
||||
|
||||
const onClick = (event: MouseEvent) => {
|
||||
if (event.button !== 0) return;
|
||||
const target = eventTarget(event);
|
||||
if (isOverlayTarget(target)) return;
|
||||
if (event.detail === 0 && generatedControlClick !== null && target === generatedControlClick) {
|
||||
generatedControlClick = null;
|
||||
return;
|
||||
}
|
||||
if (
|
||||
event.detail === 0 &&
|
||||
keyboardActivation?.target === target &&
|
||||
Date.now() - keyboardActivation.recordedAt < 500
|
||||
) {
|
||||
keyboardActivation = undefined;
|
||||
return;
|
||||
}
|
||||
|
||||
const label = target instanceof Element ? target.closest("label") : null;
|
||||
const nestedInteractive =
|
||||
target instanceof Element ? target.closest("a,button,input,select,textarea") : null;
|
||||
if (label instanceof HTMLLabelElement && !nestedInteractive) {
|
||||
commitFillSession();
|
||||
generatedControlClick = label.control;
|
||||
emitClick(event);
|
||||
return;
|
||||
}
|
||||
|
||||
const fillable = fillableFromTarget(target);
|
||||
if (fillable) {
|
||||
ensureFillSession(fillable);
|
||||
return;
|
||||
}
|
||||
|
||||
if (target instanceof Element) {
|
||||
const nearbyFillable = nearbyFillableFromSearchChrome(target);
|
||||
if (nearbyFillable) {
|
||||
ensureFillSession(nearbyFillable);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
if (isInputCompletionClick(target, fillSession)) {
|
||||
markNavigationAction();
|
||||
const sessionElement = fillSession!.element;
|
||||
scheduleInputCompletionCommit(
|
||||
sessionElement,
|
||||
syncFillSessionValue,
|
||||
commitFillSession,
|
||||
(el) => fillSession?.element === el,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
commitFillSession();
|
||||
if (target instanceof Element && target.closest("select")) return;
|
||||
emitClick(event);
|
||||
};
|
||||
|
||||
const onFocusIn = (event: FocusEvent) => {
|
||||
const target = fillableFromTarget(eventTarget(event));
|
||||
if (target) ensureFillSession(target);
|
||||
};
|
||||
|
||||
const onFocusOut = (event: FocusEvent) => {
|
||||
const target = fillableFromTarget(eventTarget(event));
|
||||
if (!target) return;
|
||||
if (!fillSession || fillSession.element !== target) return;
|
||||
syncFillSessionValue(target);
|
||||
commitFillSession();
|
||||
};
|
||||
|
||||
const onInput = (event: Event) => {
|
||||
const target = fillableFromTarget(eventTarget(event));
|
||||
if (!target) return;
|
||||
if (composing) return;
|
||||
ensureFillSession(target);
|
||||
syncFillSessionValue(target);
|
||||
};
|
||||
|
||||
const onCompositionStart = () => {
|
||||
composing = true;
|
||||
};
|
||||
|
||||
const onCompositionEnd = (event: CompositionEvent) => {
|
||||
composing = false;
|
||||
const target = fillableFromTarget(eventTarget(event));
|
||||
if (target) {
|
||||
ensureFillSession(target);
|
||||
syncFillSessionValue(target);
|
||||
}
|
||||
};
|
||||
|
||||
const onChange = (event: Event) => {
|
||||
commitFillSession();
|
||||
const target = eventTarget(event);
|
||||
if (target instanceof HTMLSelectElement) {
|
||||
const values = Array.from(target.selectedOptions).map((opt) => opt.value);
|
||||
const labels = Array.from(target.selectedOptions).map((opt) =>
|
||||
(opt.label || opt.textContent || opt.value).trim(),
|
||||
);
|
||||
const desc = describeTarget(target);
|
||||
markNavigationAction();
|
||||
emitStep({
|
||||
op: "select",
|
||||
target: desc,
|
||||
values,
|
||||
labels,
|
||||
expects_navigation: true,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const onKeyDown = (event: KeyboardEvent) => {
|
||||
if (isOverlayTarget(eventTarget(event))) return;
|
||||
const target = eventTarget(event);
|
||||
const fillable = fillableFromTarget(target);
|
||||
if (fillable) {
|
||||
ensureFillSession(fillable);
|
||||
syncFillSessionValue(fillable);
|
||||
}
|
||||
const modifiers = [
|
||||
...(event.altKey ? (["alt"] as const) : []),
|
||||
...(event.ctrlKey ? (["ctrl"] as const) : []),
|
||||
...(event.metaKey ? (["meta"] as const) : []),
|
||||
...(event.shiftKey ? (["shift"] as const) : []),
|
||||
];
|
||||
if (!shouldRecordPress(event.key, modifiers)) {
|
||||
return;
|
||||
}
|
||||
markNavigationAction();
|
||||
if (event.key === "Enter" || event.key === " ") {
|
||||
const submitTarget =
|
||||
event.key === "Enter" && fillable
|
||||
? fillable
|
||||
.closest("form")
|
||||
?.querySelector(
|
||||
'button:not([type]),button[type="submit"],input[type="submit"],input[type="image"]',
|
||||
)
|
||||
: null;
|
||||
keyboardActivation = {
|
||||
target: submitTarget ?? target,
|
||||
recordedAt: Date.now(),
|
||||
};
|
||||
}
|
||||
if (fillable) {
|
||||
commitFillSession();
|
||||
}
|
||||
const desc = describeEventTarget(target);
|
||||
if (!desc && !event.key) return;
|
||||
emitStep({
|
||||
op: "press",
|
||||
key: event.key,
|
||||
...(desc ? { target: desc } : {}),
|
||||
...(modifiers.length ? { modifiers } : {}),
|
||||
expects_navigation: event.key === "Enter",
|
||||
});
|
||||
};
|
||||
|
||||
document.addEventListener("click", onClick, true);
|
||||
document.addEventListener("focusin", onFocusIn, true);
|
||||
document.addEventListener("focusout", onFocusOut, true);
|
||||
document.addEventListener("input", onInput, true);
|
||||
document.addEventListener("compositionstart", onCompositionStart, true);
|
||||
document.addEventListener("compositionend", onCompositionEnd, true);
|
||||
document.addEventListener("change", onChange, true);
|
||||
document.addEventListener("keydown", onKeyDown, true);
|
||||
|
||||
const urlObserver = new MutationObserver(() => emitNavigateIfChanged());
|
||||
urlObserver.observe(document, { subtree: true, childList: true });
|
||||
const onUrlEvent = () => emitNavigateIfChanged();
|
||||
window.addEventListener("hashchange", onUrlEvent);
|
||||
window.addEventListener("popstate", onUrlEvent);
|
||||
window.addEventListener("pagehide", commitFillSession);
|
||||
|
||||
const originalPushState = history.pushState;
|
||||
const originalReplaceState = history.replaceState;
|
||||
history.pushState = function (...args: Parameters<History["pushState"]>) {
|
||||
originalPushState.apply(this, args);
|
||||
emitNavigateIfChanged(navigationActionPending ? true : undefined);
|
||||
};
|
||||
history.replaceState = function (...args: Parameters<History["replaceState"]>) {
|
||||
originalReplaceState.apply(this, args);
|
||||
emitNavigateIfChanged(navigationActionPending ? true : undefined);
|
||||
};
|
||||
|
||||
return {
|
||||
dispose() {
|
||||
commitFillSession();
|
||||
document.removeEventListener("click", onClick, true);
|
||||
document.removeEventListener("focusin", onFocusIn, true);
|
||||
document.removeEventListener("focusout", onFocusOut, true);
|
||||
document.removeEventListener("input", onInput, true);
|
||||
document.removeEventListener("compositionstart", onCompositionStart, true);
|
||||
document.removeEventListener("compositionend", onCompositionEnd, true);
|
||||
document.removeEventListener("change", onChange, true);
|
||||
document.removeEventListener("keydown", onKeyDown, true);
|
||||
urlObserver.disconnect();
|
||||
window.removeEventListener("hashchange", onUrlEvent);
|
||||
window.removeEventListener("popstate", onUrlEvent);
|
||||
window.removeEventListener("pagehide", commitFillSession);
|
||||
history.pushState = originalPushState;
|
||||
history.replaceState = originalReplaceState;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export type RecordContentMessage = RecordStartMessage | RecordStopMessage | RecordCancelMessage;
|
||||
|
||||
export function isRecordContentMessage(msg: unknown): msg is RecordContentMessage {
|
||||
return isRecordStartMessage(msg) || isRecordStopMessage(msg) || isRecordCancelMessage(msg);
|
||||
}
|
||||
|
||||
export function handleRecordContentMessage(
|
||||
message: RecordContentMessage,
|
||||
state: {
|
||||
activeRequestId: string | null;
|
||||
capture: RecordCaptureController | null;
|
||||
setActiveRequestId(id: string | null): void;
|
||||
setCapture(capture: RecordCaptureController | null): void;
|
||||
onStart(requestId: string): void;
|
||||
onStop(): void;
|
||||
},
|
||||
sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
|
||||
): boolean {
|
||||
if (isRecordStartMessage(message)) {
|
||||
if (!knownRecordRequests.has(message.requestId)) {
|
||||
knownRecordRequests.add(message.requestId);
|
||||
failedStepDeliveries.delete(message.requestId);
|
||||
}
|
||||
state.capture?.dispose();
|
||||
state.setCapture(
|
||||
startRecordCapture(message.requestId, (step) => {
|
||||
sendRecordStep(message.requestId, step);
|
||||
}),
|
||||
);
|
||||
state.setActiveRequestId(message.requestId);
|
||||
state.onStart(message.requestId);
|
||||
sendResponse?.({ ok: true });
|
||||
return sendResponse !== undefined;
|
||||
}
|
||||
|
||||
if (isRecordStopMessage(message) || isRecordCancelMessage(message)) {
|
||||
// Require an active recording that matches this requestId — otherwise a
|
||||
// stray STOP/CANCEL (e.g. after teardown) would still run finishStop and
|
||||
// clear overlay state even though nothing was capturing.
|
||||
if (!state.activeRequestId || state.activeRequestId !== message.requestId) {
|
||||
return false;
|
||||
}
|
||||
state.capture?.dispose();
|
||||
state.setCapture(null);
|
||||
const finishStop = () => {
|
||||
state.onStop();
|
||||
state.setActiveRequestId(null);
|
||||
};
|
||||
if (isRecordStopMessage(message) && sendResponse) {
|
||||
const pending = [...(pendingStepSends.get(message.requestId) ?? [])];
|
||||
void Promise.all(pending).then((delivered) => {
|
||||
finishStop();
|
||||
const succeeded = delivered.every(Boolean) && !failedStepDeliveries.has(message.requestId);
|
||||
if (succeeded) {
|
||||
failedStepDeliveries.delete(message.requestId);
|
||||
knownRecordRequests.delete(message.requestId);
|
||||
}
|
||||
sendResponse(
|
||||
succeeded
|
||||
? { ok: true }
|
||||
: {
|
||||
ok: false,
|
||||
error: "failed to deliver one or more recorded steps",
|
||||
},
|
||||
);
|
||||
});
|
||||
return true;
|
||||
}
|
||||
if (isRecordCancelMessage(message)) {
|
||||
failedStepDeliveries.delete(message.requestId);
|
||||
knownRecordRequests.delete(message.requestId);
|
||||
}
|
||||
finishStop();
|
||||
return false;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
export { RECORD_CANCEL, RECORD_START, RECORD_STEP, RECORD_STOP };
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
} from "@/lib/instance-id";
|
||||
import { startKeepalive } from "@/lib/keepalive";
|
||||
import {
|
||||
OVERLAY_AUTOMATION_BYPASS,
|
||||
OVERLAY_MSG_INTERRUPT,
|
||||
OVERLAY_MSG_WHO_AM_I,
|
||||
type OverlayInterruptRequest,
|
||||
@@ -27,6 +28,11 @@ import {
|
||||
requestBorrowConfirmation,
|
||||
} from "@/tools/borrow-confirmation";
|
||||
import { ToolDispatcher } from "@/tools/dispatcher";
|
||||
import {
|
||||
attachRecordFinishListener,
|
||||
attachRecordQueryListener,
|
||||
attachRecordStepListener,
|
||||
} from "@/tools/record";
|
||||
import { detectBrowserMeta } from "@/transport/handshake";
|
||||
import type { Transport } from "@/transport/transport";
|
||||
import { WSTransport } from "@/transport/ws-transport";
|
||||
@@ -67,6 +73,27 @@ export default defineBackground(() => {
|
||||
}),
|
||||
});
|
||||
dispatcher.start();
|
||||
const recordDeps = {
|
||||
tabsApi: chrome.tabs,
|
||||
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
|
||||
chrome.tabs.sendMessage(tabId, msg),
|
||||
bypassOverlay: async (tabId: number, enabled: boolean) => {
|
||||
try {
|
||||
await chrome.tabs.sendMessage(tabId, {
|
||||
type: OVERLAY_AUTOMATION_BYPASS,
|
||||
enabled,
|
||||
});
|
||||
} catch {
|
||||
// Content script may be unavailable on restricted pages.
|
||||
}
|
||||
},
|
||||
};
|
||||
// Message listeners stay up (cheap; fire only for record message types).
|
||||
// Tab / webNavigation observation attaches lazily while a recording is
|
||||
// active — see ensureBrowserObservationListeners in tools/record.ts.
|
||||
attachRecordStepListener();
|
||||
attachRecordFinishListener(recordDeps);
|
||||
attachRecordQueryListener(recordDeps);
|
||||
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
|
||||
attachBorrowNotificationClickHandler({
|
||||
onClicked: chrome.notifications.onClicked,
|
||||
|
||||
@@ -6,7 +6,13 @@ import { BorrowConfirmationOverlay } from "@/content/BorrowConfirmationOverlay";
|
||||
import { ControlOverlay } from "@/content/ControlOverlay";
|
||||
import { HelpRequestOverlay } from "@/content/HelpRequestOverlay";
|
||||
import overlayCss from "@/content/overlay.css?inline";
|
||||
import { OverlayController } from "@/content/overlay-controller";
|
||||
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
|
||||
import { RecordOverlay } from "@/content/RecordOverlay";
|
||||
import {
|
||||
handleRecordContentMessage,
|
||||
isRecordContentMessage,
|
||||
type RecordCaptureController,
|
||||
} from "@/content/record-capture";
|
||||
import {
|
||||
HELP_RESPONSE,
|
||||
type HelpCancelMessage,
|
||||
@@ -24,6 +30,12 @@ import {
|
||||
type OverlayWhoAmIResponse,
|
||||
} from "@/lib/overlay-bridge";
|
||||
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
|
||||
import {
|
||||
RECORD_FINISH,
|
||||
RECORD_QUERY,
|
||||
type RecordStartAck,
|
||||
type RecordStopAck,
|
||||
} from "@/lib/record-bridge";
|
||||
import { SESSIONS_LIVE_FLAG_KEY } from "@/lib/sessions-live-flag";
|
||||
import type {
|
||||
BorrowCancelMessage,
|
||||
@@ -45,6 +57,8 @@ export default defineContentScript({
|
||||
const overlays = new OverlayController();
|
||||
let activeHelpRespond: ((outcome: "continued" | "cancelled", note?: string) => void) | null =
|
||||
null;
|
||||
let recordCapture: RecordCaptureController | null = null;
|
||||
let activeRecordRequestId: string | null = null;
|
||||
let reactRoot: ReactDOM.Root | null = null;
|
||||
let overlayHost: HTMLElement | null = null;
|
||||
let hostLossReported = false;
|
||||
@@ -87,12 +101,13 @@ export default defineContentScript({
|
||||
requests: overlayState.borrowRequests,
|
||||
}),
|
||||
React.createElement(ControlOverlay, {
|
||||
visible: overlayState.controlVisible && overlayState.activeHelp === null,
|
||||
visible: shouldShowAgentControlOverlay(overlayState),
|
||||
interrupting: overlayState.interrupting,
|
||||
automationBypass: overlayState.automationBypassCount > 0,
|
||||
onInterrupt: handleInterrupt,
|
||||
}),
|
||||
React.createElement(HelpRequestOverlay, { request: overlayState.activeHelp }),
|
||||
React.createElement(RecordOverlay, { request: overlayState.activeRecord }),
|
||||
),
|
||||
),
|
||||
);
|
||||
@@ -104,6 +119,9 @@ export default defineContentScript({
|
||||
activeHelpRespond?.("cancelled");
|
||||
activeHelpRespond = null;
|
||||
}
|
||||
recordCapture?.dispose();
|
||||
recordCapture = null;
|
||||
activeRecordRequestId = null;
|
||||
renderOverlay();
|
||||
}
|
||||
|
||||
@@ -140,6 +158,42 @@ export default defineContentScript({
|
||||
_sender: chrome.runtime.MessageSender,
|
||||
sendResponse: (response: BorrowResponseMessage | HelpResponseMessage) => void,
|
||||
) => {
|
||||
if (isRecordContentMessage(message)) {
|
||||
const needsAsync = handleRecordContentMessage(
|
||||
message,
|
||||
{
|
||||
activeRequestId: activeRecordRequestId,
|
||||
capture: recordCapture,
|
||||
setActiveRequestId: (id) => {
|
||||
activeRecordRequestId = id;
|
||||
},
|
||||
setCapture: (capture) => {
|
||||
recordCapture = capture;
|
||||
},
|
||||
onStart: (requestId) => {
|
||||
overlays.setAgentRecordRequest({
|
||||
id: requestId,
|
||||
onFinish: () => {
|
||||
void chrome.runtime.sendMessage({
|
||||
type: RECORD_FINISH,
|
||||
requestId,
|
||||
});
|
||||
},
|
||||
});
|
||||
renderOverlay();
|
||||
},
|
||||
onStop: () => {
|
||||
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
|
||||
renderOverlay();
|
||||
},
|
||||
},
|
||||
sendResponse as unknown as
|
||||
| ((response: RecordStartAck | RecordStopAck) => void)
|
||||
| undefined,
|
||||
);
|
||||
return needsAsync;
|
||||
}
|
||||
|
||||
if (
|
||||
message &&
|
||||
typeof message === "object" &&
|
||||
@@ -228,23 +282,59 @@ export default defineContentScript({
|
||||
return false;
|
||||
};
|
||||
|
||||
async function mountOverlayIfAgent(): Promise<void> {
|
||||
async function syncAgentOverlay(): Promise<void> {
|
||||
if (!(await anySessionLive())) return;
|
||||
try {
|
||||
const reply = (await chrome.runtime.sendMessage({
|
||||
kind: OVERLAY_MSG_WHO_AM_I,
|
||||
})) as OverlayWhoAmIResponse | undefined;
|
||||
if (!reply?.sessionId) return;
|
||||
|
||||
// Query / re-arm recording *before* activating the control overlay so
|
||||
// record start (navigate → content load) does not flash
|
||||
// 「Agent 正在控制」before RecordOverlay mounts.
|
||||
let recordQuery: { active?: boolean; requestId?: string } | undefined;
|
||||
try {
|
||||
recordQuery = (await chrome.runtime.sendMessage({
|
||||
type: RECORD_QUERY,
|
||||
})) as { active?: boolean; requestId?: string } | undefined;
|
||||
} catch (err) {
|
||||
console.debug("[bsk overlay] record query failed", err);
|
||||
}
|
||||
|
||||
if (
|
||||
recordQuery?.active &&
|
||||
typeof recordQuery.requestId === "string" &&
|
||||
overlays.snapshot().activeRecord === null
|
||||
) {
|
||||
const requestId = recordQuery.requestId;
|
||||
overlays.setAgentRecordRequest({
|
||||
id: requestId,
|
||||
onFinish: () => {
|
||||
void chrome.runtime.sendMessage({
|
||||
type: RECORD_FINISH,
|
||||
requestId,
|
||||
});
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
overlays.activateAgentSession(reply.sessionId);
|
||||
renderOverlay();
|
||||
} catch (err) {
|
||||
console.debug("[bsk overlay] who_am_i failed", err);
|
||||
console.debug("[bsk overlay] syncAgentOverlay failed", err);
|
||||
}
|
||||
}
|
||||
|
||||
const onPageShow = (event: PageTransitionEvent) => {
|
||||
if (event.persisted) void syncAgentOverlay();
|
||||
};
|
||||
|
||||
ui.mount();
|
||||
chrome.runtime.onMessage.addListener(onMessage);
|
||||
void mountOverlayIfAgent();
|
||||
void syncAgentOverlay();
|
||||
|
||||
window.addEventListener("pageshow", onPageShow);
|
||||
|
||||
const hostObserver = new MutationObserver(() => {
|
||||
const connected = overlayHost?.isConnected ?? false;
|
||||
@@ -254,6 +344,7 @@ export default defineContentScript({
|
||||
remountInProgress = true;
|
||||
try {
|
||||
ui.mount();
|
||||
void syncAgentOverlay();
|
||||
} finally {
|
||||
remountInProgress = false;
|
||||
}
|
||||
@@ -268,6 +359,11 @@ export default defineContentScript({
|
||||
ctx.onInvalidated(() => {
|
||||
hostObserver.disconnect();
|
||||
chrome.runtime.onMessage.removeListener(onMessage);
|
||||
window.removeEventListener("pageshow", onPageShow);
|
||||
// Restore history hooks / remove capture listeners before the CS unloads.
|
||||
recordCapture?.dispose();
|
||||
recordCapture = null;
|
||||
activeRecordRequestId = null;
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
describeEventTarget,
|
||||
describeTarget,
|
||||
isMeaningfulClickTarget,
|
||||
} from "../describe-target";
|
||||
|
||||
describe("describeTarget", () => {
|
||||
beforeEach(() => {
|
||||
document.body.innerHTML = "";
|
||||
});
|
||||
|
||||
it("prefers aria-label and button role", () => {
|
||||
document.body.innerHTML = `<button aria-label="发布">go</button>`;
|
||||
const el = document.querySelector("button")!;
|
||||
expect(describeTarget(el)).toEqual({
|
||||
tag: "button",
|
||||
role: "button",
|
||||
name: "发布",
|
||||
});
|
||||
});
|
||||
|
||||
it("uses label text and name_attr for inputs", () => {
|
||||
document.body.innerHTML = `
|
||||
<label for="svc">服务名称</label>
|
||||
<input id="svc" name="serviceName" />
|
||||
`;
|
||||
const el = document.querySelector("input")!;
|
||||
expect(describeTarget(el)).toEqual({
|
||||
tag: "input",
|
||||
role: "textbox",
|
||||
name: "服务名称",
|
||||
name_attr: "serviceName",
|
||||
});
|
||||
});
|
||||
|
||||
it("never invents CSS selectors or @eN", () => {
|
||||
document.body.innerHTML = `<a href="/x">详情</a>`;
|
||||
const desc = describeTarget(document.querySelector("a")!);
|
||||
expect(JSON.stringify(desc)).not.toMatch(/@e\d+|nth-of-type|#/);
|
||||
expect(desc).toEqual({ tag: "a", role: "link", name: "详情" });
|
||||
});
|
||||
|
||||
it("prefers heading text inside SERP-style links", () => {
|
||||
document.body.innerHTML = `
|
||||
<a href="https://www.tencent.com">
|
||||
<h3>Tencent 腾讯</h3>
|
||||
<cite>https://www.tencent.com</cite>
|
||||
</a>
|
||||
`;
|
||||
expect(describeTarget(document.querySelector("a")!).name).toBe("Tencent 腾讯");
|
||||
});
|
||||
});
|
||||
|
||||
describe("LLM textbook click targets", () => {
|
||||
beforeEach(() => {
|
||||
document.body.innerHTML = "";
|
||||
});
|
||||
|
||||
it("rejects tag-only targets the LLM cannot find", () => {
|
||||
expect(isMeaningfulClickTarget({ tag: "div" })).toBe(false);
|
||||
});
|
||||
|
||||
it("treats a short visible name as the teaching signal", () => {
|
||||
expect(isMeaningfulClickTarget({ tag: "div", name: "发布" })).toBe(true);
|
||||
});
|
||||
|
||||
it("ignores anonymous layout chrome and lifts child clicks to the named button", () => {
|
||||
document.body.innerHTML = `
|
||||
<div id="noise">background</div>
|
||||
<button type="button"><span id="icon">发布</span></button>
|
||||
`;
|
||||
expect(describeEventTarget(document.querySelector("#noise"))).toBeNull();
|
||||
expect(describeEventTarget(document.querySelector("#icon"))).toEqual({
|
||||
tag: "button",
|
||||
role: "button",
|
||||
name: "发布",
|
||||
});
|
||||
});
|
||||
|
||||
it("does not attach layout sibling text as nearby_label on buttons", () => {
|
||||
document.body.innerHTML = `
|
||||
<div>background</div>
|
||||
<button type="button" id="pub">发布</button>
|
||||
`;
|
||||
expect(describeTarget(document.querySelector("#pub")!)).toEqual({
|
||||
tag: "button",
|
||||
role: "button",
|
||||
name: "发布",
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps nearby_label for form fields from a preceding label-like sibling", () => {
|
||||
document.body.innerHTML = `
|
||||
<div>服务名称</div>
|
||||
<input id="svc" />
|
||||
`;
|
||||
expect(describeTarget(document.querySelector("#svc")!)).toEqual(
|
||||
expect.objectContaining({
|
||||
tag: "input",
|
||||
nearby_label: "服务名称",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("does not treat plain text blocks as clicks", () => {
|
||||
document.body.innerHTML = `<p id="copy">说明文字</p>`;
|
||||
expect(describeEventTarget(document.querySelector("#copy"))).toBeNull();
|
||||
});
|
||||
|
||||
it("records custom button-like controls by their visible label", () => {
|
||||
document.body.innerHTML = `<div class="btn" id="pub" style="cursor: pointer">发布</div>`;
|
||||
expect(describeEventTarget(document.querySelector("#pub"))).toEqual(
|
||||
expect.objectContaining({ tag: "div", name: "发布" }),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,80 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { appendRecordedPayload, observeRecordedNavigation } from "../recording-step-buffer";
|
||||
|
||||
describe("recording-step-buffer", () => {
|
||||
it("stores semantic click without summary", () => {
|
||||
const buffer = { steps: [], pendingNavigation: false };
|
||||
appendRecordedPayload(buffer, {
|
||||
op: "click",
|
||||
target: { tag: "button", role: "button", name: "发布" },
|
||||
expects_navigation: true,
|
||||
});
|
||||
expect(buffer.steps).toEqual([
|
||||
{
|
||||
op: "click",
|
||||
target: { tag: "button", role: "button", name: "发布" },
|
||||
},
|
||||
]);
|
||||
expect(buffer.pendingNavigation).toBe(true);
|
||||
});
|
||||
|
||||
it("annotates navigated_to on action-caused navigation instead of wait_for_navigation", () => {
|
||||
const buffer = {
|
||||
steps: [
|
||||
{
|
||||
op: "click" as const,
|
||||
target: { tag: "button", role: "button", name: "发布" },
|
||||
},
|
||||
],
|
||||
currentUrl: "https://example.com/a",
|
||||
pendingNavigation: true,
|
||||
pendingNavigationDeadline: Date.now() + 5_000,
|
||||
};
|
||||
observeRecordedNavigation(buffer, "https://example.com/b", true);
|
||||
expect(buffer.steps).toEqual([
|
||||
{
|
||||
op: "click",
|
||||
target: { tag: "button", role: "button", name: "发布" },
|
||||
navigated_to: "https://example.com/b",
|
||||
},
|
||||
]);
|
||||
expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation");
|
||||
});
|
||||
|
||||
it("annotates navigated_to onto a select that auto-submits a navigation", () => {
|
||||
const buffer = {
|
||||
steps: [
|
||||
{
|
||||
op: "select" as const,
|
||||
target: { tag: "select", role: "combobox", name: "分类" },
|
||||
values: ["tech"],
|
||||
},
|
||||
],
|
||||
currentUrl: "https://example.com/list",
|
||||
pendingNavigation: true,
|
||||
pendingNavigationDeadline: Date.now() + 5_000,
|
||||
};
|
||||
observeRecordedNavigation(buffer, "https://example.com/list?cat=tech", true);
|
||||
expect(buffer.steps).toEqual([
|
||||
{
|
||||
op: "select",
|
||||
target: { tag: "select", role: "combobox", name: "分类" },
|
||||
values: ["tech"],
|
||||
navigated_to: "https://example.com/list?cat=tech",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("emits navigate for uncaused URL changes", () => {
|
||||
const buffer = {
|
||||
steps: [],
|
||||
currentUrl: "https://example.com/a",
|
||||
pendingNavigation: false,
|
||||
};
|
||||
observeRecordedNavigation(buffer, "https://example.com/b", false);
|
||||
expect(buffer.steps[0]).toMatchObject({
|
||||
op: "navigate",
|
||||
url: "https://example.com/b",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,141 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { DraftTraceStep } from "@/transport/types";
|
||||
import { reduceTraceSteps, resolveTraceStartUrl, shouldRecordPress } from "../trace-reducer";
|
||||
|
||||
describe("shouldRecordPress", () => {
|
||||
it("keeps Enter and Escape", () => {
|
||||
expect(shouldRecordPress("Enter")).toBe(true);
|
||||
expect(shouldRecordPress("Escape")).toBe(true);
|
||||
});
|
||||
|
||||
it("drops modifiers, clipboard shortcuts, and bare typing", () => {
|
||||
expect(shouldRecordPress("Meta")).toBe(false);
|
||||
expect(shouldRecordPress("c", ["meta"])).toBe(false);
|
||||
expect(shouldRecordPress("a", ["ctrl"])).toBe(false);
|
||||
expect(shouldRecordPress("x")).toBe(false);
|
||||
expect(shouldRecordPress("中")).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("reduceTraceSteps", () => {
|
||||
it("builds steps with pages dictionary and page id references", () => {
|
||||
const drafts: DraftTraceStep[] = [
|
||||
{
|
||||
op: "navigate",
|
||||
url: "https://example.com/search?q=hello&utm_source=x",
|
||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
||||
},
|
||||
{
|
||||
op: "fill",
|
||||
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
||||
value: "browser skill",
|
||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
||||
},
|
||||
{
|
||||
op: "press",
|
||||
key: "Enter",
|
||||
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
||||
navigated_to: "https://example.com/results/42",
|
||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
||||
},
|
||||
{
|
||||
op: "click",
|
||||
target: { tag: "button", role: "button", name: "发布" },
|
||||
navigated_to: "https://example.com/p/99",
|
||||
page_url: "https://example.com/results/42",
|
||||
},
|
||||
{
|
||||
op: "press",
|
||||
key: "a",
|
||||
page_url: "https://example.com/p/99",
|
||||
},
|
||||
];
|
||||
|
||||
const { pages, steps } = reduceTraceSteps(
|
||||
drafts,
|
||||
"https://example.com/search?q=hello&utm_source=x",
|
||||
);
|
||||
expect(JSON.stringify(steps)).not.toContain("parameters");
|
||||
expect(JSON.stringify(steps)).not.toContain("intent");
|
||||
expect(JSON.stringify(steps)).not.toContain("summary");
|
||||
expect(steps.map((s) => s.op)).toEqual(["navigate", "fill", "press", "click"]);
|
||||
expect(pages.map((p) => p.url)).toEqual([
|
||||
"https://example.com/search?q=hello&utm_source=x",
|
||||
"https://example.com/results/42",
|
||||
"https://example.com/p/99",
|
||||
]);
|
||||
|
||||
expect(steps[0]).toMatchObject({
|
||||
id: 1,
|
||||
op: "navigate",
|
||||
page: "p1",
|
||||
to: "https://example.com/search?q=hello&utm_source=x",
|
||||
});
|
||||
|
||||
expect(steps[1]).toMatchObject({
|
||||
op: "fill",
|
||||
page: "p1",
|
||||
value: "browser skill",
|
||||
});
|
||||
|
||||
expect(steps[2]).toMatchObject({
|
||||
op: "press",
|
||||
key: "Enter",
|
||||
page: "p1",
|
||||
effect: { navigated_to: "p2" },
|
||||
});
|
||||
|
||||
expect(steps[3]).toMatchObject({
|
||||
op: "click",
|
||||
page: "p2",
|
||||
effect: { navigated_to: "p3" },
|
||||
});
|
||||
});
|
||||
|
||||
it("collapses consecutive navigations", () => {
|
||||
const { steps } = reduceTraceSteps([
|
||||
{ op: "navigate", url: "https://a.example/redirect1" },
|
||||
{ op: "navigate", url: "https://a.example/final" },
|
||||
]);
|
||||
expect(steps).toHaveLength(1);
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "navigate",
|
||||
to: "https://a.example/final",
|
||||
});
|
||||
});
|
||||
|
||||
it("maps select navigated_to onto effect.navigated_to (page id)", () => {
|
||||
const { pages, steps } = reduceTraceSteps(
|
||||
[
|
||||
{
|
||||
op: "select",
|
||||
target: { tag: "select", role: "combobox", name: "分类" },
|
||||
values: ["tech"],
|
||||
labels: ["技术"],
|
||||
navigated_to: "https://example.com/list?cat=tech",
|
||||
page_url: "https://example.com/list",
|
||||
},
|
||||
],
|
||||
"https://example.com/list",
|
||||
);
|
||||
expect(pages.map((p) => p.url)).toEqual([
|
||||
"https://example.com/list",
|
||||
"https://example.com/list?cat=tech",
|
||||
]);
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "select",
|
||||
page: "p1",
|
||||
selection: [{ value: "tech", label: "技术" }],
|
||||
effect: { navigated_to: "p2" },
|
||||
});
|
||||
});
|
||||
|
||||
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
||||
expect(
|
||||
resolveTraceStartUrl(
|
||||
[{ op: "navigate", url: "https://example.com/other" }],
|
||||
"https://example.com/start",
|
||||
),
|
||||
).toBe("https://example.com/start");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,342 @@
|
||||
/**
|
||||
* Build a semantic TargetDescriptor for an interacted element.
|
||||
*
|
||||
* Trace steps are an LLM *textbook*: each click must say what to look for
|
||||
* on screen (usually a short visible name). Tag-only noise like
|
||||
* `{ "tag": "div" }` / “点击div” is useless and must not be recorded.
|
||||
*/
|
||||
|
||||
export interface TargetDescriptor {
|
||||
role?: string;
|
||||
name?: string;
|
||||
tag: string;
|
||||
name_attr?: string;
|
||||
placeholder?: string;
|
||||
nearby_label?: string;
|
||||
}
|
||||
|
||||
/** Max length for a label that is still a useful “find this on screen” hint. */
|
||||
const ACTIONABLE_LABEL_MAX = 48;
|
||||
|
||||
function normalizeWhitespace(value: string): string {
|
||||
return value.replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function truncate(value: string, max = 80): string {
|
||||
const trimmed = normalizeWhitespace(value);
|
||||
if (trimmed.length <= max) return trimmed;
|
||||
return `${trimmed.slice(0, max - 1)}…`;
|
||||
}
|
||||
|
||||
/** Short, human label an LLM can search for in a snapshot. */
|
||||
export function isActionableLabel(name: string): boolean {
|
||||
const trimmed = normalizeWhitespace(name);
|
||||
if (!trimmed) return false;
|
||||
if (trimmed.length > ACTIONABLE_LABEL_MAX) return false;
|
||||
if (/^https?:\/\//i.test(trimmed)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
function labelledByText(el: Element): string | null {
|
||||
const ids = el.getAttribute("aria-labelledby");
|
||||
if (!ids) return null;
|
||||
const parts = ids
|
||||
.split(/\s+/)
|
||||
.map((id) => document.getElementById(id)?.textContent ?? "")
|
||||
.map(normalizeWhitespace)
|
||||
.filter(Boolean);
|
||||
return parts.length ? truncate(parts.join(" ")) : null;
|
||||
}
|
||||
|
||||
function associatedLabelText(el: Element): string | null {
|
||||
if (!(el instanceof HTMLElement) || !el.id) return null;
|
||||
const label = document.querySelector(`label[for="${CSS.escape(el.id)}"]`);
|
||||
if (!label) return null;
|
||||
const text = normalizeWhitespace(label.textContent ?? "");
|
||||
return text ? truncate(text) : null;
|
||||
}
|
||||
|
||||
function wrappingLabelText(el: Element): string | null {
|
||||
const label = el.closest("label");
|
||||
if (!label) return null;
|
||||
const clone = label.cloneNode(true) as HTMLElement;
|
||||
for (const control of clone.querySelectorAll("input,textarea,select,button")) {
|
||||
control.remove();
|
||||
}
|
||||
const text = normalizeWhitespace(clone.textContent ?? "");
|
||||
return text ? truncate(text) : null;
|
||||
}
|
||||
|
||||
function placeholderOrTitle(el: Element): string | null {
|
||||
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
|
||||
const ph = normalizeWhitespace(el.placeholder);
|
||||
if (ph) return truncate(ph);
|
||||
}
|
||||
const title = normalizeWhitespace(el.getAttribute("title") ?? "");
|
||||
return title ? truncate(title) : null;
|
||||
}
|
||||
|
||||
function ownText(el: Element): string | null {
|
||||
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
|
||||
return null;
|
||||
}
|
||||
const text = normalizeWhitespace(el.textContent ?? "");
|
||||
return text ? truncate(text) : null;
|
||||
}
|
||||
|
||||
/** Prefer a compact title inside links (e.g. SERP result `<h3>`), not the whole blob. */
|
||||
function compactLinkName(el: Element): string | null {
|
||||
const heading = el.querySelector("h1, h2, h3, h4");
|
||||
if (heading) {
|
||||
const text = normalizeWhitespace(heading.textContent ?? "");
|
||||
if (text) return truncate(text);
|
||||
}
|
||||
const img = el.querySelector("img[alt]");
|
||||
if (img instanceof HTMLImageElement) {
|
||||
const alt = normalizeWhitespace(img.alt);
|
||||
if (alt) return truncate(alt);
|
||||
}
|
||||
const clone = el.cloneNode(true) as HTMLElement;
|
||||
for (const junk of clone.querySelectorAll("cite, script, style, svg, noscript")) {
|
||||
junk.remove();
|
||||
}
|
||||
const text = normalizeWhitespace(clone.textContent ?? "");
|
||||
if (!text) return null;
|
||||
const withoutUrl = text.replace(/https?:\/\/\S+/g, "").trim();
|
||||
return truncate(withoutUrl || text);
|
||||
}
|
||||
|
||||
/** Visible accessible name preference order for LLM textbooks. */
|
||||
export function accessibleName(el: Element): string | undefined {
|
||||
const ariaLabel = normalizeWhitespace(el.getAttribute("aria-label") ?? "");
|
||||
if (ariaLabel) return truncate(ariaLabel);
|
||||
|
||||
const labelledBy = labelledByText(el);
|
||||
if (labelledBy) return labelledBy;
|
||||
|
||||
const forLabel = associatedLabelText(el);
|
||||
if (forLabel) return forLabel;
|
||||
|
||||
const wrapLabel = wrappingLabelText(el);
|
||||
if (wrapLabel) return wrapLabel;
|
||||
|
||||
const alt =
|
||||
el instanceof HTMLImageElement
|
||||
? normalizeWhitespace(el.alt)
|
||||
: normalizeWhitespace(el.getAttribute("alt") ?? "");
|
||||
if (alt) return truncate(alt);
|
||||
|
||||
const ph = placeholderOrTitle(el);
|
||||
if (ph) return ph;
|
||||
|
||||
const tag = el.tagName.toLowerCase();
|
||||
if (tag === "a" || el.getAttribute("role") === "link") {
|
||||
const linkName = compactLinkName(el);
|
||||
if (linkName) return linkName;
|
||||
}
|
||||
|
||||
const text = ownText(el);
|
||||
if (text) return text;
|
||||
|
||||
if (el instanceof HTMLInputElement || el instanceof HTMLButtonElement) {
|
||||
const value = normalizeWhitespace(el.value);
|
||||
if (value && (el instanceof HTMLButtonElement || el.type === "submit" || el.type === "button")) {
|
||||
return truncate(value);
|
||||
}
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const INTERACTIVE_SELECTOR = [
|
||||
"a[href]",
|
||||
"button",
|
||||
"summary",
|
||||
"select",
|
||||
'input:not([type="hidden"])',
|
||||
"textarea",
|
||||
'[role="button"]',
|
||||
'[role="link"]',
|
||||
'[role="tab"]',
|
||||
'[role="menuitem"]',
|
||||
'[role="menuitemcheckbox"]',
|
||||
'[role="menuitemradio"]',
|
||||
'[role="option"]',
|
||||
'[role="checkbox"]',
|
||||
'[role="radio"]',
|
||||
'[role="switch"]',
|
||||
'[role="treeitem"]',
|
||||
'[role="combobox"]',
|
||||
'[contenteditable]:not([contenteditable="false"])',
|
||||
].join(",");
|
||||
|
||||
const INTERACTIVE_ROLES = new Set([
|
||||
"button",
|
||||
"link",
|
||||
"tab",
|
||||
"menuitem",
|
||||
"menuitemcheckbox",
|
||||
"menuitemradio",
|
||||
"option",
|
||||
"checkbox",
|
||||
"radio",
|
||||
"switch",
|
||||
"treeitem",
|
||||
"combobox",
|
||||
"textbox",
|
||||
"searchbox",
|
||||
"slider",
|
||||
]);
|
||||
|
||||
function looksClickable(el: Element): boolean {
|
||||
if (!(el instanceof HTMLElement)) return false;
|
||||
if (el.matches(INTERACTIVE_SELECTOR)) return true;
|
||||
const role = normalizeWhitespace(el.getAttribute("role") ?? "").toLowerCase();
|
||||
if (role && INTERACTIVE_ROLES.has(role)) return true;
|
||||
if (el.tabIndex >= 0) return true;
|
||||
try {
|
||||
if (getComputedStyle(el).cursor === "pointer") return true;
|
||||
} catch {
|
||||
// happy-dom / detached nodes
|
||||
}
|
||||
const cls = typeof el.className === "string" ? el.className : "";
|
||||
if (/\b(btn|button|link|clickable|action)\b/i.test(cls)) return true;
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the element a click should describe for the LLM textbook.
|
||||
* Never falls back to an anonymous layout `div`.
|
||||
*/
|
||||
export function resolveClickableElement(target: Element): Element | null {
|
||||
const interactive = target.closest(INTERACTIVE_SELECTOR);
|
||||
if (interactive instanceof Element) return interactive;
|
||||
|
||||
let node: Element | null = target;
|
||||
let depth = 0;
|
||||
while (node && node !== document.body && node !== document.documentElement && depth < 8) {
|
||||
const name = accessibleName(node);
|
||||
if (name && isActionableLabel(name) && looksClickable(node)) {
|
||||
return node;
|
||||
}
|
||||
node = node.parentElement;
|
||||
depth += 1;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Teachable clicks: the LLM must get a label it can later find via snapshot
|
||||
* (visible name), or at least a form `name_attr` for checkbox/radio.
|
||||
* Recording “点击div” with no name fails this bar.
|
||||
*/
|
||||
export function isMeaningfulClickTarget(target: TargetDescriptor): boolean {
|
||||
const name = target.name?.trim();
|
||||
if (name && isActionableLabel(name)) return true;
|
||||
if (
|
||||
(target.role === "checkbox" || target.role === "radio" || target.tag === "input") &&
|
||||
target.name_attr
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export function inferRole(el: Element): string | undefined {
|
||||
const explicit = normalizeWhitespace(el.getAttribute("role") ?? "").toLowerCase();
|
||||
if (explicit) return explicit;
|
||||
|
||||
const tag = el.tagName.toLowerCase();
|
||||
if (tag === "button") return "button";
|
||||
if (tag === "a" && el.hasAttribute("href")) return "link";
|
||||
if (tag === "select") return "combobox";
|
||||
if (tag === "textarea") return "textbox";
|
||||
if (tag === "img") return "img";
|
||||
if (tag === "summary") return "button";
|
||||
if (el instanceof HTMLElement && (el.isContentEditable || el.contentEditable === "true")) {
|
||||
return "textbox";
|
||||
}
|
||||
if (el instanceof HTMLInputElement) {
|
||||
const type = el.type.toLowerCase();
|
||||
if (type === "checkbox") return "checkbox";
|
||||
if (type === "radio") return "radio";
|
||||
if (type === "submit" || type === "button" || type === "image" || type === "reset") {
|
||||
return "button";
|
||||
}
|
||||
if (type === "range") return "slider";
|
||||
return "textbox";
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Nearby labels help the LLM name form fields. Only apply to fillable
|
||||
* controls — never to buttons/links, where a previous layout sibling
|
||||
* (e.g. a chrome `<div>`) would pollute the textbook target.
|
||||
*/
|
||||
function nearbyLabelText(el: Element): string | undefined {
|
||||
if (
|
||||
!(
|
||||
el instanceof HTMLInputElement ||
|
||||
el instanceof HTMLTextAreaElement ||
|
||||
el instanceof HTMLSelectElement
|
||||
)
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
if (el.id) {
|
||||
const byFor = document.querySelector(`label[for="${CSS.escape(el.id)}"]`);
|
||||
const text = byFor ? normalizeWhitespace(byFor.textContent ?? "") : "";
|
||||
if (text) return truncate(text, ACTIONABLE_LABEL_MAX);
|
||||
}
|
||||
const wrapped = el.closest("label");
|
||||
if (wrapped) {
|
||||
const clone = wrapped.cloneNode(true) as HTMLLabelElement;
|
||||
for (const control of clone.querySelectorAll("input, textarea, select, button")) {
|
||||
control.remove();
|
||||
}
|
||||
const text = normalizeWhitespace(clone.textContent ?? "");
|
||||
if (text) return truncate(text, ACTIONABLE_LABEL_MAX);
|
||||
}
|
||||
const prev = el.previousElementSibling;
|
||||
if (prev && /^(LABEL|SPAN|DIV|P|DT)$/i.test(prev.tagName)) {
|
||||
const text = normalizeWhitespace(prev.textContent ?? "");
|
||||
if (text && text.length <= ACTIONABLE_LABEL_MAX) return text;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function describeTarget(el: Element): TargetDescriptor {
|
||||
const tag = el.tagName.toLowerCase();
|
||||
const role = inferRole(el);
|
||||
const name = accessibleName(el);
|
||||
const nameAttr =
|
||||
el instanceof HTMLInputElement ||
|
||||
el instanceof HTMLTextAreaElement ||
|
||||
el instanceof HTMLSelectElement ||
|
||||
el instanceof HTMLButtonElement
|
||||
? normalizeWhitespace(el.name) || undefined
|
||||
: normalizeWhitespace(el.getAttribute("name") ?? "") || undefined;
|
||||
const placeholder =
|
||||
el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement
|
||||
? normalizeWhitespace(el.placeholder) || undefined
|
||||
: normalizeWhitespace(el.getAttribute("placeholder") ?? "") || undefined;
|
||||
const nearby = nearbyLabelText(el);
|
||||
|
||||
return {
|
||||
tag,
|
||||
...(role ? { role } : {}),
|
||||
...(name ? { name } : {}),
|
||||
...(nameAttr ? { name_attr: nameAttr } : {}),
|
||||
...(placeholder ? { placeholder } : {}),
|
||||
...(nearby && nearby !== name ? { nearby_label: nearby } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function describeEventTarget(target: EventTarget | null): TargetDescriptor | null {
|
||||
if (!(target instanceof Element)) return null;
|
||||
const clickable = resolveClickableElement(target);
|
||||
if (!clickable) return null;
|
||||
const desc = describeTarget(clickable);
|
||||
return isMeaningfulClickTarget(desc) ? desc : null;
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
/**
|
||||
* Wire protocol for user-action recording, sent between the background
|
||||
* service worker and a tab's content script.
|
||||
*/
|
||||
|
||||
import type { TargetDescriptor } from "./describe-target";
|
||||
|
||||
export const RECORD_START = "bsk-record-start";
|
||||
export const RECORD_STEP = "bsk-record-step";
|
||||
export const RECORD_STOP = "bsk-record-stop";
|
||||
export const RECORD_CANCEL = "bsk-record-cancel";
|
||||
export const RECORD_FINISH = "bsk-record-finish";
|
||||
export const RECORD_QUERY = "bsk-record-query";
|
||||
|
||||
export interface RecordStartAck {
|
||||
ok: true;
|
||||
}
|
||||
|
||||
export type RecordStopAck = { ok: true } | { ok: false; error: string };
|
||||
|
||||
export interface RecordQueryMessage {
|
||||
type: typeof RECORD_QUERY;
|
||||
}
|
||||
|
||||
export interface RecordQueryResponse {
|
||||
active: boolean;
|
||||
requestId?: string;
|
||||
}
|
||||
|
||||
export interface RecordStartMessage {
|
||||
type: typeof RECORD_START;
|
||||
requestId: string;
|
||||
}
|
||||
|
||||
export interface RecordStepPayload {
|
||||
op: "click" | "fill" | "press" | "select" | "navigate";
|
||||
target?: TargetDescriptor;
|
||||
value?: string;
|
||||
key?: string;
|
||||
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">;
|
||||
values?: string[];
|
||||
labels?: string[];
|
||||
url?: string;
|
||||
redacted?: boolean;
|
||||
/** Page URL when the step was captured. */
|
||||
page_url?: string;
|
||||
/** Capture-only hint; never persisted unless converted to navigated_to. */
|
||||
expects_navigation?: boolean;
|
||||
/** Whether an observed URL change was synchronously caused by the action. */
|
||||
navigation_caused_by_action?: boolean;
|
||||
}
|
||||
|
||||
export interface RecordStepMessage {
|
||||
type: typeof RECORD_STEP;
|
||||
requestId: string;
|
||||
step: RecordStepPayload;
|
||||
}
|
||||
|
||||
export interface RecordStopMessage {
|
||||
type: typeof RECORD_STOP;
|
||||
requestId: string;
|
||||
}
|
||||
|
||||
export interface RecordCancelMessage {
|
||||
type: typeof RECORD_CANCEL;
|
||||
requestId: string;
|
||||
}
|
||||
|
||||
export interface RecordFinishMessage {
|
||||
type: typeof RECORD_FINISH;
|
||||
requestId: string;
|
||||
}
|
||||
|
||||
export function isRecordStartMessage(msg: unknown): msg is RecordStartMessage {
|
||||
if (typeof msg !== "object" || msg === null) return false;
|
||||
const m = msg as Record<string, unknown>;
|
||||
return m.type === RECORD_START && typeof m.requestId === "string";
|
||||
}
|
||||
|
||||
export function isRecordStopMessage(msg: unknown): msg is RecordStopMessage {
|
||||
if (typeof msg !== "object" || msg === null) return false;
|
||||
const m = msg as Record<string, unknown>;
|
||||
return m.type === RECORD_STOP && typeof m.requestId === "string";
|
||||
}
|
||||
|
||||
export function isRecordCancelMessage(msg: unknown): msg is RecordCancelMessage {
|
||||
if (typeof msg !== "object" || msg === null) return false;
|
||||
const m = msg as Record<string, unknown>;
|
||||
return m.type === RECORD_CANCEL && typeof m.requestId === "string";
|
||||
}
|
||||
|
||||
export function isRecordFinishMessage(msg: unknown): msg is RecordFinishMessage {
|
||||
if (typeof msg !== "object" || msg === null) return false;
|
||||
const m = msg as Record<string, unknown>;
|
||||
return m.type === RECORD_FINISH && typeof m.requestId === "string";
|
||||
}
|
||||
|
||||
export function isRecordQueryMessage(msg: unknown): msg is RecordQueryMessage {
|
||||
if (typeof msg !== "object" || msg === null) return false;
|
||||
const m = msg as Record<string, unknown>;
|
||||
return m.type === RECORD_QUERY;
|
||||
}
|
||||
|
||||
export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage {
|
||||
if (typeof msg !== "object" || msg === null) return false;
|
||||
const m = msg as Record<string, unknown>;
|
||||
if (m.type !== RECORD_STEP || typeof m.requestId !== "string") return false;
|
||||
const step = m.step;
|
||||
if (typeof step !== "object" || step === null) return false;
|
||||
return typeof (step as RecordStepPayload).op === "string";
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
import type { DraftTraceStep } from "@/transport/types";
|
||||
import type { RecordStepPayload } from "./record-bridge";
|
||||
|
||||
export interface RecordingStepBuffer {
|
||||
steps: DraftTraceStep[];
|
||||
currentUrl?: string;
|
||||
pendingNavigation: boolean;
|
||||
pendingNavigationDeadline?: number;
|
||||
}
|
||||
|
||||
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
|
||||
|
||||
function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
||||
const pageUrl = payload.page_url;
|
||||
switch (payload.op) {
|
||||
case "click":
|
||||
return payload.target
|
||||
? {
|
||||
op: "click",
|
||||
target: payload.target,
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
}
|
||||
: null;
|
||||
case "fill":
|
||||
return payload.target
|
||||
? {
|
||||
op: "fill",
|
||||
target: payload.target,
|
||||
value: payload.value ?? "",
|
||||
...(payload.redacted ? { redacted: true } : {}),
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
}
|
||||
: null;
|
||||
case "press":
|
||||
return payload.key
|
||||
? {
|
||||
op: "press",
|
||||
key: payload.key,
|
||||
...(payload.target ? { target: payload.target } : {}),
|
||||
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
}
|
||||
: null;
|
||||
case "select":
|
||||
return payload.target && payload.values
|
||||
? {
|
||||
op: "select",
|
||||
target: payload.target,
|
||||
values: payload.values,
|
||||
...(payload.labels?.length ? { labels: payload.labels } : {}),
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
}
|
||||
: null;
|
||||
case "navigate":
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): boolean {
|
||||
for (let i = buffer.steps.length - 1; i >= 0; i -= 1) {
|
||||
const step = buffer.steps[i];
|
||||
if (!step) continue;
|
||||
if (step.op === "click" || step.op === "press" || step.op === "select") {
|
||||
buffer.steps[i] = { ...step, navigated_to: url };
|
||||
return true;
|
||||
}
|
||||
break;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
export function observeRecordedNavigation(
|
||||
buffer: RecordingStepBuffer,
|
||||
url: string,
|
||||
causedByAction?: boolean,
|
||||
): void {
|
||||
if (!url || url === buffer.currentUrl) return;
|
||||
buffer.currentUrl = url;
|
||||
const pendingIsCurrent =
|
||||
buffer.pendingNavigation &&
|
||||
(buffer.pendingNavigationDeadline === undefined ||
|
||||
buffer.pendingNavigationDeadline >= Date.now());
|
||||
|
||||
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
|
||||
buffer.pendingNavigation = false;
|
||||
buffer.pendingNavigationDeadline = undefined;
|
||||
if (!annotateLastStepNavigation(buffer, url)) {
|
||||
buffer.steps.push({
|
||||
op: "navigate",
|
||||
url,
|
||||
page_url: url,
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
buffer.pendingNavigation = false;
|
||||
buffer.pendingNavigationDeadline = undefined;
|
||||
buffer.steps.push({
|
||||
op: "navigate",
|
||||
url,
|
||||
page_url: url,
|
||||
});
|
||||
}
|
||||
|
||||
export function appendRecordedPayload(
|
||||
buffer: RecordingStepBuffer,
|
||||
payload: RecordStepPayload,
|
||||
): void {
|
||||
if (payload.op === "navigate") {
|
||||
if (payload.url) {
|
||||
observeRecordedNavigation(buffer, payload.url, payload.navigation_caused_by_action);
|
||||
}
|
||||
return;
|
||||
}
|
||||
const step = toDraftStep({
|
||||
...payload,
|
||||
page_url: payload.page_url ?? buffer.currentUrl,
|
||||
});
|
||||
if (!step) return;
|
||||
buffer.steps.push(step);
|
||||
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
||||
buffer.pendingNavigation = payload.expects_navigation === true;
|
||||
buffer.pendingNavigationDeadline = buffer.pendingNavigation
|
||||
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
|
||||
: undefined;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
import type { DraftTraceStep, PageRef, SelectedOption, Step } from "@/transport/types";
|
||||
|
||||
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
|
||||
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
|
||||
|
||||
export function shouldRecordPress(
|
||||
key: string,
|
||||
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">,
|
||||
): boolean {
|
||||
if (MODIFIER_ONLY_KEYS.has(key)) return false;
|
||||
const mods = modifiers ?? [];
|
||||
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
|
||||
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
|
||||
if (key === "Enter" || key === "Escape") return true;
|
||||
// Drop bare character typing — FillSession already records the value.
|
||||
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
|
||||
return false;
|
||||
}
|
||||
|
||||
function shouldIncludeDraft(step: DraftTraceStep): boolean {
|
||||
if (step.op === "fill" && !(step.value ?? "").trim() && !step.redacted) return false;
|
||||
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Collapse consecutive navigations to the last hop. */
|
||||
function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
|
||||
const out: DraftTraceStep[] = [];
|
||||
for (const step of steps) {
|
||||
const prev = out[out.length - 1];
|
||||
if (step.op === "navigate" && prev?.op === "navigate") {
|
||||
out[out.length - 1] = step;
|
||||
continue;
|
||||
}
|
||||
out.push(step);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
||||
const urls: string[] = [];
|
||||
if (startUrl) urls.push(startUrl);
|
||||
for (const step of steps) {
|
||||
if (step.op === "navigate") {
|
||||
urls.push(step.url);
|
||||
continue;
|
||||
}
|
||||
if ("page_url" in step && step.page_url) urls.push(step.page_url);
|
||||
if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to);
|
||||
}
|
||||
const seen = new Set<string>();
|
||||
const unique: string[] = [];
|
||||
for (const url of urls) {
|
||||
if (!url || seen.has(url)) continue;
|
||||
seen.add(url);
|
||||
unique.push(url);
|
||||
}
|
||||
return unique;
|
||||
}
|
||||
|
||||
function buildPageRegistry(
|
||||
steps: DraftTraceStep[],
|
||||
startUrl?: string,
|
||||
): { pages: PageRef[]; urlToId: Map<string, string> } {
|
||||
const urls = collectUrls(steps, startUrl);
|
||||
const urlToId = new Map<string, string>();
|
||||
const pages = urls.map((url, index) => {
|
||||
const id = `p${index + 1}`;
|
||||
urlToId.set(url, id);
|
||||
return { id, url };
|
||||
});
|
||||
return { pages, urlToId };
|
||||
}
|
||||
|
||||
function pageIdFor(
|
||||
url: string | undefined,
|
||||
urlToId: Map<string, string>,
|
||||
fallbackUrl?: string,
|
||||
): string {
|
||||
if (url && urlToId.has(url)) return urlToId.get(url)!;
|
||||
if (fallbackUrl && urlToId.has(fallbackUrl)) return urlToId.get(fallbackUrl)!;
|
||||
return urlToId.values().next().value ?? "p1";
|
||||
}
|
||||
|
||||
function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined {
|
||||
if (step.op === "navigate") return step.page_url ?? step.url;
|
||||
if ("page_url" in step && step.page_url) return step.page_url;
|
||||
return fallbackUrl;
|
||||
}
|
||||
|
||||
function effectForNavigation(
|
||||
navigatedTo: string | undefined,
|
||||
urlToId: Map<string, string>,
|
||||
): Step["effect"] {
|
||||
if (!navigatedTo) return undefined;
|
||||
const pageId = urlToId.get(navigatedTo);
|
||||
if (!pageId) return undefined;
|
||||
return { navigated_to: pageId };
|
||||
}
|
||||
|
||||
function withEffect(step: Step, effect: Step["effect"]): Step {
|
||||
if (!effect) return step;
|
||||
return { ...step, effect };
|
||||
}
|
||||
|
||||
function toSelection(values: string[], labels?: string[]): SelectedOption[] {
|
||||
return values.map((value, index) => ({
|
||||
value,
|
||||
...(labels?.[index] ? { label: labels[index] } : {}),
|
||||
}));
|
||||
}
|
||||
|
||||
function toV2Step(
|
||||
step: DraftTraceStep,
|
||||
id: number,
|
||||
urlToId: Map<string, string>,
|
||||
fallbackUrl?: string,
|
||||
): Step | null {
|
||||
if (!shouldIncludeDraft(step)) return null;
|
||||
|
||||
const pageUrl = pageUrlForDraft(step, fallbackUrl);
|
||||
const page = pageIdFor(pageUrl, urlToId, fallbackUrl);
|
||||
|
||||
switch (step.op) {
|
||||
case "navigate":
|
||||
return {
|
||||
op: "navigate",
|
||||
id,
|
||||
page: pageIdFor(step.url, urlToId, fallbackUrl),
|
||||
to: step.url,
|
||||
};
|
||||
case "click":
|
||||
return withEffect(
|
||||
{
|
||||
op: "click",
|
||||
id,
|
||||
page,
|
||||
target: step.target,
|
||||
},
|
||||
effectForNavigation(step.navigated_to, urlToId),
|
||||
);
|
||||
case "fill":
|
||||
return {
|
||||
op: "fill",
|
||||
id,
|
||||
page,
|
||||
target: step.target,
|
||||
value: step.value,
|
||||
...(step.redacted ? { redacted: true } : {}),
|
||||
};
|
||||
case "press":
|
||||
return withEffect(
|
||||
{
|
||||
op: "press",
|
||||
id,
|
||||
page,
|
||||
key: step.key,
|
||||
...(step.target ? { target: step.target } : {}),
|
||||
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
|
||||
},
|
||||
effectForNavigation(step.navigated_to, urlToId),
|
||||
);
|
||||
case "select":
|
||||
return withEffect(
|
||||
{
|
||||
op: "select",
|
||||
id,
|
||||
page,
|
||||
target: step.target,
|
||||
selection: toSelection(step.values, step.labels),
|
||||
},
|
||||
effectForNavigation(step.navigated_to, urlToId),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export interface ReducedTrace {
|
||||
pages: PageRef[];
|
||||
steps: Step[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Compile capture drafts into record-only trace v2 steps.
|
||||
* Variable inputs are NOT classified here — executing agents infer that at run time.
|
||||
*/
|
||||
export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): ReducedTrace {
|
||||
const collapsed = collapseNavigations(steps);
|
||||
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
|
||||
const out: Step[] = [];
|
||||
let id = 1;
|
||||
let lastUrl = startUrl;
|
||||
for (const draft of collapsed) {
|
||||
if (draft.op === "navigate") lastUrl = draft.url;
|
||||
else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to;
|
||||
else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url;
|
||||
const step = toV2Step(draft, id, urlToId, lastUrl);
|
||||
if (!step) continue;
|
||||
out.push(step);
|
||||
id += 1;
|
||||
}
|
||||
return { pages, steps: out };
|
||||
}
|
||||
|
||||
export function resolveTraceStartUrl(
|
||||
drafts: DraftTraceStep[],
|
||||
startUrl?: string,
|
||||
pages?: PageRef[],
|
||||
): string {
|
||||
if (startUrl) return startUrl;
|
||||
const navigate = drafts.find((step): step is Extract<DraftTraceStep, { op: "navigate" }> => {
|
||||
return step.op === "navigate";
|
||||
});
|
||||
if (navigate) return navigate.url;
|
||||
for (const draft of drafts) {
|
||||
if ("page_url" in draft && draft.page_url) return draft.page_url;
|
||||
}
|
||||
return pages?.[0]?.url ?? "about:blank";
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
ensureBrowserObservationListeners,
|
||||
isBrowserObservationAttachedForTests,
|
||||
releaseBrowserObservationListenersIfIdle,
|
||||
resetBrowserObservationForTests,
|
||||
setBrowserObservationAttachForTests,
|
||||
} from "../record";
|
||||
|
||||
describe("browser observation lifecycle", () => {
|
||||
afterEach(() => {
|
||||
resetBrowserObservationForTests();
|
||||
});
|
||||
|
||||
it("does not attach listeners until ensure is called", () => {
|
||||
const tab = vi.fn(() => () => undefined);
|
||||
const nav = vi.fn(() => () => undefined);
|
||||
setBrowserObservationAttachForTests(tab, nav);
|
||||
|
||||
expect(isBrowserObservationAttachedForTests()).toBe(false);
|
||||
expect(tab).not.toHaveBeenCalled();
|
||||
expect(nav).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("attaches once on ensure and detaches when idle", () => {
|
||||
const detachTab = vi.fn();
|
||||
const detachNav = vi.fn();
|
||||
const tab = vi.fn(() => detachTab);
|
||||
const nav = vi.fn(() => detachNav);
|
||||
setBrowserObservationAttachForTests(tab, nav);
|
||||
|
||||
ensureBrowserObservationListeners({
|
||||
tabsApi: { get: vi.fn(), query: vi.fn(), create: vi.fn() } as never,
|
||||
sendToTab: vi.fn(),
|
||||
});
|
||||
expect(isBrowserObservationAttachedForTests()).toBe(true);
|
||||
expect(tab).toHaveBeenCalledTimes(1);
|
||||
expect(nav).toHaveBeenCalledTimes(1);
|
||||
|
||||
// Idempotent.
|
||||
ensureBrowserObservationListeners({
|
||||
tabsApi: { get: vi.fn(), query: vi.fn(), create: vi.fn() } as never,
|
||||
sendToTab: vi.fn(),
|
||||
});
|
||||
expect(tab).toHaveBeenCalledTimes(1);
|
||||
|
||||
releaseBrowserObservationListenersIfIdle();
|
||||
expect(isBrowserObservationAttachedForTests()).toBe(false);
|
||||
expect(detachTab).toHaveBeenCalledTimes(1);
|
||||
expect(detachNav).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
@@ -13,6 +13,9 @@ import type {
|
||||
NetworkParams,
|
||||
PressParams,
|
||||
ProtocolFrame,
|
||||
RecordAwaitParams,
|
||||
RecordStartParams,
|
||||
RecordStopParams,
|
||||
ReloadParams,
|
||||
RequestFrame,
|
||||
RequestHelpParams,
|
||||
@@ -42,6 +45,7 @@ import {
|
||||
handleScreenshot,
|
||||
handleSnapshot,
|
||||
} from "./observation";
|
||||
import { handleRecordAwait, handleRecordStart, handleRecordStop } from "./record";
|
||||
import {
|
||||
handleSessionStart,
|
||||
handleSessionStop,
|
||||
@@ -396,6 +400,45 @@ export class ToolDispatcher {
|
||||
notificationCopy: this.helpNotificationCopy?.(),
|
||||
signal,
|
||||
});
|
||||
case "tool.record_start":
|
||||
return handleRecordStart(this.sessions, req.params as RecordStartParams, {
|
||||
tabsApi: chromeTabsApi,
|
||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||
bypassOverlay: async (tabId, enabled) => {
|
||||
try {
|
||||
await chrome.tabs.sendMessage(tabId, {
|
||||
type: OVERLAY_AUTOMATION_BYPASS,
|
||||
enabled,
|
||||
});
|
||||
} catch {
|
||||
// Content script may be unavailable on restricted pages.
|
||||
}
|
||||
},
|
||||
...(this.cdp ? { cdp: this.cdp } : {}),
|
||||
signal,
|
||||
});
|
||||
case "tool.record_stop":
|
||||
return handleRecordStop(this.sessions, req.params as RecordStopParams, {
|
||||
tabsApi: chromeTabsApi,
|
||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||
bypassOverlay: async (tabId, enabled) => {
|
||||
try {
|
||||
await chrome.tabs.sendMessage(tabId, {
|
||||
type: OVERLAY_AUTOMATION_BYPASS,
|
||||
enabled,
|
||||
});
|
||||
} catch {
|
||||
// Content script may be unavailable on restricted pages.
|
||||
}
|
||||
},
|
||||
signal,
|
||||
});
|
||||
case "tool.record_await":
|
||||
return handleRecordAwait(this.sessions, req.params as RecordAwaitParams, {
|
||||
tabsApi: chromeTabsApi,
|
||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||
signal,
|
||||
});
|
||||
default:
|
||||
return {
|
||||
code: "unknown_method",
|
||||
|
||||
@@ -0,0 +1,825 @@
|
||||
// `tool.record_start` / `tool.record_stop` / `tool.record_await` — capture
|
||||
// user actions in the Agent Window via the content script and return a
|
||||
// semantic (LLM textbook) trace.
|
||||
|
||||
import {
|
||||
isRecordFinishMessage,
|
||||
isRecordQueryMessage,
|
||||
isRecordStepMessage,
|
||||
RECORD_CANCEL,
|
||||
RECORD_START,
|
||||
RECORD_STEP,
|
||||
RECORD_STOP,
|
||||
type RecordCancelMessage,
|
||||
type RecordFinishMessage,
|
||||
type RecordQueryResponse,
|
||||
type RecordStartAck,
|
||||
type RecordStartMessage,
|
||||
type RecordStopMessage,
|
||||
} from "@/lib/record-bridge";
|
||||
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording-step-buffer";
|
||||
import { reduceTraceSteps, resolveTraceStartUrl } from "@/lib/trace-reducer";
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import type {
|
||||
DraftTraceStep,
|
||||
RecordAwaitParams,
|
||||
RecordAwaitResult,
|
||||
RecordStartParams,
|
||||
RecordStartResult,
|
||||
RecordStopParams,
|
||||
RecordStopResult,
|
||||
RpcError,
|
||||
Trace,
|
||||
} from "@/transport/types";
|
||||
import { handleNavigate } from "./navigation";
|
||||
import {
|
||||
type CdpRunner,
|
||||
type ChromeTabsApi,
|
||||
chromeTabsApi,
|
||||
isRpcError,
|
||||
lookupSession,
|
||||
resolveTargetTab,
|
||||
} from "./shared";
|
||||
|
||||
interface ActiveRecording {
|
||||
requestId: string;
|
||||
tabId: number;
|
||||
agentWindowId: number;
|
||||
startUrl?: string;
|
||||
purpose?: string;
|
||||
steps: DraftTraceStep[];
|
||||
startedAt: string;
|
||||
finishPromise: Promise<Trace>;
|
||||
resolveFinish: (trace: Trace) => void;
|
||||
rejectFinish: (err: Error) => void;
|
||||
settled: boolean;
|
||||
finishing: boolean;
|
||||
currentUrl?: string;
|
||||
pendingNavigation: boolean;
|
||||
pendingNavigationDeadline?: number;
|
||||
}
|
||||
|
||||
const recordings = new Map<string, ActiveRecording>();
|
||||
|
||||
const RECORD_START_RETRIES = 3;
|
||||
const RECORD_START_RETRY_DELAY_MS = 500;
|
||||
const RECORD_REARM_DEBOUNCE_MS = 150;
|
||||
const RECORD_REARM_MAX_ATTEMPTS = 12;
|
||||
const RECORD_REARM_RETRY_DELAY_MS = 400;
|
||||
|
||||
const rearmTimers = new Map<number, ReturnType<typeof setTimeout>>();
|
||||
|
||||
function makeRequestId(tabId: number): string {
|
||||
return `rec-${tabId}-${Date.now().toString(36)}`;
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/** Pages where MV3 content scripts cannot attach (Agent Window boots here). */
|
||||
function isContentScriptRestrictedUrl(url: string | undefined): boolean {
|
||||
if (!url) return true;
|
||||
const lower = url.toLowerCase();
|
||||
return (
|
||||
lower === "about:blank" ||
|
||||
lower.startsWith("about:") ||
|
||||
lower.startsWith("chrome://") ||
|
||||
lower.startsWith("chrome-extension://") ||
|
||||
lower.startsWith("edge://") ||
|
||||
lower.startsWith("devtools://") ||
|
||||
lower.startsWith("devtools:") ||
|
||||
lower.startsWith("https://chrome.google.com/webstore")
|
||||
);
|
||||
}
|
||||
|
||||
async function waitForTabReady(
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
timeoutMs = 10_000,
|
||||
): Promise<void> {
|
||||
try {
|
||||
const tab = await tabsApi.get(tabId);
|
||||
if (tab.status === "complete") return;
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
chrome.tabs.onUpdated.removeListener(listener);
|
||||
reject(new Error("tab load timeout"));
|
||||
}, timeoutMs);
|
||||
const listener = (updatedTabId: number, info: chrome.tabs.TabChangeInfo) => {
|
||||
if (updatedTabId !== tabId || info.status !== "complete") return;
|
||||
clearTimeout(timer);
|
||||
chrome.tabs.onUpdated.removeListener(listener);
|
||||
resolve();
|
||||
};
|
||||
chrome.tabs.onUpdated.addListener(listener);
|
||||
});
|
||||
}
|
||||
|
||||
function isRecordStartAck(response: unknown): response is RecordStartAck {
|
||||
return (
|
||||
typeof response === "object" &&
|
||||
response !== null &&
|
||||
"ok" in response &&
|
||||
(response as RecordStartAck).ok === true
|
||||
);
|
||||
}
|
||||
|
||||
async function sendRecordStartWithAck(
|
||||
tabId: number,
|
||||
msg: RecordStartMessage,
|
||||
sendToTab: RecordDeps["sendToTab"],
|
||||
): Promise<void> {
|
||||
let lastError: unknown;
|
||||
for (let attempt = 0; attempt < RECORD_START_RETRIES; attempt += 1) {
|
||||
try {
|
||||
const response = await sendToTab(tabId, msg);
|
||||
if (isRecordStartAck(response)) return;
|
||||
lastError = new Error("content script did not ack RECORD_START");
|
||||
} catch (err) {
|
||||
lastError = err;
|
||||
}
|
||||
if (attempt + 1 < RECORD_START_RETRIES) {
|
||||
await sleep(RECORD_START_RETRY_DELAY_MS);
|
||||
}
|
||||
}
|
||||
throw lastError ?? new Error("failed to start recording in content script");
|
||||
}
|
||||
|
||||
function buildTrace(recording: ActiveRecording): Trace {
|
||||
const { pages, steps } = reduceTraceSteps(recording.steps, recording.startUrl);
|
||||
const startUrl = resolveTraceStartUrl(recording.steps, recording.startUrl, pages);
|
||||
return {
|
||||
...(recording.purpose ? { purpose: recording.purpose } : {}),
|
||||
recorded_at: new Date().toISOString(),
|
||||
started_at: recording.startedAt,
|
||||
entry: { start_url: startUrl },
|
||||
pages,
|
||||
steps,
|
||||
};
|
||||
}
|
||||
|
||||
export interface RecordDeps {
|
||||
tabsApi: ChromeTabsApi;
|
||||
sendToTab(
|
||||
tabId: number,
|
||||
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
|
||||
): Promise<unknown>;
|
||||
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
cdp?: CdpRunner;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
let defaultDeps: RecordDeps | null = null;
|
||||
function getDefaultDeps(): RecordDeps {
|
||||
if (!defaultDeps) {
|
||||
defaultDeps = {
|
||||
tabsApi: chromeTabsApi,
|
||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||
};
|
||||
}
|
||||
return defaultDeps;
|
||||
}
|
||||
|
||||
/** Disposer for lazily attached tab / webNavigation observers. */
|
||||
let detachBrowserObservation: (() => void) | null = null;
|
||||
|
||||
type AttachObservation = (deps: RecordDeps) => () => void;
|
||||
|
||||
// Deferred wrappers so we do not capture attach* before their declarations.
|
||||
let attachTabObservation: AttachObservation = (deps) => attachRecordTabListener(deps);
|
||||
let attachNavObservation: AttachObservation = (deps) => attachRecordNavigationListener(deps);
|
||||
|
||||
/** Test seam: swap real chrome listeners for fakes. */
|
||||
export function setBrowserObservationAttachForTests(
|
||||
tab: AttachObservation | null,
|
||||
nav: AttachObservation | null,
|
||||
): void {
|
||||
attachTabObservation = tab ?? ((deps) => attachRecordTabListener(deps));
|
||||
attachNavObservation = nav ?? ((deps) => attachRecordNavigationListener(deps));
|
||||
}
|
||||
|
||||
export function isBrowserObservationAttachedForTests(): boolean {
|
||||
return detachBrowserObservation !== null;
|
||||
}
|
||||
|
||||
export function resetBrowserObservationForTests(): void {
|
||||
detachBrowserObservation?.();
|
||||
detachBrowserObservation = null;
|
||||
recordings.clear();
|
||||
attachTabObservation = (deps) => attachRecordTabListener(deps);
|
||||
attachNavObservation = (deps) => attachRecordNavigationListener(deps);
|
||||
}
|
||||
|
||||
/**
|
||||
* Attach tab/webNavigation listeners while any recording is active.
|
||||
* Must run before navigate-on-start so rearm observes the destination load.
|
||||
*/
|
||||
export function ensureBrowserObservationListeners(deps: RecordDeps = getDefaultDeps()): void {
|
||||
if (detachBrowserObservation) return;
|
||||
const detachTab = attachTabObservation(deps);
|
||||
const detachNav = attachNavObservation(deps);
|
||||
detachBrowserObservation = () => {
|
||||
detachTab();
|
||||
detachNav();
|
||||
};
|
||||
}
|
||||
|
||||
/** Detach when the recordings map is empty. */
|
||||
export function releaseBrowserObservationListenersIfIdle(): void {
|
||||
if (recordings.size > 0) return;
|
||||
if (!detachBrowserObservation) return;
|
||||
detachBrowserObservation();
|
||||
detachBrowserObservation = null;
|
||||
}
|
||||
|
||||
export function attachRecordStepListener(): () => void {
|
||||
const listener = (
|
||||
message: unknown,
|
||||
_sender: chrome.runtime.MessageSender,
|
||||
_sendResponse: () => void,
|
||||
) => {
|
||||
if (!isRecordStepMessage(message)) return false;
|
||||
for (const recording of recordings.values()) {
|
||||
if (recording.requestId !== message.requestId) continue;
|
||||
appendRecordedPayload(recording, message.step);
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
chrome.runtime.onMessage.addListener(listener);
|
||||
return () => chrome.runtime.onMessage.removeListener(listener);
|
||||
}
|
||||
|
||||
export function attachRecordFinishListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||
const listener = (
|
||||
message: unknown,
|
||||
sender: chrome.runtime.MessageSender,
|
||||
_sendResponse: () => void,
|
||||
) => {
|
||||
if (!isRecordFinishMessage(message)) return false;
|
||||
const tabId = sender.tab?.id;
|
||||
if (tabId === undefined) return false;
|
||||
void finishRecordingByRequest(message.requestId, tabId, deps);
|
||||
return false;
|
||||
};
|
||||
chrome.runtime.onMessage.addListener(listener);
|
||||
return () => chrome.runtime.onMessage.removeListener(listener);
|
||||
}
|
||||
|
||||
function findRecordingByTabId(tabId: number): ActiveRecording | null {
|
||||
for (const recording of recordings.values()) {
|
||||
if (recording.tabId === tabId && !recording.settled) return recording;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function findRecordingForTab(
|
||||
tabId: number,
|
||||
deps: RecordDeps,
|
||||
): Promise<ActiveRecording | null> {
|
||||
const direct = findRecordingByTabId(tabId);
|
||||
if (direct) return direct;
|
||||
|
||||
try {
|
||||
const tab = await deps.tabsApi.get(tabId);
|
||||
const windowId = tab.windowId;
|
||||
if (typeof windowId !== "number") return null;
|
||||
for (const recording of recordings.values()) {
|
||||
if (!recording.settled && recording.agentWindowId === windowId) return recording;
|
||||
}
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function clearRearmTimer(tabId: number): void {
|
||||
const timer = rearmTimers.get(tabId);
|
||||
if (timer) {
|
||||
clearTimeout(timer);
|
||||
rearmTimers.delete(tabId);
|
||||
}
|
||||
}
|
||||
|
||||
async function clearRearmTimersForRecording(
|
||||
recording: ActiveRecording,
|
||||
deps: RecordDeps,
|
||||
): Promise<void> {
|
||||
clearRearmTimer(recording.tabId);
|
||||
try {
|
||||
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
|
||||
for (const tab of tabs) {
|
||||
if (typeof tab.id === "number") clearRearmTimer(tab.id);
|
||||
}
|
||||
} catch {
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
}
|
||||
|
||||
async function stopRecordingOnAllAgentTabs(
|
||||
recording: ActiveRecording,
|
||||
deps: RecordDeps,
|
||||
): Promise<void> {
|
||||
const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
|
||||
let tabIds = [recording.tabId];
|
||||
try {
|
||||
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
|
||||
tabIds = [
|
||||
...new Set([
|
||||
recording.tabId,
|
||||
...tabs.flatMap((tab) => (typeof tab.id === "number" ? [tab.id] : [])),
|
||||
]),
|
||||
];
|
||||
} catch {
|
||||
// Fall back to the current recording tab.
|
||||
}
|
||||
|
||||
for (const tabId of tabIds) {
|
||||
try {
|
||||
const response = await deps.sendToTab(tabId, stopMsg);
|
||||
if (tabId === recording.tabId && !isRecordStartAck(response)) {
|
||||
throw new Error("content script did not confirm recorded steps");
|
||||
}
|
||||
} catch {
|
||||
if (tabId === recording.tabId) {
|
||||
throw new Error("failed to flush recorded steps");
|
||||
}
|
||||
}
|
||||
if (deps.bypassOverlay) {
|
||||
try {
|
||||
await deps.bypassOverlay(tabId, false);
|
||||
} catch {
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function rearmRecording(
|
||||
recording: ActiveRecording,
|
||||
targetTabId: number,
|
||||
deps: RecordDeps,
|
||||
): Promise<boolean> {
|
||||
// Do NOT toggle automation-bypass here: each retry used to increment the
|
||||
// content-script counter, and a single stop decrement left the ControlOverlay
|
||||
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
|
||||
// already hides the control chrome while activeRecord is set.
|
||||
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
|
||||
const startMsg: RecordStartMessage = { type: RECORD_START, requestId: recording.requestId };
|
||||
try {
|
||||
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
|
||||
recording.tabId = targetTabId;
|
||||
return true;
|
||||
} catch {
|
||||
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
|
||||
await sleep(RECORD_REARM_RETRY_DELAY_MS);
|
||||
}
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
|
||||
const existing = rearmTimers.get(tabId);
|
||||
if (existing) clearTimeout(existing);
|
||||
rearmTimers.set(
|
||||
tabId,
|
||||
setTimeout(() => {
|
||||
rearmTimers.delete(tabId);
|
||||
void (async () => {
|
||||
const current = await findRecordingForTab(tabId, deps);
|
||||
if (current) await rearmRecording(current, tabId, deps);
|
||||
})();
|
||||
}, RECORD_REARM_DEBOUNCE_MS),
|
||||
);
|
||||
}
|
||||
|
||||
export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||
const onCreated = (tab: chrome.tabs.Tab) => {
|
||||
const tabId = tab.id;
|
||||
const windowId = tab.windowId;
|
||||
if (tabId === undefined || windowId === undefined) return;
|
||||
for (const recording of recordings.values()) {
|
||||
if (recording.settled || recording.agentWindowId !== windowId) continue;
|
||||
scheduleRearmForTab(tabId, deps);
|
||||
return;
|
||||
}
|
||||
};
|
||||
|
||||
const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => {
|
||||
scheduleRearmForTab(activeInfo.tabId, deps);
|
||||
};
|
||||
|
||||
chrome.tabs.onCreated.addListener(onCreated);
|
||||
chrome.tabs.onActivated.addListener(onActivated);
|
||||
return () => {
|
||||
chrome.tabs.onCreated.removeListener(onCreated);
|
||||
chrome.tabs.onActivated.removeListener(onActivated);
|
||||
};
|
||||
}
|
||||
|
||||
export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||
const observeMainFrame = (tabId: number, url?: string, causedByAction?: boolean) => {
|
||||
void (async () => {
|
||||
const recording = await findRecordingForTab(tabId, deps);
|
||||
if (recording && url) {
|
||||
observeRecordedNavigation(recording, url, causedByAction);
|
||||
}
|
||||
})();
|
||||
};
|
||||
const onMainFrameComplete = (tabId: number, url?: string) => {
|
||||
void (async () => {
|
||||
observeMainFrame(tabId, url);
|
||||
scheduleRearmForTab(tabId, deps);
|
||||
})();
|
||||
};
|
||||
|
||||
if (chrome.webNavigation?.onCompleted) {
|
||||
const completedListener = (
|
||||
details: chrome.webNavigation.WebNavigationFramedCallbackDetails,
|
||||
) => {
|
||||
if (details.frameId !== 0) return;
|
||||
onMainFrameComplete(details.tabId, details.url);
|
||||
};
|
||||
const committedListener = (
|
||||
details: chrome.webNavigation.WebNavigationTransitionCallbackDetails,
|
||||
) => {
|
||||
if (details.frameId !== 0) return;
|
||||
const causedByAction =
|
||||
details.transitionType === "link" || details.transitionType === "form_submit"
|
||||
? true
|
||||
: details.transitionType === "typed" ||
|
||||
details.transitionType === "auto_bookmark" ||
|
||||
details.transitionType === "generated" ||
|
||||
details.transitionType === "keyword" ||
|
||||
details.transitionType === "keyword_generated" ||
|
||||
details.transitionType === "reload"
|
||||
? false
|
||||
: undefined;
|
||||
observeMainFrame(details.tabId, details.url, causedByAction);
|
||||
};
|
||||
chrome.webNavigation.onCompleted.addListener(completedListener);
|
||||
chrome.webNavigation.onCommitted?.addListener(committedListener);
|
||||
return () => {
|
||||
chrome.webNavigation.onCompleted.removeListener(completedListener);
|
||||
chrome.webNavigation.onCommitted?.removeListener(committedListener);
|
||||
};
|
||||
}
|
||||
|
||||
const listener = (tabId: number, info: chrome.tabs.TabChangeInfo) => {
|
||||
if (info.status !== "complete") return;
|
||||
onMainFrameComplete(tabId, info.url);
|
||||
};
|
||||
chrome.tabs.onUpdated.addListener(listener);
|
||||
return () => chrome.tabs.onUpdated.removeListener(listener);
|
||||
}
|
||||
|
||||
export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||
const listener = (
|
||||
message: unknown,
|
||||
sender: chrome.runtime.MessageSender,
|
||||
sendResponse: (response: RecordQueryResponse) => void,
|
||||
) => {
|
||||
if (!isRecordQueryMessage(message)) return false;
|
||||
const tabId = sender.tab?.id;
|
||||
if (tabId === undefined) {
|
||||
sendResponse({ active: false });
|
||||
return false;
|
||||
}
|
||||
void (async () => {
|
||||
const recording = await findRecordingForTab(tabId, deps);
|
||||
if (!recording) {
|
||||
sendResponse({ active: false });
|
||||
return;
|
||||
}
|
||||
await rearmRecording(recording, tabId, deps);
|
||||
sendResponse({ active: true, requestId: recording.requestId });
|
||||
})();
|
||||
return true;
|
||||
};
|
||||
chrome.runtime.onMessage.addListener(listener);
|
||||
return () => chrome.runtime.onMessage.removeListener(listener);
|
||||
}
|
||||
|
||||
async function finishRecordingByRequest(
|
||||
requestId: string,
|
||||
tabId: number,
|
||||
deps: RecordDeps,
|
||||
): Promise<void> {
|
||||
for (const [sessionId, recording] of recordings) {
|
||||
if (recording.requestId !== requestId || recording.settled) continue;
|
||||
const match = await findRecordingForTab(tabId, deps);
|
||||
if (match !== recording) continue;
|
||||
await finishRecording(sessionId, deps);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
async function finishRecording(sessionId: string, deps: RecordDeps): Promise<Trace | null> {
|
||||
const recording = recordings.get(sessionId);
|
||||
if (!recording || recording.settled || recording.finishing) return null;
|
||||
recording.finishing = true;
|
||||
|
||||
await clearRearmTimersForRecording(recording, deps);
|
||||
try {
|
||||
await stopRecordingOnAllAgentTabs(recording, deps);
|
||||
} catch {
|
||||
recording.finishing = false;
|
||||
return null;
|
||||
}
|
||||
|
||||
recording.settled = true;
|
||||
recordings.delete(sessionId);
|
||||
releaseBrowserObservationListenersIfIdle();
|
||||
const trace = buildTrace(recording);
|
||||
recording.resolveFinish(trace);
|
||||
return trace;
|
||||
}
|
||||
|
||||
export async function handleRecordStart(
|
||||
manager: SessionManager,
|
||||
params: RecordStartParams,
|
||||
deps: RecordDeps = getDefaultDeps(),
|
||||
): Promise<RecordStartResult | RpcError> {
|
||||
const ctxOrErr = lookupSession(manager, params, "record_start");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
if (recordings.has(params.session_id)) {
|
||||
return {
|
||||
code: "protocol_error",
|
||||
message: `session ${params.session_id} is already recording`,
|
||||
};
|
||||
}
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
|
||||
// Register the recording *before* navigate so content-script syncAgentOverlay
|
||||
// on the destination page can RECORD_QUERY → rearm → show RecordOverlay
|
||||
// instead of flashing ControlOverlay ("Agent 正在控制").
|
||||
const requestId = makeRequestId(target.tabId);
|
||||
let resolveFinish!: (trace: Trace) => void;
|
||||
let rejectFinish!: (err: Error) => void;
|
||||
const finishPromise = new Promise<Trace>((resolve, reject) => {
|
||||
resolveFinish = resolve;
|
||||
rejectFinish = reject;
|
||||
});
|
||||
const provisionalUrl = params.url;
|
||||
recordings.set(params.session_id, {
|
||||
requestId,
|
||||
tabId: target.tabId,
|
||||
agentWindowId: ctx.agentWindowId,
|
||||
startUrl: provisionalUrl,
|
||||
...(params.purpose ? { purpose: params.purpose } : {}),
|
||||
steps: [],
|
||||
startedAt: new Date().toISOString(),
|
||||
finishPromise,
|
||||
resolveFinish,
|
||||
rejectFinish,
|
||||
settled: false,
|
||||
finishing: false,
|
||||
currentUrl: provisionalUrl,
|
||||
pendingNavigation: false,
|
||||
pendingNavigationDeadline: undefined,
|
||||
});
|
||||
// Observe navigations for the whole recording lifetime; attach before
|
||||
// optional navigate so the destination load can rearm capture.
|
||||
ensureBrowserObservationListeners(deps);
|
||||
|
||||
const abortPending = async (notifyContent: boolean) => {
|
||||
recordings.delete(params.session_id);
|
||||
releaseBrowserObservationListenersIfIdle();
|
||||
if (notifyContent) {
|
||||
try {
|
||||
await deps.sendToTab(target.tabId, {
|
||||
type: RECORD_CANCEL,
|
||||
requestId,
|
||||
});
|
||||
} catch {
|
||||
// Content may never have received RECORD_START.
|
||||
}
|
||||
}
|
||||
if (deps.bypassOverlay) {
|
||||
try {
|
||||
await deps.bypassOverlay(target.tabId, false);
|
||||
} catch {
|
||||
// Ignore cleanup errors.
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const cancelledError = (): RpcError => ({
|
||||
code: "cancelled",
|
||||
message: "record_start aborted",
|
||||
});
|
||||
|
||||
/** Dispatcher may race-cancel while we still hold a provisional recording. */
|
||||
const abortIfCancelled = async (notifyContent: boolean): Promise<RpcError | null> => {
|
||||
if (!deps.signal?.aborted) return null;
|
||||
await abortPending(notifyContent);
|
||||
return cancelledError();
|
||||
};
|
||||
|
||||
{
|
||||
const cancelled = await abortIfCancelled(false);
|
||||
if (cancelled) return cancelled;
|
||||
}
|
||||
|
||||
if (params.url && deps.cdp) {
|
||||
const nav = await handleNavigate(
|
||||
manager,
|
||||
{
|
||||
session_id: params.session_id,
|
||||
url: params.url,
|
||||
tab_id: target.tabId,
|
||||
},
|
||||
{ cdp: deps.cdp, tabsApi: deps.tabsApi, signal: deps.signal },
|
||||
);
|
||||
if (isRpcError(nav)) {
|
||||
await abortPending(false);
|
||||
return nav;
|
||||
}
|
||||
{
|
||||
const cancelled = await abortIfCancelled(false);
|
||||
if (cancelled) return cancelled;
|
||||
}
|
||||
try {
|
||||
await waitForTabReady(target.tabId, deps.tabsApi);
|
||||
} catch {
|
||||
// Proceed with retries even if the tab never reports complete.
|
||||
}
|
||||
{
|
||||
const cancelled = await abortIfCancelled(false);
|
||||
if (cancelled) return cancelled;
|
||||
}
|
||||
}
|
||||
|
||||
let startUrl: string | undefined;
|
||||
try {
|
||||
const tab = await deps.tabsApi.get(target.tabId);
|
||||
startUrl = tab.url;
|
||||
} catch {
|
||||
startUrl = params.url;
|
||||
}
|
||||
|
||||
const active = recordings.get(params.session_id);
|
||||
if (!active) {
|
||||
// Cleared by a concurrent abort / session teardown.
|
||||
return cancelledError();
|
||||
}
|
||||
active.startUrl = startUrl;
|
||||
active.currentUrl = startUrl;
|
||||
|
||||
if (isContentScriptRestrictedUrl(startUrl)) {
|
||||
await abortPending(false);
|
||||
return {
|
||||
code: "invalid_params",
|
||||
message: params.url
|
||||
? `cannot record on restricted URL (${startUrl}); use an http(s) page`
|
||||
: "Agent Window starts on about:blank where the content script cannot inject — pass --url https://… before recording",
|
||||
};
|
||||
}
|
||||
|
||||
{
|
||||
const cancelled = await abortIfCancelled(false);
|
||||
if (cancelled) return cancelled;
|
||||
}
|
||||
|
||||
if (deps.bypassOverlay) {
|
||||
try {
|
||||
// Single ref for the initial race before RecordOverlay mounts; rearm must
|
||||
// not stack additional refs (see rearmRecording). Cleared on stop.
|
||||
await deps.bypassOverlay(target.tabId, true);
|
||||
} catch {
|
||||
// Best-effort; activeRecord also hides the control overlay.
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
const cancelled = await abortIfCancelled(true);
|
||||
if (cancelled) return cancelled;
|
||||
}
|
||||
|
||||
const startMsg: RecordStartMessage = { type: RECORD_START, requestId };
|
||||
|
||||
try {
|
||||
await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
|
||||
} catch {
|
||||
await abortPending(true);
|
||||
return {
|
||||
code: "protocol_error",
|
||||
message:
|
||||
"failed to start recording in content script — reload the BrowserSkill extension, then retry with --url https://…",
|
||||
};
|
||||
}
|
||||
|
||||
{
|
||||
const cancelled = await abortIfCancelled(true);
|
||||
if (cancelled) return cancelled;
|
||||
}
|
||||
|
||||
return { tab_id: target.tabId, recording: true };
|
||||
}
|
||||
|
||||
export async function handleRecordStop(
|
||||
manager: SessionManager,
|
||||
params: RecordStopParams,
|
||||
deps: RecordDeps = getDefaultDeps(),
|
||||
): Promise<RecordStopResult | RpcError> {
|
||||
const ctxOrErr = lookupSession(manager, params, "record_stop");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
|
||||
const recording = recordings.get(params.session_id);
|
||||
if (!recording) {
|
||||
return {
|
||||
code: "not_found",
|
||||
message: `no active recording for session ${params.session_id}`,
|
||||
};
|
||||
}
|
||||
|
||||
const trace = await finishRecording(params.session_id, deps);
|
||||
if (!trace) {
|
||||
return {
|
||||
code: "protocol_error",
|
||||
message: `failed to flush recorded steps for session ${params.session_id}; the recording is still active — retry \`bsk record stop\``,
|
||||
};
|
||||
}
|
||||
return { trace };
|
||||
}
|
||||
|
||||
export async function handleRecordAwait(
|
||||
manager: SessionManager,
|
||||
params: RecordAwaitParams,
|
||||
deps: RecordDeps = getDefaultDeps(),
|
||||
): Promise<RecordAwaitResult | RpcError> {
|
||||
const ctxOrErr = lookupSession(manager, params, "record_await");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
|
||||
const recording = recordings.get(params.session_id);
|
||||
if (!recording) {
|
||||
return {
|
||||
code: "not_found",
|
||||
message: `no active recording for session ${params.session_id}`,
|
||||
};
|
||||
}
|
||||
|
||||
if (deps.signal?.aborted) {
|
||||
return { code: "cancelled", message: "record_await aborted" };
|
||||
}
|
||||
|
||||
const outcome = await new Promise<{ trace: Trace } | { error: RpcError }>((resolve) => {
|
||||
let settled = false;
|
||||
const finish = (result: { trace: Trace } | { error: RpcError }) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
if (timer) clearTimeout(timer);
|
||||
deps.signal?.removeEventListener("abort", onAbort);
|
||||
resolve(result);
|
||||
};
|
||||
const onAbort = () => finish({ error: { code: "cancelled", message: "record_await aborted" } });
|
||||
const timer =
|
||||
params.timeout_ms === undefined
|
||||
? undefined
|
||||
: setTimeout(
|
||||
() =>
|
||||
finish({
|
||||
error: {
|
||||
code: "timeout",
|
||||
message: `record_await timed out after ${params.timeout_ms}ms`,
|
||||
},
|
||||
}),
|
||||
params.timeout_ms,
|
||||
);
|
||||
deps.signal?.addEventListener("abort", onAbort, { once: true });
|
||||
void recording.finishPromise.then(
|
||||
(trace) => finish({ trace }),
|
||||
() =>
|
||||
finish({
|
||||
error: { code: "cancelled", message: "recording was cleared" },
|
||||
}),
|
||||
);
|
||||
});
|
||||
return "trace" in outcome ? { trace: outcome.trace } : outcome.error;
|
||||
}
|
||||
|
||||
export function clearRecordingForSession(sessionId: string): void {
|
||||
const recording = recordings.get(sessionId);
|
||||
if (!recording) {
|
||||
recordings.delete(sessionId);
|
||||
releaseBrowserObservationListenersIfIdle();
|
||||
return;
|
||||
}
|
||||
void clearRearmTimersForRecording(recording, getDefaultDeps());
|
||||
if (!recording.settled) {
|
||||
recording.settled = true;
|
||||
recording.rejectFinish(new Error("recording cleared"));
|
||||
}
|
||||
recordings.delete(sessionId);
|
||||
releaseBrowserObservationListenersIfIdle();
|
||||
}
|
||||
|
||||
export type { RecordFinishMessage };
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { clearRecordingForSession } from "./record";
|
||||
import { returnBorrowedTab, type TabManagementDeps } from "./tabs";
|
||||
|
||||
export interface SessionStartParams {
|
||||
@@ -156,6 +157,8 @@ export async function handleSessionStop(
|
||||
// Step 2: clear the per-session RefStore (review M6/M7 parity).
|
||||
ctx.refStore.clear();
|
||||
|
||||
clearRecordingForSession(params.session_id);
|
||||
|
||||
// Step 3: detach CDP sessions this session opened (no-op if none).
|
||||
await deps.cdp?.detachSession(params.session_id);
|
||||
|
||||
|
||||
@@ -537,3 +537,135 @@ export interface RequestHelpResult {
|
||||
tab_id: number;
|
||||
resolved_targets?: ResolvedTarget[];
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Semantic record payloads — mirror bsk-protocol record.rs (Trace v2)
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
export interface TargetDescriptor {
|
||||
role?: string;
|
||||
name?: string;
|
||||
tag: string;
|
||||
name_attr?: string;
|
||||
placeholder?: string;
|
||||
nearby_label?: string;
|
||||
}
|
||||
|
||||
export interface TraceEntry {
|
||||
start_url: string;
|
||||
}
|
||||
|
||||
export interface PageRef {
|
||||
id: string;
|
||||
url: string;
|
||||
title?: string;
|
||||
}
|
||||
|
||||
export interface SelectedOption {
|
||||
value: string;
|
||||
label?: string;
|
||||
}
|
||||
|
||||
export interface StepEffect {
|
||||
navigated_to: string;
|
||||
}
|
||||
|
||||
export interface StepCommon {
|
||||
id: number;
|
||||
page: string;
|
||||
effect?: StepEffect;
|
||||
}
|
||||
|
||||
/** Capture/buffer draft before v2 reduction. */
|
||||
export type DraftTraceStep =
|
||||
| {
|
||||
op: "click";
|
||||
target: TargetDescriptor;
|
||||
navigated_to?: string;
|
||||
page_url?: string;
|
||||
}
|
||||
| {
|
||||
op: "fill";
|
||||
target: TargetDescriptor;
|
||||
value: string;
|
||||
redacted?: boolean;
|
||||
page_url?: string;
|
||||
}
|
||||
| {
|
||||
op: "press";
|
||||
key: string;
|
||||
target?: TargetDescriptor;
|
||||
modifiers?: KeyModifier[];
|
||||
navigated_to?: string;
|
||||
page_url?: string;
|
||||
}
|
||||
| {
|
||||
op: "select";
|
||||
target: TargetDescriptor;
|
||||
values: string[];
|
||||
labels?: string[];
|
||||
navigated_to?: string;
|
||||
page_url?: string;
|
||||
}
|
||||
| {
|
||||
op: "navigate";
|
||||
url: string;
|
||||
page_url?: string;
|
||||
};
|
||||
|
||||
/** Exported record-only step (trace v2). */
|
||||
export type Step =
|
||||
| ({ op: "navigate" } & StepCommon & { to: string })
|
||||
| ({ op: "click" } & StepCommon & { target: TargetDescriptor })
|
||||
| ({ op: "fill" } & StepCommon & {
|
||||
target: TargetDescriptor;
|
||||
value: string;
|
||||
redacted?: boolean;
|
||||
})
|
||||
| ({ op: "select" } & StepCommon & {
|
||||
target: TargetDescriptor;
|
||||
selection: SelectedOption[];
|
||||
})
|
||||
| ({ op: "press" } & StepCommon & {
|
||||
key: string;
|
||||
modifiers?: KeyModifier[];
|
||||
target?: TargetDescriptor;
|
||||
});
|
||||
|
||||
export interface Trace {
|
||||
recorded_at: string;
|
||||
started_at?: string;
|
||||
purpose?: string;
|
||||
entry: TraceEntry;
|
||||
pages: PageRef[];
|
||||
steps: Step[];
|
||||
}
|
||||
|
||||
export interface RecordStartParams {
|
||||
session_id: string;
|
||||
tab_id?: number;
|
||||
url?: string;
|
||||
purpose?: string;
|
||||
}
|
||||
|
||||
export interface RecordStartResult {
|
||||
tab_id: number;
|
||||
recording: boolean;
|
||||
}
|
||||
|
||||
export interface RecordStopParams {
|
||||
session_id: string;
|
||||
}
|
||||
|
||||
export interface RecordStopResult {
|
||||
trace: Trace;
|
||||
}
|
||||
|
||||
export interface RecordAwaitParams {
|
||||
session_id: string;
|
||||
timeout_ms?: number;
|
||||
}
|
||||
|
||||
export interface RecordAwaitResult {
|
||||
trace: Trace;
|
||||
}
|
||||
|
||||
@@ -25,7 +25,7 @@ export default defineConfig({
|
||||
name: "BrowserSkill",
|
||||
description:
|
||||
"Let AI agents use your logged-in browser in a separate Agent Window—without interrupting your work. Powered by the bsk CLI.",
|
||||
permissions: ["alarms", "debugger", "notifications", "tabs", "storage", "windows"],
|
||||
permissions: ["alarms", "debugger", "notifications", "tabs", "storage", "webNavigation", "windows"],
|
||||
host_permissions: ["<all_urls>"],
|
||||
icons: {
|
||||
16: "icon/logo.png",
|
||||
|
||||
@@ -46,6 +46,21 @@ Optional: `bsk session start --browser <instance-id-or-label>` when multiple bro
|
||||
|
||||
Emergency cleanup: `bsk session stop --all` or the Agent Window overlay **Stop all**.
|
||||
|
||||
## Stop when the goal is met
|
||||
|
||||
Every task is a **bounded goal**, not open-ended browsing. The goal may come from the user's request, a recorded `trace.json`, or both.
|
||||
|
||||
1. **Define success first** — one concrete, observable condition derived from the user's words, `purpose`, or the last meaningful step in a trace (e.g. "form submitted", "item added to cart", "playback started").
|
||||
2. **Take the shortest path** — snapshot → act → at most one check. Do not wander, re-try unrelated actions, or stack exploratory steps.
|
||||
3. **Stop as soon as success is reached** — run `bsk session stop <id>` immediately unless the user explicitly asked to keep the session open (e.g. "don't close yet", "keep browsing").
|
||||
4. **No post-success work** — once the goal is met, do not click, refresh, navigate, re-search, switch tabs, or "double-check" that it worked. Further verification is a new task.
|
||||
5. **When blocked, pause — do not brute-force** — if the page requires human input (login, captcha, OTP, payment confirmation) or an action fails twice with no progress, call `bsk request-help` instead of retrying blindly. See **Ask the human for help** below.
|
||||
6. **When unsure** — at most one extra `bsk snapshot`. If success looks met, stop. If not, ask the user; do not keep clicking.
|
||||
|
||||
**With a trace:** replay steps in order using `target` role/name/tag and raw `value`/`selection` fields. After the last step (or when its `effect.navigated_to` / success hint is satisfied), apply rules 3–4 immediately. The trace guides execution; it does not extend control beyond the goal.
|
||||
|
||||
**Without a trace:** the user's request *is* the success condition. Satisfying it ends the task — same stop rules apply.
|
||||
|
||||
## Core interaction loop
|
||||
|
||||
Write operations only affect tabs in the **Agent Window** (or tabs you **borrowed** into it).
|
||||
@@ -191,6 +206,23 @@ acts. The result `outcome` is one of:
|
||||
`note` carries any text the user typed back. `resolved_targets` reports
|
||||
which refs/selectors matched a live element.
|
||||
|
||||
### Recording — `bsk record`
|
||||
|
||||
Capture the user's own actions in the Agent Window to a `trace.json`, for later LLM-driven automation:
|
||||
|
||||
```bash
|
||||
bsk record start --url https://… [--purpose "publish a wiki doc"] [--output trace.json]
|
||||
# `--url` is required on a fresh session (about:blank cannot host the content script).
|
||||
# Blocks until the user clicks Finish in the recording panel, then writes ./trace.json and closes the window.
|
||||
|
||||
bsk record stop [--output trace.json] # terminal fallback if the browser panel is unavailable
|
||||
```
|
||||
|
||||
- The trace is a **record-only action log** (a `pages[]` dictionary + `navigate`/`click`/`fill`/`select`/`press` steps with `target` descriptors). It records *what the user did*; deciding which inputs are variable is left to the executing agent.
|
||||
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
||||
- There is **no** `bsk replay` — to redo a flow, read the trace and reuse the existing `session` / `snapshot` / `@eN` / `click` / `fill` tools. Follow **Stop when the goal is met**.
|
||||
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
||||
|
||||
## Error handling
|
||||
|
||||
### Exit codes (`echo $?` after `bsk …`)
|
||||
@@ -221,8 +253,9 @@ Always **`bsk session stop <id>`** in a `finally`-style path so the Agent Window
|
||||
1. **No token theft** — do not `bsk evaluate` on sensitive sites to read `localStorage`, cookies, or auth headers for exfiltration.
|
||||
2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks.
|
||||
3. **No skip stop** — always `bsk session stop <id>`; never assume idle timeout will clean up.
|
||||
4. **No observe escalation before snapshot** — use `bsk snapshot` first; only use `bsk get-html` or `bsk screenshot` when the snapshot is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot ref — never skip snapshot just to grab a visual.
|
||||
5. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
|
||||
4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open.
|
||||
5. **No observe escalation before snapshot** — use `bsk snapshot` first; only use `bsk get-html` or `bsk screenshot` when the snapshot is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot ref — never skip snapshot just to grab a visual.
|
||||
6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -19,6 +19,8 @@ pub mod interaction;
|
||||
pub mod logs;
|
||||
pub mod navigate;
|
||||
pub mod network;
|
||||
pub mod record;
|
||||
pub mod record_state;
|
||||
pub mod render_error;
|
||||
pub mod screenshot;
|
||||
pub mod session;
|
||||
@@ -39,6 +41,7 @@ use crate::cli::install_skill::InstallSkillArgs;
|
||||
use crate::cli::interaction::{ClickArgs, FillArgs, PressArgs, SelectArgs};
|
||||
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
|
||||
use crate::cli::network::NetworkArgs;
|
||||
use crate::cli::record::RecordCmd;
|
||||
use crate::cli::screenshot::ScreenshotArgs;
|
||||
use crate::cli::session::SessionCmd;
|
||||
use crate::cli::snapshot::SnapshotArgs;
|
||||
@@ -169,6 +172,9 @@ pub enum Command {
|
||||
/// Ask the human to complete an in-page step (captcha / login / confirm).
|
||||
#[command(name = "request-help")]
|
||||
RequestHelp(RequestHelpArgs),
|
||||
|
||||
/// Record user actions into a semantic `trace.json` textbook for LLMs.
|
||||
Record(RecordCmd),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Args, Default)]
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
//! `bsk record start|stop` — capture user actions in the Agent Window.
|
||||
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::Context;
|
||||
use bsk_protocol::Method;
|
||||
use bsk_protocol::tools::{
|
||||
RecordAwaitParams, RecordAwaitResult, RecordStartParams, RecordStartResult, RecordStopParams,
|
||||
RecordStopResult, Trace,
|
||||
};
|
||||
use clap::{Args, Subcommand};
|
||||
|
||||
use crate::cli::TOOL_IPC_TIMEOUT;
|
||||
use crate::cli::business_rpc;
|
||||
use crate::cli::ensure_daemon::ensure_daemon;
|
||||
use crate::cli::error::{CliError, Format};
|
||||
use crate::cli::record_state;
|
||||
use crate::cli::session::{start_session, stop_session};
|
||||
|
||||
/// Max wait for the user to click 结束 in the browser (24 hours).
|
||||
const RECORD_AWAIT_TIMEOUT_MS: u32 = 86_400_000;
|
||||
|
||||
/// `bsk record …` subcommand tree.
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct RecordCmd {
|
||||
#[command(subcommand)]
|
||||
pub sub: RecordSub,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Subcommand)]
|
||||
pub enum RecordSub {
|
||||
/// Open the Agent Window, record user actions, and block until finished in the browser.
|
||||
Start(RecordStartArgs),
|
||||
/// Stop recording from the terminal (fallback), write trace JSON, and close the window.
|
||||
/// Works even while `record start` is blocked in `record_await` (daemon forwards
|
||||
/// `tool.record_stop` without the per-session busy gate).
|
||||
Stop(RecordStopArgs),
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct RecordStartArgs {
|
||||
#[arg(long = "tab-id")]
|
||||
pub tab_id: Option<i64>,
|
||||
|
||||
/// Navigate to this http(s) URL before recording (required for a fresh
|
||||
/// session — Agent Window starts on about:blank, which cannot host the
|
||||
/// content script).
|
||||
#[arg(long)]
|
||||
pub url: Option<String>,
|
||||
|
||||
/// Optional goal text stored on the exported trace for LLM context.
|
||||
#[arg(long)]
|
||||
pub purpose: Option<String>,
|
||||
|
||||
/// Output path for the trace JSON (default `./trace.json`).
|
||||
#[arg(long, default_value = "trace.json")]
|
||||
pub output: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct RecordStopArgs {
|
||||
/// Output path for the trace JSON (default `./trace.json`).
|
||||
#[arg(long, default_value = "trace.json")]
|
||||
pub output: PathBuf,
|
||||
}
|
||||
|
||||
pub fn dispatch(cmd: RecordCmd, format: Format) -> Result<(), CliError> {
|
||||
match cmd.sub {
|
||||
RecordSub::Start(args) => dispatch_start(args, format),
|
||||
RecordSub::Stop(args) => dispatch_stop(args, format),
|
||||
}
|
||||
}
|
||||
|
||||
fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError> {
|
||||
if record_state::read().is_ok() {
|
||||
return Err(CliError::Local(anyhow::anyhow!(
|
||||
"a recording is already in progress; run `bsk record stop` first"
|
||||
)));
|
||||
}
|
||||
|
||||
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||
let session = start_session(info.sock_path.clone(), None)?;
|
||||
|
||||
let start_params = RecordStartParams {
|
||||
session_id: session.session_id.clone(),
|
||||
tab_id: args.tab_id,
|
||||
url: args.url,
|
||||
purpose: args.purpose.clone(),
|
||||
};
|
||||
let start_result = business_rpc::call::<RecordStartParams, RecordStartResult>(
|
||||
info.sock_path.clone(),
|
||||
"record-start",
|
||||
Method::ToolRecordStart,
|
||||
Some(start_params),
|
||||
TOOL_IPC_TIMEOUT,
|
||||
);
|
||||
|
||||
let start_result = match start_result {
|
||||
Ok(result) => result,
|
||||
Err(err) => {
|
||||
let _ = stop_session(info.sock_path, &session.session_id);
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
|
||||
if let Err(err) = record_state::write(&session.session_id) {
|
||||
let _ = stop_session(info.sock_path.clone(), &session.session_id);
|
||||
return Err(CliError::Local(err));
|
||||
}
|
||||
|
||||
if format == Format::Human {
|
||||
println!(
|
||||
"recording on tab={} — click 结束 in the browser when done",
|
||||
start_result.tab_id
|
||||
);
|
||||
}
|
||||
|
||||
let await_params = RecordAwaitParams {
|
||||
session_id: session.session_id.clone(),
|
||||
timeout_ms: Some(RECORD_AWAIT_TIMEOUT_MS),
|
||||
};
|
||||
let await_result = business_rpc::call::<RecordAwaitParams, RecordAwaitResult>(
|
||||
info.sock_path.clone(),
|
||||
"record-await",
|
||||
Method::ToolRecordAwait,
|
||||
Some(await_params),
|
||||
record_await_ipc_timeout(RECORD_AWAIT_TIMEOUT_MS),
|
||||
);
|
||||
|
||||
// Keep write/render inside a Result so `?` cannot skip session teardown.
|
||||
let run_result: Result<(), CliError> = match await_result {
|
||||
Ok(result) => (|| {
|
||||
write_trace_file(&args.output, &result.trace)?;
|
||||
render_finish(&result.trace, &args.output, format)
|
||||
})(),
|
||||
Err(err) => Err(err),
|
||||
};
|
||||
|
||||
let session_stop_result = stop_session(info.sock_path, &session.session_id);
|
||||
record_state::clear();
|
||||
|
||||
run_result?;
|
||||
session_stop_result?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn dispatch_stop(args: RecordStopArgs, format: Format) -> Result<(), CliError> {
|
||||
let state = record_state::read().map_err(CliError::Local)?;
|
||||
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||
let session_id = state.session_id.clone();
|
||||
|
||||
let run_result: Result<(), CliError> = (|| {
|
||||
let params = RecordStopParams {
|
||||
session_id: session_id.clone(),
|
||||
};
|
||||
let result = business_rpc::call::<RecordStopParams, RecordStopResult>(
|
||||
info.sock_path.clone(),
|
||||
"record-stop",
|
||||
Method::ToolRecordStop,
|
||||
Some(params),
|
||||
TOOL_IPC_TIMEOUT,
|
||||
)?;
|
||||
write_trace_file(&args.output, &result.trace)?;
|
||||
render_stop(&result, &args.output, format)
|
||||
})();
|
||||
|
||||
let session_stop_result = stop_session(info.sock_path, &session_id);
|
||||
record_state::clear();
|
||||
|
||||
run_result?;
|
||||
session_stop_result?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn record_await_ipc_timeout(timeout_ms: u32) -> Duration {
|
||||
Duration::from_millis(u64::from(timeout_ms))
|
||||
.checked_add(Duration::from_secs(15))
|
||||
.unwrap_or(Duration::from_secs(u64::from(timeout_ms / 1_000) + 15))
|
||||
}
|
||||
|
||||
fn write_trace_file(output: &PathBuf, trace: &Trace) -> Result<(), CliError> {
|
||||
let json = serde_json::to_string_pretty(trace)
|
||||
.context("serialize trace JSON")
|
||||
.map_err(CliError::Local)?;
|
||||
if let Some(parent) = output.parent() {
|
||||
if !parent.as_os_str().is_empty() {
|
||||
fs::create_dir_all(parent)
|
||||
.with_context(|| format!("create output directory {}", parent.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
}
|
||||
}
|
||||
fs::write(output, format!("{json}\n"))
|
||||
.with_context(|| format!("write trace to {}", output.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn render_finish(trace: &Trace, output: &PathBuf, format: Format) -> Result<(), CliError> {
|
||||
match format {
|
||||
Format::Json => {
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string(&serde_json::json!({
|
||||
"output": output,
|
||||
"trace": trace,
|
||||
"window_closed": true,
|
||||
}))
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
|
||||
);
|
||||
}
|
||||
Format::Human => {
|
||||
println!("saved {} steps to {}", trace.steps.len(), output.display());
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn render_stop(
|
||||
result: &RecordStopResult,
|
||||
output: &PathBuf,
|
||||
format: Format,
|
||||
) -> Result<(), CliError> {
|
||||
render_finish(&result.trace, output, format)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn default_output_is_trace_json() {
|
||||
let args = RecordStopArgs {
|
||||
output: PathBuf::from("trace.json"),
|
||||
};
|
||||
assert_eq!(args.output, PathBuf::from("trace.json"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn start_args_default_output_is_trace_json() {
|
||||
let args = RecordStartArgs {
|
||||
tab_id: None,
|
||||
url: None,
|
||||
purpose: None,
|
||||
output: PathBuf::from("trace.json"),
|
||||
};
|
||||
assert_eq!(args.output, PathBuf::from("trace.json"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn record_await_ipc_timeout_covers_long_wait() {
|
||||
let got = record_await_ipc_timeout(RECORD_AWAIT_TIMEOUT_MS);
|
||||
assert!(got >= Duration::from_secs(86_400));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,117 @@
|
||||
//! Persist the session id between `bsk record start` and `bsk record stop`.
|
||||
|
||||
use std::fs;
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::daemon::paths;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct RecordSessionState {
|
||||
pub session_id: String,
|
||||
pub started_at: String,
|
||||
}
|
||||
|
||||
pub fn write(session_id: &str) -> Result<()> {
|
||||
let path = paths::record_session_path()?;
|
||||
paths::ensure_bsk_home()?;
|
||||
if path.exists() {
|
||||
return Err(anyhow::anyhow!(
|
||||
"a recording is already in progress; run `bsk record stop` first"
|
||||
));
|
||||
}
|
||||
let state = RecordSessionState {
|
||||
session_id: session_id.to_string(),
|
||||
started_at: started_at_unix_ms(),
|
||||
};
|
||||
let json = serde_json::to_string_pretty(&state).context("serialize record session state")?;
|
||||
fs::write(&path, format!("{json}\n")).with_context(|| format!("write {}", path.display()))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn read() -> Result<RecordSessionState> {
|
||||
let path = paths::record_session_path()?;
|
||||
if !path.exists() {
|
||||
return Err(anyhow::anyhow!(
|
||||
"no recording in progress; run `bsk record start` first"
|
||||
));
|
||||
}
|
||||
let raw = fs::read_to_string(&path).with_context(|| format!("read {}", path.display()))?;
|
||||
serde_json::from_str(&raw).with_context(|| format!("parse {}", path.display()))
|
||||
}
|
||||
|
||||
pub fn clear() {
|
||||
if let Ok(path) = paths::record_session_path() {
|
||||
let _ = fs::remove_file(path);
|
||||
}
|
||||
}
|
||||
|
||||
fn started_at_unix_ms() -> String {
|
||||
let ms = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_millis();
|
||||
format!("{ms}")
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Mutex;
|
||||
|
||||
fn env_guard() -> &'static Mutex<()> {
|
||||
static GUARD: Mutex<()> = Mutex::new(());
|
||||
&GUARD
|
||||
}
|
||||
|
||||
fn with_temp_home<F: FnOnce()>(f: F) {
|
||||
let _lock = env_guard().lock().unwrap_or_else(|e| e.into_inner());
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
unsafe {
|
||||
std::env::set_var(crate::daemon::paths::BSK_HOME_ENV, tmp.path());
|
||||
}
|
||||
f();
|
||||
clear();
|
||||
unsafe {
|
||||
std::env::remove_var(crate::daemon::paths::BSK_HOME_ENV);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_read_round_trips() {
|
||||
with_temp_home(|| {
|
||||
write("abcd").unwrap();
|
||||
let state = read().unwrap();
|
||||
assert_eq!(state.session_id, "abcd");
|
||||
assert!(!state.started_at.is_empty());
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_rejects_duplicate() {
|
||||
with_temp_home(|| {
|
||||
write("abcd").unwrap();
|
||||
let err = write("efgh").unwrap_err();
|
||||
assert!(err.to_string().contains("already in progress"));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_errors_when_missing() {
|
||||
with_temp_home(|| {
|
||||
let err = read().unwrap_err();
|
||||
assert!(err.to_string().contains("no recording in progress"));
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn clear_removes_state_file() {
|
||||
with_temp_home(|| {
|
||||
write("abcd").unwrap();
|
||||
clear();
|
||||
assert!(read().is_err());
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -63,11 +63,11 @@ struct StartParams {
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct StartReply {
|
||||
session_id: String,
|
||||
browser_instance_id: String,
|
||||
pub struct StartReply {
|
||||
pub session_id: String,
|
||||
pub browser_instance_id: String,
|
||||
#[serde(default)]
|
||||
agent_window_id: Option<i64>,
|
||||
pub agent_window_id: Option<i64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
@@ -78,21 +78,21 @@ struct StopParams {
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct StopReply {
|
||||
stopped: Vec<String>,
|
||||
pub struct StopReply {
|
||||
pub stopped: Vec<String>,
|
||||
#[serde(default)]
|
||||
failed: Vec<StopFailure>,
|
||||
pub failed: Vec<StopFailure>,
|
||||
#[serde(default)]
|
||||
returned_tab_ids: Vec<i64>,
|
||||
pub returned_tab_ids: Vec<i64>,
|
||||
#[serde(default)]
|
||||
return_failures: Vec<ReturnFailure>,
|
||||
pub return_failures: Vec<ReturnFailure>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, Serialize)]
|
||||
struct StopFailure {
|
||||
session_id: String,
|
||||
code: ErrorCode,
|
||||
message: String,
|
||||
pub struct StopFailure {
|
||||
pub session_id: String,
|
||||
pub code: ErrorCode,
|
||||
pub message: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
@@ -113,14 +113,7 @@ pub fn dispatch(cmd: SessionCmd, format: Format) -> Result<(), CliError> {
|
||||
}
|
||||
|
||||
fn run_start(sock: PathBuf, args: SessionStartArgs, format: Format) -> Result<(), CliError> {
|
||||
let result: Result<StartReply, CliError> = call(
|
||||
sock,
|
||||
Method::SessionStart,
|
||||
Some(StartParams {
|
||||
browser_instance_id: args.browser,
|
||||
}),
|
||||
Duration::from_secs(30),
|
||||
);
|
||||
let result = start_session(sock, args.browser);
|
||||
match result {
|
||||
Ok(reply) => match format {
|
||||
Format::Json => {
|
||||
@@ -143,6 +136,31 @@ fn run_start(sock: PathBuf, args: SessionStartArgs, format: Format) -> Result<()
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Start a session and open the Agent Window. Used by `session start` and `record start`.
|
||||
pub fn start_session(sock: PathBuf, browser: Option<String>) -> Result<StartReply, CliError> {
|
||||
call(
|
||||
sock,
|
||||
Method::SessionStart,
|
||||
Some(StartParams {
|
||||
browser_instance_id: browser,
|
||||
}),
|
||||
Duration::from_secs(30),
|
||||
)
|
||||
}
|
||||
|
||||
/// Stop a single session by id.
|
||||
pub fn stop_session(sock: PathBuf, session_id: &str) -> Result<StopReply, CliError> {
|
||||
call(
|
||||
sock,
|
||||
Method::SessionStop,
|
||||
Some(StopParams {
|
||||
session_id: Some(session_id.to_string()),
|
||||
all: false,
|
||||
}),
|
||||
SESSION_STOP_IPC_TIMEOUT,
|
||||
)
|
||||
}
|
||||
|
||||
/// Render a `session.start` failure (review I3).
|
||||
///
|
||||
/// The previous implementation hand-rolled `eprintln!("error: ...")`
|
||||
|
||||
@@ -242,7 +242,10 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
|
||||
| Method::ToolSelect
|
||||
| Method::ToolEvaluate
|
||||
| Method::ToolWaitForNavigation
|
||||
| Method::ToolRequestHelp => {
|
||||
| Method::ToolRequestHelp
|
||||
| Method::ToolRecordStart
|
||||
| Method::ToolRecordStop
|
||||
| Method::ToolRecordAwait => {
|
||||
handle_tool_dispatch(&state, rpc_id, method, params).await
|
||||
}
|
||||
Method::ToolWaitMs => handle_wait_ms(&state.abort_registry, rpc_id, params).await,
|
||||
@@ -319,10 +322,19 @@ async fn handle_tool_dispatch(
|
||||
}
|
||||
};
|
||||
let entry = inflight_guard.entry();
|
||||
let outcome = state
|
||||
.tool_queues
|
||||
.dispatch(&session_id, method, params, timeout, Some(entry))
|
||||
.await;
|
||||
// `record_stop` must reach the extension while `record_await` holds the
|
||||
// serial busy lock — finishing the recording unblocks await.
|
||||
let outcome = if method == Method::ToolRecordStop {
|
||||
state
|
||||
.tool_queues
|
||||
.dispatch_unlocked(&session_id, method, params, timeout, Some(entry))
|
||||
.await
|
||||
} else {
|
||||
state
|
||||
.tool_queues
|
||||
.dispatch(&session_id, method, params, timeout, Some(entry))
|
||||
.await
|
||||
};
|
||||
drop(inflight_guard);
|
||||
match outcome {
|
||||
Ok(v) => ResponseBody::Ok(v),
|
||||
|
||||
@@ -95,6 +95,11 @@ pub fn sock_path() -> Result<PathBuf> {
|
||||
Ok(bsk_home()?.join("run").join("daemon.sock"))
|
||||
}
|
||||
|
||||
/// Path to the in-progress recording session state (`record-session.json`).
|
||||
pub fn record_session_path() -> Result<PathBuf> {
|
||||
Ok(bsk_home()?.join("record-session.json"))
|
||||
}
|
||||
|
||||
/// Windows named-pipe name. Include the resolved `BSK_HOME` path in the
|
||||
/// token so test homes and custom installs do not share a predictable
|
||||
/// per-username pipe.
|
||||
@@ -163,6 +168,10 @@ mod tests {
|
||||
assert_eq!(log_path().unwrap(), home.join("daemon.log"));
|
||||
assert_eq!(update_check_path().unwrap(), home.join("update-check.json"));
|
||||
assert_eq!(sock_path().unwrap(), home.join("run").join("daemon.sock"));
|
||||
assert_eq!(
|
||||
record_session_path().unwrap(),
|
||||
home.join("record-session.json")
|
||||
);
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -221,6 +221,13 @@ impl ToolQueueRegistry {
|
||||
guard.get(sid).is_some_and(|entry| entry.accepting)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn force_busy_for_tests(&self, sid: &SessionId, busy: bool) {
|
||||
let guard = self.queues.lock().expect("tool queue registry poisoned");
|
||||
let entry = guard.get(sid).expect("queue must exist");
|
||||
entry.state.lock().expect("queue state poisoned").busy = busy;
|
||||
}
|
||||
|
||||
/// Spawn the worker task for a session id. Idempotent: spawning the
|
||||
/// same id twice replaces the previous queue (the previous worker
|
||||
/// closes when its sender drops).
|
||||
@@ -320,6 +327,42 @@ impl ToolQueueRegistry {
|
||||
outcome
|
||||
}
|
||||
|
||||
/// Forward a tool RPC to the extension without taking the
|
||||
/// per-session busy lock. Used by `tool.record_stop` so a second
|
||||
/// CLI can finish an in-flight `tool.record_await` (extension
|
||||
/// dispatch already runs concurrent `void dispatch(msg)` handlers).
|
||||
pub async fn dispatch_unlocked(
|
||||
&self,
|
||||
sid: &SessionId,
|
||||
method: Method,
|
||||
params: Value,
|
||||
timeout: Duration,
|
||||
inflight: Option<Arc<ToolInflightEntry>>,
|
||||
) -> Result<Value, DispatchError> {
|
||||
{
|
||||
let guard = self.queues.lock().expect("tool queue registry poisoned");
|
||||
let Some(entry) = guard.get(sid) else {
|
||||
return Err(DispatchError::SessionNotFound);
|
||||
};
|
||||
if !entry.accepting {
|
||||
return Err(DispatchError::SessionStopping);
|
||||
}
|
||||
}
|
||||
let (respond_tx, _respond_rx) = oneshot::channel();
|
||||
let job = ToolJob {
|
||||
method,
|
||||
params,
|
||||
timeout,
|
||||
respond: respond_tx,
|
||||
inflight,
|
||||
session_id: sid.clone(),
|
||||
};
|
||||
match forward_one(sid, &self.browsers, &self.sessions, &job).await {
|
||||
Ok(v) => Ok(v),
|
||||
Err(err) => Err(DispatchError::Rpc(err)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Stop accepting new jobs for `sid`, enqueue one final control RPC,
|
||||
/// and wait for it behind any already-queued tools. Used by
|
||||
/// `session.stop` so extension teardown cannot race an in-flight
|
||||
@@ -789,3 +832,50 @@ mod tool_job_session_id_tests {
|
||||
assert_eq!(job.session_id.0, "sess-A");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod dispatch_unlocked_tests {
|
||||
use super::*;
|
||||
use crate::daemon::browsers::BrowserRegistry;
|
||||
use crate::daemon::sessions::{SessionId, SessionRegistry};
|
||||
use bsk_protocol::Method;
|
||||
use serde_json::json;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[tokio::test]
|
||||
async fn unlocked_dispatch_skips_session_busy_gate() {
|
||||
let browsers = Arc::new(BrowserRegistry::new());
|
||||
let sessions = Arc::new(SessionRegistry::new());
|
||||
let queues = ToolQueueRegistry::new(browsers, sessions);
|
||||
let sid = SessionId("s-record".into());
|
||||
queues.spawn(sid.clone());
|
||||
queues.force_busy_for_tests(&sid, true);
|
||||
|
||||
let busy = queues
|
||||
.dispatch(
|
||||
&sid,
|
||||
Method::ToolTabList,
|
||||
json!({ "session_id": "s-record" }),
|
||||
Duration::from_secs(1),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(busy, Err(DispatchError::SessionBusy)));
|
||||
|
||||
let unlocked = queues
|
||||
.dispatch_unlocked(
|
||||
&sid,
|
||||
Method::ToolRecordStop,
|
||||
json!({ "session_id": "s-record" }),
|
||||
Duration::from_secs(1),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
!matches!(unlocked, Err(DispatchError::SessionBusy)),
|
||||
"record_stop must not be rejected as SessionBusy; got {unlocked:?}"
|
||||
);
|
||||
// No session row → forward_one returns NotFound as Rpc.
|
||||
assert!(matches!(unlocked, Err(DispatchError::Rpc(_))));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,6 +97,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
|
||||
Command::WaitForNavigation(args) => cli::waits::dispatch_wait_for_navigation(args, format),
|
||||
Command::WaitMs(args) => cli::waits::dispatch_wait_ms(args, format),
|
||||
Command::RequestHelp(args) => cli::human_loop::dispatch(args, format),
|
||||
Command::Record(cmd) => cli::record::dispatch(cmd, format),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "RecordAwaitParams",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"session_id"
|
||||
],
|
||||
"properties": {
|
||||
"session_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"timeout_ms": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,417 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "RecordAwaitResult",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"trace"
|
||||
],
|
||||
"properties": {
|
||||
"trace": {
|
||||
"$ref": "#/definitions/Trace"
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"KeyModifier": {
|
||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"alt",
|
||||
"ctrl",
|
||||
"meta",
|
||||
"shift"
|
||||
]
|
||||
},
|
||||
"PageRef": {
|
||||
"description": "Page context dictionary entry — referenced by steps via `page` id.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"url"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"title": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"SelectedOption": {
|
||||
"description": "One selected option (`select` op).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"label": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"Step": {
|
||||
"description": "One recorded user action — discriminated union by `op`.",
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"to"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"navigate"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"to": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"click"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target",
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"fill"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"redacted": {
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"selection",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"select"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"selection": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/SelectedOption"
|
||||
}
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"key",
|
||||
"op",
|
||||
"page"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"key": {
|
||||
"type": "string"
|
||||
},
|
||||
"modifiers": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/KeyModifier"
|
||||
}
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"press"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"StepEffect": {
|
||||
"description": "Observed navigation after a step (objective fact only).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"navigated_to"
|
||||
],
|
||||
"properties": {
|
||||
"navigated_to": {
|
||||
"description": "Reference into `pages[]` for the destination page.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TargetDescriptor": {
|
||||
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"tag"
|
||||
],
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"name_attr": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"nearby_label": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"placeholder": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"role": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"tag": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"Trace": {
|
||||
"description": "Persisted user-action trace exported by `tool.record_stop` / `await`.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"entry",
|
||||
"pages",
|
||||
"recorded_at",
|
||||
"steps"
|
||||
],
|
||||
"properties": {
|
||||
"entry": {
|
||||
"$ref": "#/definitions/TraceEntry"
|
||||
},
|
||||
"pages": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/PageRef"
|
||||
}
|
||||
},
|
||||
"purpose": {
|
||||
"description": "Optional user-provided goal from `--purpose` (metadata only).",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"recorded_at": {
|
||||
"description": "RFC 3339 timestamp when recording stopped.",
|
||||
"type": "string"
|
||||
},
|
||||
"started_at": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"steps": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/Step"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceEntry": {
|
||||
"description": "Recording entry point — first URL the flow starts from.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"start_url"
|
||||
],
|
||||
"properties": {
|
||||
"start_url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "RecordStartParams",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"session_id"
|
||||
],
|
||||
"properties": {
|
||||
"purpose": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"session_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"tab_id": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "int64"
|
||||
},
|
||||
"url": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "RecordStartResult",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"recording",
|
||||
"tab_id"
|
||||
],
|
||||
"properties": {
|
||||
"recording": {
|
||||
"type": "boolean"
|
||||
},
|
||||
"tab_id": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "RecordStopParams",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"session_id"
|
||||
],
|
||||
"properties": {
|
||||
"session_id": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,417 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "RecordStopResult",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"trace"
|
||||
],
|
||||
"properties": {
|
||||
"trace": {
|
||||
"$ref": "#/definitions/Trace"
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"KeyModifier": {
|
||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"alt",
|
||||
"ctrl",
|
||||
"meta",
|
||||
"shift"
|
||||
]
|
||||
},
|
||||
"PageRef": {
|
||||
"description": "Page context dictionary entry — referenced by steps via `page` id.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"url"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"title": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"SelectedOption": {
|
||||
"description": "One selected option (`select` op).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"label": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"Step": {
|
||||
"description": "One recorded user action — discriminated union by `op`.",
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"to"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"navigate"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"to": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"click"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target",
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"fill"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"redacted": {
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"selection",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"select"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"selection": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/SelectedOption"
|
||||
}
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"key",
|
||||
"op",
|
||||
"page"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"key": {
|
||||
"type": "string"
|
||||
},
|
||||
"modifiers": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/KeyModifier"
|
||||
}
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"press"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"StepEffect": {
|
||||
"description": "Observed navigation after a step (objective fact only).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"navigated_to"
|
||||
],
|
||||
"properties": {
|
||||
"navigated_to": {
|
||||
"description": "Reference into `pages[]` for the destination page.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TargetDescriptor": {
|
||||
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"tag"
|
||||
],
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"name_attr": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"nearby_label": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"placeholder": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"role": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"tag": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"Trace": {
|
||||
"description": "Persisted user-action trace exported by `tool.record_stop` / `await`.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"entry",
|
||||
"pages",
|
||||
"recorded_at",
|
||||
"steps"
|
||||
],
|
||||
"properties": {
|
||||
"entry": {
|
||||
"$ref": "#/definitions/TraceEntry"
|
||||
},
|
||||
"pages": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/PageRef"
|
||||
}
|
||||
},
|
||||
"purpose": {
|
||||
"description": "Optional user-provided goal from `--purpose` (metadata only).",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"recorded_at": {
|
||||
"description": "RFC 3339 timestamp when recording stopped.",
|
||||
"type": "string"
|
||||
},
|
||||
"started_at": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"steps": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/Step"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceEntry": {
|
||||
"description": "Recording entry point — first URL the flow starts from.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"start_url"
|
||||
],
|
||||
"properties": {
|
||||
"start_url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,406 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "Trace",
|
||||
"description": "Persisted user-action trace exported by `tool.record_stop` / `await`.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"entry",
|
||||
"pages",
|
||||
"recorded_at",
|
||||
"steps"
|
||||
],
|
||||
"properties": {
|
||||
"entry": {
|
||||
"$ref": "#/definitions/TraceEntry"
|
||||
},
|
||||
"pages": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/PageRef"
|
||||
}
|
||||
},
|
||||
"purpose": {
|
||||
"description": "Optional user-provided goal from `--purpose` (metadata only).",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"recorded_at": {
|
||||
"description": "RFC 3339 timestamp when recording stopped.",
|
||||
"type": "string"
|
||||
},
|
||||
"started_at": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"steps": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/Step"
|
||||
}
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"KeyModifier": {
|
||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"alt",
|
||||
"ctrl",
|
||||
"meta",
|
||||
"shift"
|
||||
]
|
||||
},
|
||||
"PageRef": {
|
||||
"description": "Page context dictionary entry — referenced by steps via `page` id.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"url"
|
||||
],
|
||||
"properties": {
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"title": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"SelectedOption": {
|
||||
"description": "One selected option (`select` op).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"label": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"Step": {
|
||||
"description": "One recorded user action — discriminated union by `op`.",
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"to"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"navigate"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"to": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"click"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target",
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"fill"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"redacted": {
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"selection",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"select"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"selection": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/SelectedOption"
|
||||
}
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"key",
|
||||
"op",
|
||||
"page"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"key": {
|
||||
"type": "string"
|
||||
},
|
||||
"modifiers": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/KeyModifier"
|
||||
}
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"press"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
"StepEffect": {
|
||||
"description": "Observed navigation after a step (objective fact only).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"navigated_to"
|
||||
],
|
||||
"properties": {
|
||||
"navigated_to": {
|
||||
"description": "Reference into `pages[]` for the destination page.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TargetDescriptor": {
|
||||
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"tag"
|
||||
],
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"name_attr": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"nearby_label": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"placeholder": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"role": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"tag": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TraceEntry": {
|
||||
"description": "Recording entry point — first URL the flow starts from.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"start_url"
|
||||
],
|
||||
"properties": {
|
||||
"start_url": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,328 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "Step",
|
||||
"description": "One recorded user action — discriminated union by `op`.",
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"to"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"navigate"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"to": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"click"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"target",
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"fill"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"redacted": {
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"op",
|
||||
"page",
|
||||
"selection",
|
||||
"target"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"select"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"selection": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/SelectedOption"
|
||||
}
|
||||
},
|
||||
"target": {
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"description": "Fields shared by every step variant (flattened in JSON).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"id",
|
||||
"key",
|
||||
"op",
|
||||
"page"
|
||||
],
|
||||
"properties": {
|
||||
"effect": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/StepEffect"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"id": {
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"key": {
|
||||
"type": "string"
|
||||
},
|
||||
"modifiers": {
|
||||
"type": [
|
||||
"array",
|
||||
"null"
|
||||
],
|
||||
"items": {
|
||||
"$ref": "#/definitions/KeyModifier"
|
||||
}
|
||||
},
|
||||
"op": {
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"press"
|
||||
]
|
||||
},
|
||||
"page": {
|
||||
"description": "Reference into `pages[]`.",
|
||||
"type": "string"
|
||||
},
|
||||
"target": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/TargetDescriptor"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"definitions": {
|
||||
"KeyModifier": {
|
||||
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"alt",
|
||||
"ctrl",
|
||||
"meta",
|
||||
"shift"
|
||||
]
|
||||
},
|
||||
"SelectedOption": {
|
||||
"description": "One selected option (`select` op).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"value"
|
||||
],
|
||||
"properties": {
|
||||
"label": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"value": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"StepEffect": {
|
||||
"description": "Observed navigation after a step (objective fact only).",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"navigated_to"
|
||||
],
|
||||
"properties": {
|
||||
"navigated_to": {
|
||||
"description": "Reference into `pages[]` for the destination page.",
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"TargetDescriptor": {
|
||||
"description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"tag"
|
||||
],
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"name_attr": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"nearby_label": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"placeholder": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"role": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"tag": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -100,4 +100,13 @@ fn main() {
|
||||
dump!(WaitMsResult, "tool_wait_ms_result");
|
||||
dump!(RequestHelpParams, "tool_request_help_params");
|
||||
dump!(RequestHelpResult, "tool_request_help_result");
|
||||
|
||||
dump!(Trace, "trace");
|
||||
dump!(Step, "trace_step");
|
||||
dump!(RecordStartParams, "tool_record_start_params");
|
||||
dump!(RecordStartResult, "tool_record_start_result");
|
||||
dump!(RecordStopParams, "tool_record_stop_params");
|
||||
dump!(RecordStopResult, "tool_record_stop_result");
|
||||
dump!(RecordAwaitParams, "tool_record_await_params");
|
||||
dump!(RecordAwaitResult, "tool_record_await_result");
|
||||
}
|
||||
|
||||
@@ -74,6 +74,12 @@ pub enum Method {
|
||||
ToolWaitMs,
|
||||
#[serde(rename = "tool.request_help")]
|
||||
ToolRequestHelp,
|
||||
#[serde(rename = "tool.record_start")]
|
||||
ToolRecordStart,
|
||||
#[serde(rename = "tool.record_stop")]
|
||||
ToolRecordStop,
|
||||
#[serde(rename = "tool.record_await")]
|
||||
ToolRecordAwait,
|
||||
|
||||
#[serde(rename = "cancel")]
|
||||
Cancel,
|
||||
@@ -121,9 +127,16 @@ impl Method {
|
||||
| Method::ToolFill
|
||||
| Method::ToolPress
|
||||
| Method::ToolSelect
|
||||
| Method::ToolEvaluate => true,
|
||||
| Method::ToolEvaluate
|
||||
// May navigate via optional `url` and changes Agent Window
|
||||
// chrome; gate behind pending-interrupt like other writes.
|
||||
| Method::ToolRecordStart => true,
|
||||
|
||||
// Read-only tool calls — transparent.
|
||||
//
|
||||
// `record_stop` / `record_await` observe / finish a recording
|
||||
// without driving new automation gestures, so they stay
|
||||
// ungated (teardown after interrupt must still work).
|
||||
Method::ToolTabList
|
||||
| Method::ToolSnapshot
|
||||
| Method::ToolGetHtml
|
||||
@@ -132,7 +145,9 @@ impl Method {
|
||||
| Method::ToolNetwork
|
||||
| Method::ToolWaitForNavigation
|
||||
| Method::ToolWaitMs
|
||||
| Method::ToolRequestHelp => false,
|
||||
| Method::ToolRequestHelp
|
||||
| Method::ToolRecordStop
|
||||
| Method::ToolRecordAwait => false,
|
||||
|
||||
// Session lifecycle — not gated.
|
||||
Method::SessionStart
|
||||
@@ -218,6 +233,13 @@ mod tests {
|
||||
assert!(Method::ToolPress.is_mutating());
|
||||
assert!(Method::ToolSelect.is_mutating());
|
||||
assert!(Method::ToolEvaluate.is_mutating());
|
||||
assert!(Method::ToolRecordStart.is_mutating());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn is_mutating_classifies_record_stop_await_as_non_mutating() {
|
||||
assert!(!Method::ToolRecordStop.is_mutating());
|
||||
assert!(!Method::ToolRecordAwait.is_mutating());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! Typed params/results for the 21 `tool.*` RPC methods (§7).
|
||||
//! Typed params/results for the `tool.*` RPC methods (§7).
|
||||
|
||||
pub mod console;
|
||||
pub mod dialog;
|
||||
@@ -7,6 +7,7 @@ pub mod interaction;
|
||||
pub mod navigation;
|
||||
pub mod network;
|
||||
pub mod observation;
|
||||
pub mod record;
|
||||
pub mod script;
|
||||
pub mod session;
|
||||
pub mod tabs;
|
||||
@@ -19,6 +20,7 @@ pub use interaction::*;
|
||||
pub use navigation::*;
|
||||
pub use network::*;
|
||||
pub use observation::*;
|
||||
pub use record::*;
|
||||
pub use script::*;
|
||||
pub use session::*;
|
||||
pub use tabs::*;
|
||||
|
||||
@@ -0,0 +1,458 @@
|
||||
//! Semantic user-action recording (`tool.record_start` / `stop` / `await`).
|
||||
//!
|
||||
//! Trace v2 is a **record-only** log of user actions: what was clicked, filled,
|
||||
//! selected, and where navigation occurred. No LLM runs during recording, so
|
||||
//! variable-vs-constant classification is **not** stored — executing agents
|
||||
//! infer that at run time from raw values and control names.
|
||||
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::interaction::KeyModifier;
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Target
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Stable semantic handle for an interacted element.
|
||||
///
|
||||
/// `name` and `nearby_label` are **untrusted page text**.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct TargetDescriptor {
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub role: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub name: Option<String>,
|
||||
pub tag: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub name_attr: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub placeholder: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub nearby_label: Option<String>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Trace envelope
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Recording entry point — first URL the flow starts from.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct TraceEntry {
|
||||
pub start_url: String,
|
||||
}
|
||||
|
||||
/// Page context dictionary entry — referenced by steps via `page` id.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct PageRef {
|
||||
pub id: String,
|
||||
pub url: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub title: Option<String>,
|
||||
}
|
||||
|
||||
/// One selected option (`select` op).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct SelectedOption {
|
||||
pub value: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub label: Option<String>,
|
||||
}
|
||||
|
||||
/// Observed navigation after a step (objective fact only).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct StepEffect {
|
||||
/// Reference into `pages[]` for the destination page.
|
||||
pub navigated_to: String,
|
||||
}
|
||||
|
||||
/// Fields shared by every step variant (flattened in JSON).
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct StepCommon {
|
||||
pub id: u32,
|
||||
/// Reference into `pages[]`.
|
||||
pub page: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub effect: Option<StepEffect>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Step op-specific payloads
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// One recorded user action — discriminated union by `op`.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(tag = "op", rename_all = "snake_case")]
|
||||
pub enum Step {
|
||||
Navigate {
|
||||
#[serde(flatten)]
|
||||
common: StepCommon,
|
||||
to: String,
|
||||
},
|
||||
Click {
|
||||
#[serde(flatten)]
|
||||
common: StepCommon,
|
||||
target: TargetDescriptor,
|
||||
},
|
||||
Fill {
|
||||
#[serde(flatten)]
|
||||
common: StepCommon,
|
||||
target: TargetDescriptor,
|
||||
value: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
redacted: Option<bool>,
|
||||
},
|
||||
Select {
|
||||
#[serde(flatten)]
|
||||
common: StepCommon,
|
||||
target: TargetDescriptor,
|
||||
selection: Vec<SelectedOption>,
|
||||
},
|
||||
Press {
|
||||
#[serde(flatten)]
|
||||
common: StepCommon,
|
||||
key: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
modifiers: Option<Vec<KeyModifier>>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
target: Option<TargetDescriptor>,
|
||||
},
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Trace root
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Persisted user-action trace exported by `tool.record_stop` / `await`.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct Trace {
|
||||
/// RFC 3339 timestamp when recording stopped.
|
||||
pub recorded_at: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub started_at: Option<String>,
|
||||
/// Optional user-provided goal from `--purpose` (metadata only).
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub purpose: Option<String>,
|
||||
pub entry: TraceEntry,
|
||||
pub pages: Vec<PageRef>,
|
||||
pub steps: Vec<Step>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// RPC params / results
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct RecordStartParams {
|
||||
pub session_id: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub tab_id: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub url: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub purpose: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct RecordStartResult {
|
||||
pub tab_id: i64,
|
||||
pub recording: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct RecordStopParams {
|
||||
pub session_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct RecordStopResult {
|
||||
pub trace: Trace,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct RecordAwaitParams {
|
||||
pub session_id: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub timeout_ms: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct RecordAwaitResult {
|
||||
pub trace: Trace,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use serde_json::json;
|
||||
|
||||
fn sample_common(id: u32) -> StepCommon {
|
||||
StepCommon {
|
||||
id,
|
||||
page: "p1".into(),
|
||||
effect: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_target() -> TargetDescriptor {
|
||||
TargetDescriptor {
|
||||
role: Some("button".into()),
|
||||
name: Some("发布".into()),
|
||||
tag: "button".into(),
|
||||
name_attr: None,
|
||||
placeholder: None,
|
||||
nearby_label: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn sample_trace() -> Trace {
|
||||
Trace {
|
||||
recorded_at: "2026-07-17T09:01:10Z".into(),
|
||||
started_at: Some("2026-07-17T09:00:00Z".into()),
|
||||
purpose: Some("发布一篇文章".into()),
|
||||
entry: TraceEntry {
|
||||
start_url: "https://x.com/editor".into(),
|
||||
},
|
||||
pages: vec![
|
||||
PageRef {
|
||||
id: "p1".into(),
|
||||
url: "https://x.com/editor".into(),
|
||||
title: Some("写文章".into()),
|
||||
},
|
||||
PageRef {
|
||||
id: "p2".into(),
|
||||
url: "https://x.com/p/99".into(),
|
||||
title: None,
|
||||
},
|
||||
],
|
||||
steps: vec![
|
||||
Step::Fill {
|
||||
common: sample_common(1),
|
||||
target: TargetDescriptor {
|
||||
role: Some("textbox".into()),
|
||||
name: Some("标题".into()),
|
||||
tag: "input".into(),
|
||||
name_attr: None,
|
||||
placeholder: None,
|
||||
nearby_label: None,
|
||||
},
|
||||
value: "我的第一篇文章".into(),
|
||||
redacted: None,
|
||||
},
|
||||
Step::Select {
|
||||
common: sample_common(3),
|
||||
target: TargetDescriptor {
|
||||
role: Some("combobox".into()),
|
||||
name: Some("分类".into()),
|
||||
tag: "select".into(),
|
||||
name_attr: None,
|
||||
placeholder: None,
|
||||
nearby_label: None,
|
||||
},
|
||||
selection: vec![SelectedOption {
|
||||
value: "tech".into(),
|
||||
label: Some("技术分享".into()),
|
||||
}],
|
||||
},
|
||||
Step::Click {
|
||||
common: StepCommon {
|
||||
effect: Some(StepEffect {
|
||||
navigated_to: "p2".into(),
|
||||
}),
|
||||
..sample_common(4)
|
||||
},
|
||||
target: sample_target(),
|
||||
},
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn step_click_round_trips() {
|
||||
let step = Step::Click {
|
||||
common: sample_common(1),
|
||||
target: sample_target(),
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert_eq!(v.get("op").and_then(|v| v.as_str()), Some("click"));
|
||||
assert_eq!(v.get("page").and_then(|v| v.as_str()), Some("p1"));
|
||||
assert!(v.get("key").is_none());
|
||||
let round: Step = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, step);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn step_fill_with_raw_value_round_trips() {
|
||||
let step = Step::Fill {
|
||||
common: sample_common(2),
|
||||
target: TargetDescriptor {
|
||||
role: Some("textbox".into()),
|
||||
name: Some("服务名称".into()),
|
||||
tag: "input".into(),
|
||||
name_attr: Some("serviceName".into()),
|
||||
placeholder: None,
|
||||
nearby_label: None,
|
||||
},
|
||||
value: "my-svc".into(),
|
||||
redacted: None,
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert_eq!(v["op"], "fill");
|
||||
assert_eq!(v["value"], "my-svc");
|
||||
let round: Step = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, step);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn step_fill_password_is_redacted() {
|
||||
let step = Step::Fill {
|
||||
common: sample_common(1),
|
||||
target: TargetDescriptor {
|
||||
role: Some("textbox".into()),
|
||||
name: Some("密码".into()),
|
||||
tag: "input".into(),
|
||||
name_attr: None,
|
||||
placeholder: None,
|
||||
nearby_label: None,
|
||||
},
|
||||
value: "***".into(),
|
||||
redacted: Some(true),
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert_eq!(v["value"], "***");
|
||||
assert_eq!(v["redacted"], true);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn step_select_uses_object_array() {
|
||||
let step = Step::Select {
|
||||
common: sample_common(3),
|
||||
target: TargetDescriptor {
|
||||
role: Some("combobox".into()),
|
||||
name: Some("分类".into()),
|
||||
tag: "select".into(),
|
||||
name_attr: None,
|
||||
placeholder: None,
|
||||
nearby_label: None,
|
||||
},
|
||||
selection: vec![SelectedOption {
|
||||
value: "tech".into(),
|
||||
label: Some("技术分享".into()),
|
||||
}],
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert_eq!(v["selection"][0]["value"], "tech");
|
||||
assert_eq!(v["selection"][0]["label"], "技术分享");
|
||||
let round: Step = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, step);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn step_navigate_round_trips() {
|
||||
let step = Step::Navigate {
|
||||
common: sample_common(1),
|
||||
to: "https://example.com".into(),
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert_eq!(v["op"], "navigate");
|
||||
assert_eq!(v["to"], "https://example.com");
|
||||
let round: Step = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, step);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn step_press_with_modifiers_round_trips() {
|
||||
let step = Step::Press {
|
||||
common: StepCommon {
|
||||
effect: Some(StepEffect {
|
||||
navigated_to: "p2".into(),
|
||||
}),
|
||||
..sample_common(2)
|
||||
},
|
||||
key: "Enter".into(),
|
||||
modifiers: Some(vec![KeyModifier::Ctrl, KeyModifier::Shift]),
|
||||
target: Some(sample_target()),
|
||||
};
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert_eq!(v["key"], "Enter");
|
||||
assert_eq!(v["modifiers"], json!(["ctrl", "shift"]));
|
||||
assert_eq!(v["effect"]["navigated_to"], "p2");
|
||||
let round: Step = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, step);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn trace_record_only_round_trips() {
|
||||
let trace = sample_trace();
|
||||
let v = serde_json::to_value(&trace).unwrap();
|
||||
assert!(v.get("version").is_none());
|
||||
assert_eq!(
|
||||
v.get("purpose").and_then(|v| v.as_str()),
|
||||
Some("发布一篇文章")
|
||||
);
|
||||
assert!(v.get("parameters").is_none());
|
||||
assert!(v.get("site").is_none());
|
||||
assert!(v.get("goal").is_none());
|
||||
assert_eq!(
|
||||
v.get("entry")
|
||||
.and_then(|e| e.get("start_url"))
|
||||
.and_then(|u| u.as_str()),
|
||||
Some("https://x.com/editor")
|
||||
);
|
||||
let round: Trace = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, trace);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn discriminated_union_click_ignores_extra_key_field() {
|
||||
let bad = json!({
|
||||
"op": "click",
|
||||
"id": 1,
|
||||
"page": "p1",
|
||||
"target": { "tag": "button", "name": "OK" },
|
||||
"key": "Enter"
|
||||
});
|
||||
let step: Step = serde_json::from_value(bad).unwrap();
|
||||
assert!(matches!(step, Step::Click { .. }));
|
||||
let v = serde_json::to_value(&step).unwrap();
|
||||
assert!(v.get("key").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extension_trace_deserializes() {
|
||||
let v = json!({
|
||||
"recorded_at": "2026-07-21T08:00:00Z",
|
||||
"started_at": "2026-07-21T07:59:00Z",
|
||||
"purpose": "demo",
|
||||
"entry": { "start_url": "https://example.com/editor" },
|
||||
"pages": [
|
||||
{ "id": "p1", "url": "https://example.com/editor" },
|
||||
{ "id": "p2", "url": "https://example.com/p/99" }
|
||||
],
|
||||
"steps": [
|
||||
{
|
||||
"op": "fill",
|
||||
"id": 1,
|
||||
"page": "p1",
|
||||
"target": { "tag": "input", "role": "textbox", "name": "标题" },
|
||||
"value": "hello"
|
||||
},
|
||||
{
|
||||
"op": "click",
|
||||
"id": 2,
|
||||
"page": "p1",
|
||||
"target": { "tag": "button", "role": "button", "name": "发布" },
|
||||
"effect": { "navigated_to": "p2" }
|
||||
}
|
||||
]
|
||||
});
|
||||
let trace: Trace = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(trace.entry.start_url, "https://example.com/editor");
|
||||
assert_eq!(trace.pages.len(), 2);
|
||||
assert_eq!(trace.steps.len(), 2);
|
||||
}
|
||||
}
|
||||
@@ -68,5 +68,9 @@
|
||||
"expand": "Expand",
|
||||
"dragHandle": "Drag to move",
|
||||
"notificationTitle": "BrowserSkill: Agent needs your help"
|
||||
},
|
||||
"recordOverlay": {
|
||||
"recording": "Recording your actions",
|
||||
"finish": "Finish"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -68,5 +68,9 @@
|
||||
"expand": "展开",
|
||||
"dragHandle": "拖动以移动",
|
||||
"notificationTitle": "BrowserSkill:Agent 需要你的帮助"
|
||||
},
|
||||
"recordOverlay": {
|
||||
"recording": "正在录制用户操作",
|
||||
"finish": "结束"
|
||||
}
|
||||
}
|
||||
|
||||
+35
-2
@@ -46,6 +46,21 @@ Optional: `bsk session start --browser <instance-id-or-label>` when multiple bro
|
||||
|
||||
Emergency cleanup: `bsk session stop --all` or the Agent Window overlay **Stop all**.
|
||||
|
||||
## Stop when the goal is met
|
||||
|
||||
Every task is a **bounded goal**, not open-ended browsing. The goal may come from the user's request, a recorded `trace.json`, or both.
|
||||
|
||||
1. **Define success first** — one concrete, observable condition derived from the user's words, `purpose`, or the last meaningful step in a trace (e.g. "form submitted", "item added to cart", "playback started").
|
||||
2. **Take the shortest path** — snapshot → act → at most one check. Do not wander, re-try unrelated actions, or stack exploratory steps.
|
||||
3. **Stop as soon as success is reached** — run `bsk session stop <id>` immediately unless the user explicitly asked to keep the session open (e.g. "don't close yet", "keep browsing").
|
||||
4. **No post-success work** — once the goal is met, do not click, refresh, navigate, re-search, switch tabs, or "double-check" that it worked. Further verification is a new task.
|
||||
5. **When blocked, pause — do not brute-force** — if the page requires human input (login, captcha, OTP, payment confirmation) or an action fails twice with no progress, call `bsk request-help` instead of retrying blindly. See **Ask the human for help** below.
|
||||
6. **When unsure** — at most one extra `bsk snapshot`. If success looks met, stop. If not, ask the user; do not keep clicking.
|
||||
|
||||
**With a trace:** replay steps in order using `target` role/name/tag and raw `value`/`selection` fields. After the last step (or when its `effect.navigated_to` / success hint is satisfied), apply rules 3–4 immediately. The trace guides execution; it does not extend control beyond the goal.
|
||||
|
||||
**Without a trace:** the user's request *is* the success condition. Satisfying it ends the task — same stop rules apply.
|
||||
|
||||
## Core interaction loop
|
||||
|
||||
Write operations only affect tabs in the **Agent Window** (or tabs you **borrowed** into it).
|
||||
@@ -191,6 +206,23 @@ acts. The result `outcome` is one of:
|
||||
`note` carries any text the user typed back. `resolved_targets` reports
|
||||
which refs/selectors matched a live element.
|
||||
|
||||
### Recording — `bsk record`
|
||||
|
||||
Capture the user's own actions in the Agent Window to a `trace.json`, for later LLM-driven automation:
|
||||
|
||||
```bash
|
||||
bsk record start --url https://… [--purpose "publish a wiki doc"] [--output trace.json]
|
||||
# `--url` is required on a fresh session (about:blank cannot host the content script).
|
||||
# Blocks until the user clicks Finish in the recording panel, then writes ./trace.json and closes the window.
|
||||
|
||||
bsk record stop [--output trace.json] # terminal fallback if the browser panel is unavailable
|
||||
```
|
||||
|
||||
- The trace is a **record-only action log** (a `pages[]` dictionary + `navigate`/`click`/`fill`/`select`/`press` steps with `target` descriptors). It records *what the user did*; deciding which inputs are variable is left to the executing agent.
|
||||
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
||||
- There is **no** `bsk replay` — to redo a flow, read the trace and reuse the existing `session` / `snapshot` / `@eN` / `click` / `fill` tools. Follow **Stop when the goal is met**.
|
||||
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
||||
|
||||
## Error handling
|
||||
|
||||
### Exit codes (`echo $?` after `bsk …`)
|
||||
@@ -221,8 +253,9 @@ Always **`bsk session stop <id>`** in a `finally`-style path so the Agent Window
|
||||
1. **No token theft** — do not `bsk evaluate` on sensitive sites to read `localStorage`, cookies, or auth headers for exfiltration.
|
||||
2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks.
|
||||
3. **No skip stop** — always `bsk session stop <id>`; never assume idle timeout will clean up.
|
||||
4. **No observe escalation before snapshot** — use `bsk snapshot` first; only use `bsk get-html` or `bsk screenshot` when the snapshot is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot ref — never skip snapshot just to grab a visual.
|
||||
5. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
|
||||
4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open.
|
||||
5. **No observe escalation before snapshot** — use `bsk snapshot` first; only use `bsk get-html` or `bsk screenshot` when the snapshot is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot ref — never skip snapshot just to grab a visual.
|
||||
6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
|
||||
|
||||
---
|
||||
|
||||
|
||||
Reference in New Issue
Block a user