diff --git a/apps/extension/src/content/ControlOverlay.tsx b/apps/extension/src/content/ControlOverlay.tsx index 51ed2f2..c6afc5b 100644 --- a/apps/extension/src/content/ControlOverlay.tsx +++ b/apps/extension/src/content/ControlOverlay.tsx @@ -58,6 +58,7 @@ export function ControlOverlay({ />
{ if (automationBypass) return; event.preventDefault(); @@ -80,13 +81,16 @@ export function ControlOverlay({ />
void; +} + +type Props = { + request: RecordRequestData | null; +}; + +export function RecordOverlay({ request }: Props) { + const { t } = useTranslation("extension"); + const [show, setShow] = useState(false); + + useEffect(() => { + if (request) { + const raf = requestAnimationFrame(() => setShow(true)); + return () => cancelAnimationFrame(raf); + } + setShow(false); + }, [request]); + + if (!request) return null; + + return ( + <> + + +
+ + + {t("recordOverlay.recording")} + + +
+ + ); +} diff --git a/apps/extension/src/content/__tests__/ControlOverlay.test.tsx b/apps/extension/src/content/__tests__/ControlOverlay.test.tsx index b7aae0c..d35c609 100644 --- a/apps/extension/src/content/__tests__/ControlOverlay.test.tsx +++ b/apps/extension/src/content/__tests__/ControlOverlay.test.tsx @@ -7,7 +7,7 @@ describe("ControlOverlay", () => { cleanup(); }); - it("sets pointer-events none on blocker and pill when automationBypass is true", () => { + it("keeps page blocker none under automationBypass but Interrupt stays clickable", () => { const { container } = render( { />, ); - const blocker = container.querySelector("[data-slot='control-overlay']")?.nextElementSibling; + const blocker = container.querySelector("[data-slot='control-overlay-blocker']"); expect(blocker).toBeTruthy(); expect((blocker as HTMLElement).style.pointerEvents).toBe("none"); + const pill = container.querySelector("[data-slot='control-overlay-pill']"); + expect(pill).toBeTruthy(); + expect((pill as HTMLElement).style.pointerEvents).toBe("auto"); + const stopBtn = container.querySelector("[data-slot='control-overlay-stop-all']"); expect(stopBtn).toBeTruthy(); - expect((stopBtn as HTMLElement).style.pointerEvents).toBe("none"); + expect((stopBtn as HTMLElement).style.pointerEvents).toBe("auto"); }); it("uses pointer-events auto on blocker when automationBypass is false", () => { @@ -36,7 +40,7 @@ describe("ControlOverlay", () => { />, ); - const blocker = container.querySelector("[data-slot='control-overlay']")?.nextElementSibling; + const blocker = container.querySelector("[data-slot='control-overlay-blocker']"); expect(blocker).toBeTruthy(); expect((blocker as HTMLElement).style.pointerEvents).toBe("auto"); }); diff --git a/apps/extension/src/content/__tests__/RecordOverlay.test.tsx b/apps/extension/src/content/__tests__/RecordOverlay.test.tsx new file mode 100644 index 0000000..9ed5e5a --- /dev/null +++ b/apps/extension/src/content/__tests__/RecordOverlay.test.tsx @@ -0,0 +1,37 @@ +import { cleanup, fireEvent, render } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { RecordOverlay } from "../RecordOverlay"; + +describe("RecordOverlay", () => { + afterEach(() => { + cleanup(); + }); + + it("renders the bottom pill with orange finish CTA and no logo", () => { + const onFinish = vi.fn(); + const { container } = render(); + + const pill = container.querySelector("[data-slot='record-overlay-pill']"); + expect(pill).toBeTruthy(); + expect(pill?.querySelector("img")).toBeNull(); + expect((pill as HTMLElement).style.borderRadius).toBe("9999px"); + expect((pill as HTMLElement).style.backgroundColor).toBe("#fff"); + + const finish = container.querySelector("[data-slot='record-overlay-finish']"); + expect(finish).toBeTruthy(); + expect((finish as HTMLElement).style.backgroundColor).toBe("#f97316"); + expect((finish as HTMLElement).style.borderRadius).toBe("9999px"); + + fireEvent.click(finish!); + expect(onFinish).toHaveBeenCalledTimes(1); + }); + + it("shows a pulsing recording indicator and no full-screen glow layer", () => { + const { container } = render(); + expect(container.querySelector("[data-slot='record-overlay']")).toBeNull(); + + const indicator = container.querySelector("[data-slot='record-overlay-indicator']"); + expect(indicator).toBeTruthy(); + expect((indicator as HTMLElement).style.animation).toContain("bsk-rec-pulse"); + }); +}); diff --git a/apps/extension/src/content/__tests__/overlay-controller.test.ts b/apps/extension/src/content/__tests__/overlay-controller.test.ts index c0b397a..7e8fd93 100644 --- a/apps/extension/src/content/__tests__/overlay-controller.test.ts +++ b/apps/extension/src/content/__tests__/overlay-controller.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it, vi } from "vitest"; -import { OverlayController } from "../overlay-controller"; +import { OverlayController, shouldShowAgentControlOverlay } from "../overlay-controller"; describe("OverlayController", () => { it("resets agent overlays without clearing user-tab borrow requests", () => { @@ -31,6 +31,7 @@ describe("OverlayController", () => { expect(state.controlVisible).toBe(false); expect(state.activeSessionId).toBeNull(); expect(state.activeHelp).toBeNull(); + expect(state.activeRecord).toBeNull(); expect(state.automationBypassCount).toBe(0); }); @@ -69,4 +70,50 @@ describe("OverlayController", () => { expect(previous?.id).toBe("help-1"); expect(controller.snapshot().activeHelp?.id).toBe("help-2"); }); + + it("tracks active record request state", () => { + const controller = new OverlayController(); + + controller.activateAgentSession("sess-1"); + controller.setAgentRecordRequest({ + id: "rec-1", + onFinish: vi.fn(), + }); + + expect(controller.snapshot().activeRecord?.id).toBe("rec-1"); + }); + + it("hides agent control overlay while recording is active", () => { + const controller = new OverlayController(); + controller.activateAgentSession("sess-1"); + expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(true); + + controller.setAgentRecordRequest({ + id: "rec-1", + onFinish: vi.fn(), + }); + expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(false); + expect(controller.snapshot().controlVisible).toBe(true); + }); + + it("keeps control hidden after record ends until session overlays reset", () => { + const controller = new OverlayController(); + controller.activateAgentSession("sess-1"); + controller.setAgentRecordRequest({ + id: "rec-1", + onFinish: vi.fn(), + }); + controller.setAutomationBypass(true); + controller.setAutomationBypass(true); + + controller.clearAgentRecordRequest("rec-1"); + expect(controller.snapshot().activeRecord).toBeNull(); + expect(controller.snapshot().suppressControlAfterRecord).toBe(true); + expect(controller.snapshot().automationBypassCount).toBe(0); + expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(false); + + controller.resetAgentOverlays("sess-1"); + expect(controller.snapshot().suppressControlAfterRecord).toBe(false); + expect(controller.snapshot().controlVisible).toBe(false); + }); }); diff --git a/apps/extension/src/content/__tests__/record-capture.test.ts b/apps/extension/src/content/__tests__/record-capture.test.ts new file mode 100644 index 0000000..0cada90 --- /dev/null +++ b/apps/extension/src/content/__tests__/record-capture.test.ts @@ -0,0 +1,163 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { handleRecordContentMessage, startRecordCapture } from "../record-capture"; +import { RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge"; + +vi.stubGlobal("chrome", { + runtime: { + sendMessage: vi.fn(() => Promise.resolve()), + }, +}); + +describe("handleRecordContentMessage stop/cancel", () => { + it("ignores STOP when no recording is active", () => { + const dispose = vi.fn(); + const onStop = vi.fn(); + const setActiveRequestId = vi.fn(); + const setCapture = vi.fn(); + const sendResponse = vi.fn(); + + const needsAsync = handleRecordContentMessage( + { type: RECORD_STOP, requestId: "rec-stale" }, + { + activeRequestId: null, + capture: { dispose }, + setActiveRequestId, + setCapture, + onStart: vi.fn(), + onStop, + }, + sendResponse, + ); + + expect(needsAsync).toBe(false); + expect(dispose).not.toHaveBeenCalled(); + expect(onStop).not.toHaveBeenCalled(); + expect(setActiveRequestId).not.toHaveBeenCalled(); + expect(setCapture).not.toHaveBeenCalled(); + expect(sendResponse).not.toHaveBeenCalled(); + }); + + it("ignores STOP for a mismatched requestId", () => { + const dispose = vi.fn(); + const onStop = vi.fn(); + + const needsAsync = handleRecordContentMessage( + { type: RECORD_STOP, requestId: "rec-other" }, + { + activeRequestId: "rec-1", + capture: { dispose }, + setActiveRequestId: vi.fn(), + setCapture: vi.fn(), + onStart: vi.fn(), + onStop, + }, + vi.fn(), + ); + + expect(needsAsync).toBe(false); + expect(dispose).not.toHaveBeenCalled(); + expect(onStop).not.toHaveBeenCalled(); + }); +}); + +describe("record-capture semantic", () => { + let steps: RecordStepPayload[]; + + beforeEach(() => { + steps = []; + document.body.innerHTML = ` + + + +
+
建议项
+
+ `; + }); + + it("commits final fill value and records semantic click", () => { + const capture = startRecordCapture("rec-1", (step) => steps.push(step)); + const input = document.querySelector("input")!; + const button = document.querySelector("button")!; + + input.dispatchEvent(new FocusEvent("focusin", { bubbles: true })); + input.value = "hello"; + input.dispatchEvent(new Event("input", { bubbles: true })); + input.dispatchEvent(new FocusEvent("focusout", { bubbles: true })); + + button.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + + capture.dispose(); + + expect(steps).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + op: "fill", + value: "hello", + target: expect.objectContaining({ name: "查询", tag: "input" }), + }), + expect.objectContaining({ + op: "click", + target: expect.objectContaining({ name: "搜索", role: "button" }), + }), + ]), + ); + expect(JSON.stringify(steps)).not.toMatch(/@e\d+/); + expect(steps.some((s) => "selector" in s)).toBe(false); + }); + + it("ignores autocomplete suggestion clicks while a fill session is open", () => { + const capture = startRecordCapture("rec-2", (step) => steps.push(step)); + const input = document.querySelector("input")!; + const option = document.querySelector('[role="option"]')!; + + input.dispatchEvent(new FocusEvent("focusin", { bubbles: true })); + input.value = "hel"; + input.dispatchEvent(new Event("input", { bubbles: true })); + option.dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + input.value = "hello"; + input.dispatchEvent(new Event("input", { bubbles: true })); + input.dispatchEvent(new FocusEvent("focusout", { bubbles: true })); + capture.dispose(); + + expect(steps.filter((s) => s.op === "click")).toHaveLength(0); + expect(steps.some((s) => s.op === "fill" && s.value === "hello")).toBe(true); + }); + + it("records Enter press but not bare typing keys", () => { + const capture = startRecordCapture("rec-press", (step) => steps.push(step)); + const input = document.querySelector("input")!; + input.dispatchEvent(new FocusEvent("focusin", { bubbles: true })); + input.dispatchEvent( + new KeyboardEvent("keydown", { key: "a", bubbles: true }), + ); + input.dispatchEvent( + new KeyboardEvent("keydown", { key: "Enter", bubbles: true }), + ); + capture.dispose(); + expect(steps.filter((s) => s.op === "press")).toEqual([ + expect.objectContaining({ op: "press", key: "Enter" }), + ]); + }); + + it("does not record clicks on anonymous layout divs", () => { + document.body.innerHTML = ` +
page chrome
+ + `; + const capture = startRecordCapture("rec-3", (step) => steps.push(step)); + document + .querySelector("#chrome")! + .dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + document + .querySelector("button")! + .dispatchEvent(new MouseEvent("click", { bubbles: true, button: 0, detail: 1 })); + capture.dispose(); + + expect(steps).toHaveLength(1); + expect(steps[0]).toMatchObject({ + op: "click", + target: { name: "下一步", role: "button" }, + }); + }); +}); diff --git a/apps/extension/src/content/overlay-controller.ts b/apps/extension/src/content/overlay-controller.ts index 15077a6..a0f6560 100644 --- a/apps/extension/src/content/overlay-controller.ts +++ b/apps/extension/src/content/overlay-controller.ts @@ -1,13 +1,21 @@ import type { BorrowRequestData } from "./BorrowConfirmationOverlay"; import type { HelpRequestData } from "./HelpRequestOverlay"; +import type { RecordRequestData } from "./RecordOverlay"; export interface OverlayState { borrowRequests: BorrowRequestData[]; activeHelp: HelpRequestData | null; + activeRecord: RecordRequestData | null; controlVisible: boolean; interrupting: boolean; activeSessionId: string | null; automationBypassCount: number; + /** + * After record Finish/Stop clears `activeRecord`, the session is still + * alive until CLI `session.stop`. Suppress the control mask for that gap + * so 「Agent 正在控制」does not flash between RecordOverlay and teardown. + */ + suppressControlAfterRecord: boolean; } type MutableOverlayState = Omit; @@ -20,9 +28,11 @@ export class OverlayController { private state: MutableOverlayState = { borrowRequests: [], activeHelp: null, + activeRecord: null, interrupting: false, activeSessionId: null, automationBypassCount: 0, + suppressControlAfterRecord: false, }; snapshot(): OverlayState { @@ -70,6 +80,25 @@ export class OverlayController { this.state.activeHelp = null; } + setAgentRecordRequest(request: RecordRequestData): RecordRequestData | null { + const previous = this.state.activeRecord; + this.state.activeRecord = request; + this.state.suppressControlAfterRecord = false; + return previous; + } + + clearAgentRecordRequest(requestId?: string): void { + if (!this.state.activeRecord) return; + if (requestId && this.state.activeRecord.id !== requestId) return; + this.state.activeRecord = null; + // Hide control until session teardown — see OverlayState.suppressControlAfterRecord. + this.state.suppressControlAfterRecord = true; + // Record start/rearm may have stacked automation-bypass refs; drop them so a + // stray ControlOverlay cannot sit with pointer-events:none (page usable, + // Interrupt unclickable). + this.state.automationBypassCount = 0; + } + setAutomationBypass(enabled: boolean): void { if (enabled) { this.state.automationBypassCount += 1; @@ -86,10 +115,22 @@ export class OverlayController { this.state = { ...this.state, activeHelp: null, + activeRecord: null, interrupting: false, activeSessionId: null, automationBypassCount: 0, + suppressControlAfterRecord: false, }; return previousHelp; } } + +/** Control mask ("Agent 正在控制") must hide while help/record overlays own the chrome. */ +export function shouldShowAgentControlOverlay(state: OverlayState): boolean { + return ( + state.controlVisible && + !state.suppressControlAfterRecord && + state.activeHelp === null && + state.activeRecord === null + ); +} diff --git a/apps/extension/src/content/record-capture.ts b/apps/extension/src/content/record-capture.ts new file mode 100644 index 0000000..e6962a1 --- /dev/null +++ b/apps/extension/src/content/record-capture.ts @@ -0,0 +1,541 @@ +import { + describeEventTarget, + describeTarget, + type TargetDescriptor, +} from "@/lib/describe-target"; +import { + isRecordCancelMessage, + isRecordStartMessage, + isRecordStopMessage, + RECORD_CANCEL, + RECORD_START, + RECORD_STEP, + RECORD_STOP, + type RecordCancelMessage, + type RecordStartAck, + type RecordStartMessage, + type RecordStepPayload, + type RecordStopAck, + type RecordStopMessage, +} from "@/lib/record-bridge"; +import { shouldRecordPress } from "@/lib/trace-reducer"; + +const pendingStepSends = new Map>>(); +const failedStepDeliveries = new Set(); +const knownRecordRequests = new Set(); + +function sendRecordStep(requestId: string, step: RecordStepPayload): void { + const payload = { type: RECORD_STEP, requestId, step }; + const pending = Promise.resolve(chrome.runtime.sendMessage(payload)).then( + () => true, + () => { + failedStepDeliveries.add(requestId); + return false; + }, + ); + const sends = pendingStepSends.get(requestId) ?? new Set>(); + sends.add(pending); + pendingStepSends.set(requestId, sends); + void pending.then(() => { + sends.delete(pending); + if (sends.size === 0) pendingStepSends.delete(requestId); + }); +} + +export interface RecordCaptureController { + dispose(): void; +} + +interface FillSession { + element: FillableElement; + target: TargetDescriptor; + baselineValue: string; + lastValue: string; +} + +type FillableElement = HTMLInputElement | HTMLTextAreaElement | HTMLElement; + +function eventTarget(event: Event): EventTarget | null { + return event.composedPath()[0] ?? event.target; +} + +function isOverlayTarget(target: EventTarget | null): boolean { + if (!(target instanceof Node)) return false; + const root = document.documentElement; + let node: Node | null = target; + while (node && node !== root) { + if (node instanceof Element && node.hasAttribute("data-bsk-overlay")) { + return true; + } + const rootNode: Node | Document | ShadowRoot = node.getRootNode(); + if (rootNode instanceof ShadowRoot) { + const host: Element = rootNode.host; + if (host.hasAttribute("data-bsk-overlay")) { + return true; + } + node = host; + } else { + node = node.parentNode; + } + } + return false; +} + +function isTextFillable(el: Element): el is FillableElement { + if (el instanceof HTMLElement && (el.isContentEditable || el.contentEditable === "true")) { + return true; + } + if (el instanceof HTMLTextAreaElement) return true; + if (!(el instanceof HTMLInputElement)) return false; + const type = el.type.toLowerCase(); + return type !== "checkbox" && type !== "radio" && type !== "file" && type !== "button"; +} + +function fillableFromTarget(target: EventTarget | null): FillableElement | null { + if (!(target instanceof Element)) return null; + if (isTextFillable(target)) return target; + const el = target.closest('input,textarea,[contenteditable]:not([contenteditable="false"])'); + if (el && isTextFillable(el)) return el; + return null; +} + +/** Clicks on search chrome that only focus the nearby input should not become steps. */ +function nearbyFillableFromSearchChrome(target: Element): FillableElement | null { + const container = target.closest( + '[id*="chat-input"], [id*="search"], [class*="search"], form, [role="search"]', + ); + if (!container) return null; + const fillable = container.querySelector( + 'textarea, input[type="search"], input[name="q"], #chat-textarea', + ); + if ( + fillable instanceof HTMLElement && + isTextFillable(fillable) && + fillable !== target && + !fillable.contains(target) + ) { + return fillable; + } + return null; +} + +function fillableValue(el: FillableElement): string { + if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) { + return el.value; + } + return el.textContent ?? ""; +} + +/** Clicks that only pick an autocomplete/suggestion value — not a semantic submit action. */ +function isInputCompletionClick(target: EventTarget | null, session: FillSession | null): boolean { + if (!session || !(target instanceof Element)) return false; + if (session.element.contains(target)) return false; + + const option = target.closest('[role="option"]'); + if (option?.closest('[role="listbox"]')) return true; + + const suggestionRoot = target.closest( + [ + '[id*="Sug"]', + '[id*="sug"]', + '[class*="suggest"]', + '[class*="autocomplete"]', + '[class*="typeahead"]', + "[data-autocomplete]", + ].join(", "), + ); + if (suggestionRoot && !suggestionRoot.contains(session.element)) return true; + + return false; +} + +function scheduleInputCompletionCommit( + sessionElement: FillableElement, + syncFillSessionValue: (el: FillableElement) => void, + commitFillSession: () => void, + hasSessionFor: (el: FillableElement) => boolean, +): void { + const syncAndCommit = () => { + if (!hasSessionFor(sessionElement)) return; + syncFillSessionValue(sessionElement); + commitFillSession(); + }; + syncFillSessionValue(sessionElement); + queueMicrotask(syncAndCommit); + setTimeout(syncAndCommit, 0); +} + +export function startRecordCapture( + _requestId: string, + sendStep: (step: RecordStepPayload) => void, +): RecordCaptureController { + const emitStep = (step: RecordStepPayload) => { + sendStep({ page_url: location.href, ...step }); + }; + let fillSession: FillSession | null = null; + let composing = false; + let lastUrl = location.href; + let keyboardActivation: { target: EventTarget | null; recordedAt: number } | undefined; + let generatedControlClick: Element | null = null; + let navigationActionPending = false; + let navigationActionVersion = 0; + const committedValues = new WeakMap(); + + const markNavigationAction = () => { + navigationActionPending = true; + const version = ++navigationActionVersion; + setTimeout(() => { + if (navigationActionVersion === version) { + navigationActionPending = false; + } + }, 0); + }; + + const emitFill = (session: FillSession) => { + if (session.lastValue === session.baselineValue) return; + const isPassword = + session.element instanceof HTMLInputElement && session.element.type === "password"; + const value = isPassword ? "***" : session.lastValue; + emitStep({ + op: "fill", + target: session.target, + value, + ...(isPassword ? { redacted: true } : {}), + }); + }; + + const commitFillSession = () => { + if (!fillSession || composing) return; + const session = fillSession; + fillSession = null; + emitFill(session); + committedValues.set(session.element, session.lastValue); + }; + + const syncFillSessionValue = (el: FillableElement) => { + if (!fillSession || fillSession.element !== el) return; + fillSession.lastValue = fillableValue(el); + }; + + const ensureFillSession = (el: FillableElement) => { + if (fillSession?.element === el) return; + if (fillSession) commitFillSession(); + const currentValue = fillableValue(el); + const baselineValue = committedValues.get(el) ?? currentValue; + fillSession = { + element: el, + target: describeTarget(el), + baselineValue, + lastValue: currentValue, + }; + }; + + const emitNavigateIfChanged = (causedByAction?: boolean) => { + if (location.href === lastUrl) return; + commitFillSession(); + lastUrl = location.href; + emitStep({ + op: "navigate", + url: location.href, + ...(causedByAction !== undefined ? { navigation_caused_by_action: causedByAction } : {}), + }); + }; + + const emitClick = (event: MouseEvent) => { + // Only record clicks an LLM can re-identify (named interactive controls). + const target = describeEventTarget(eventTarget(event)); + if (!target) return; + markNavigationAction(); + emitStep({ + op: "click", + target, + expects_navigation: true, + }); + }; + + const onClick = (event: MouseEvent) => { + if (event.button !== 0) return; + const target = eventTarget(event); + if (isOverlayTarget(target)) return; + if (event.detail === 0 && generatedControlClick !== null && target === generatedControlClick) { + generatedControlClick = null; + return; + } + if ( + event.detail === 0 && + keyboardActivation?.target === target && + Date.now() - keyboardActivation.recordedAt < 500 + ) { + keyboardActivation = undefined; + return; + } + + const label = target instanceof Element ? target.closest("label") : null; + const nestedInteractive = + target instanceof Element ? target.closest("a,button,input,select,textarea") : null; + if (label instanceof HTMLLabelElement && !nestedInteractive) { + commitFillSession(); + generatedControlClick = label.control; + emitClick(event); + return; + } + + const fillable = fillableFromTarget(target); + if (fillable) { + ensureFillSession(fillable); + return; + } + + if (target instanceof Element) { + const nearbyFillable = nearbyFillableFromSearchChrome(target); + if (nearbyFillable) { + ensureFillSession(nearbyFillable); + return; + } + } + + if (isInputCompletionClick(target, fillSession)) { + markNavigationAction(); + const sessionElement = fillSession!.element; + scheduleInputCompletionCommit( + sessionElement, + syncFillSessionValue, + commitFillSession, + (el) => fillSession?.element === el, + ); + return; + } + + commitFillSession(); + if (target instanceof Element && target.closest("select")) return; + emitClick(event); + }; + + const onFocusIn = (event: FocusEvent) => { + const target = fillableFromTarget(eventTarget(event)); + if (target) ensureFillSession(target); + }; + + const onFocusOut = (event: FocusEvent) => { + const target = fillableFromTarget(eventTarget(event)); + if (!target) return; + if (!fillSession || fillSession.element !== target) return; + syncFillSessionValue(target); + commitFillSession(); + }; + + const onInput = (event: Event) => { + const target = fillableFromTarget(eventTarget(event)); + if (!target) return; + if (composing) return; + ensureFillSession(target); + syncFillSessionValue(target); + }; + + const onCompositionStart = () => { + composing = true; + }; + + const onCompositionEnd = (event: CompositionEvent) => { + composing = false; + const target = fillableFromTarget(eventTarget(event)); + if (target) { + ensureFillSession(target); + syncFillSessionValue(target); + } + }; + + const onChange = (event: Event) => { + commitFillSession(); + const target = eventTarget(event); + if (target instanceof HTMLSelectElement) { + const values = Array.from(target.selectedOptions).map((opt) => opt.value); + const labels = Array.from(target.selectedOptions).map((opt) => + (opt.label || opt.textContent || opt.value).trim(), + ); + const desc = describeTarget(target); + markNavigationAction(); + emitStep({ + op: "select", + target: desc, + values, + labels, + expects_navigation: true, + }); + } + }; + + const onKeyDown = (event: KeyboardEvent) => { + if (isOverlayTarget(eventTarget(event))) return; + const target = eventTarget(event); + const fillable = fillableFromTarget(target); + if (fillable) { + ensureFillSession(fillable); + syncFillSessionValue(fillable); + } + const modifiers = [ + ...(event.altKey ? (["alt"] as const) : []), + ...(event.ctrlKey ? (["ctrl"] as const) : []), + ...(event.metaKey ? (["meta"] as const) : []), + ...(event.shiftKey ? (["shift"] as const) : []), + ]; + if (!shouldRecordPress(event.key, modifiers)) { + return; + } + markNavigationAction(); + if (event.key === "Enter" || event.key === " ") { + const submitTarget = + event.key === "Enter" && fillable + ? fillable + .closest("form") + ?.querySelector( + 'button:not([type]),button[type="submit"],input[type="submit"],input[type="image"]', + ) + : null; + keyboardActivation = { + target: submitTarget ?? target, + recordedAt: Date.now(), + }; + } + if (fillable) { + commitFillSession(); + } + const desc = describeEventTarget(target); + if (!desc && !event.key) return; + emitStep({ + op: "press", + key: event.key, + ...(desc ? { target: desc } : {}), + ...(modifiers.length ? { modifiers } : {}), + expects_navigation: event.key === "Enter", + }); + }; + + document.addEventListener("click", onClick, true); + document.addEventListener("focusin", onFocusIn, true); + document.addEventListener("focusout", onFocusOut, true); + document.addEventListener("input", onInput, true); + document.addEventListener("compositionstart", onCompositionStart, true); + document.addEventListener("compositionend", onCompositionEnd, true); + document.addEventListener("change", onChange, true); + document.addEventListener("keydown", onKeyDown, true); + + const urlObserver = new MutationObserver(() => emitNavigateIfChanged()); + urlObserver.observe(document, { subtree: true, childList: true }); + const onUrlEvent = () => emitNavigateIfChanged(); + window.addEventListener("hashchange", onUrlEvent); + window.addEventListener("popstate", onUrlEvent); + window.addEventListener("pagehide", commitFillSession); + + const originalPushState = history.pushState; + const originalReplaceState = history.replaceState; + history.pushState = function (...args: Parameters) { + originalPushState.apply(this, args); + emitNavigateIfChanged(navigationActionPending ? true : undefined); + }; + history.replaceState = function (...args: Parameters) { + originalReplaceState.apply(this, args); + emitNavigateIfChanged(navigationActionPending ? true : undefined); + }; + + return { + dispose() { + commitFillSession(); + document.removeEventListener("click", onClick, true); + document.removeEventListener("focusin", onFocusIn, true); + document.removeEventListener("focusout", onFocusOut, true); + document.removeEventListener("input", onInput, true); + document.removeEventListener("compositionstart", onCompositionStart, true); + document.removeEventListener("compositionend", onCompositionEnd, true); + document.removeEventListener("change", onChange, true); + document.removeEventListener("keydown", onKeyDown, true); + urlObserver.disconnect(); + window.removeEventListener("hashchange", onUrlEvent); + window.removeEventListener("popstate", onUrlEvent); + window.removeEventListener("pagehide", commitFillSession); + history.pushState = originalPushState; + history.replaceState = originalReplaceState; + }, + }; +} + +export type RecordContentMessage = RecordStartMessage | RecordStopMessage | RecordCancelMessage; + +export function isRecordContentMessage(msg: unknown): msg is RecordContentMessage { + return isRecordStartMessage(msg) || isRecordStopMessage(msg) || isRecordCancelMessage(msg); +} + +export function handleRecordContentMessage( + message: RecordContentMessage, + state: { + activeRequestId: string | null; + capture: RecordCaptureController | null; + setActiveRequestId(id: string | null): void; + setCapture(capture: RecordCaptureController | null): void; + onStart(requestId: string): void; + onStop(): void; + }, + sendResponse?: (response: RecordStartAck | RecordStopAck) => void, +): boolean { + if (isRecordStartMessage(message)) { + if (!knownRecordRequests.has(message.requestId)) { + knownRecordRequests.add(message.requestId); + failedStepDeliveries.delete(message.requestId); + } + state.capture?.dispose(); + state.setCapture( + startRecordCapture(message.requestId, (step) => { + sendRecordStep(message.requestId, step); + }), + ); + state.setActiveRequestId(message.requestId); + state.onStart(message.requestId); + sendResponse?.({ ok: true }); + return sendResponse !== undefined; + } + + if (isRecordStopMessage(message) || isRecordCancelMessage(message)) { + // Require an active recording that matches this requestId — otherwise a + // stray STOP/CANCEL (e.g. after teardown) would still run finishStop and + // clear overlay state even though nothing was capturing. + if (!state.activeRequestId || state.activeRequestId !== message.requestId) { + return false; + } + state.capture?.dispose(); + state.setCapture(null); + const finishStop = () => { + state.onStop(); + state.setActiveRequestId(null); + }; + if (isRecordStopMessage(message) && sendResponse) { + const pending = [...(pendingStepSends.get(message.requestId) ?? [])]; + void Promise.all(pending).then((delivered) => { + finishStop(); + const succeeded = delivered.every(Boolean) && !failedStepDeliveries.has(message.requestId); + if (succeeded) { + failedStepDeliveries.delete(message.requestId); + knownRecordRequests.delete(message.requestId); + } + sendResponse( + succeeded + ? { ok: true } + : { + ok: false, + error: "failed to deliver one or more recorded steps", + }, + ); + }); + return true; + } + if (isRecordCancelMessage(message)) { + failedStepDeliveries.delete(message.requestId); + knownRecordRequests.delete(message.requestId); + } + finishStop(); + return false; + } + + return false; +} + +export { RECORD_CANCEL, RECORD_START, RECORD_STEP, RECORD_STOP }; diff --git a/apps/extension/src/entrypoints/background.ts b/apps/extension/src/entrypoints/background.ts index c1bafb5..26fd7b0 100644 --- a/apps/extension/src/entrypoints/background.ts +++ b/apps/extension/src/entrypoints/background.ts @@ -8,6 +8,7 @@ import { } from "@/lib/instance-id"; import { startKeepalive } from "@/lib/keepalive"; import { + OVERLAY_AUTOMATION_BYPASS, OVERLAY_MSG_INTERRUPT, OVERLAY_MSG_WHO_AM_I, type OverlayInterruptRequest, @@ -27,6 +28,11 @@ import { requestBorrowConfirmation, } from "@/tools/borrow-confirmation"; import { ToolDispatcher } from "@/tools/dispatcher"; +import { + attachRecordFinishListener, + attachRecordQueryListener, + attachRecordStepListener, +} from "@/tools/record"; import { detectBrowserMeta } from "@/transport/handshake"; import type { Transport } from "@/transport/transport"; import { WSTransport } from "@/transport/ws-transport"; @@ -67,6 +73,27 @@ export default defineBackground(() => { }), }); dispatcher.start(); + const recordDeps = { + tabsApi: chrome.tabs, + sendToTab: (tabId: number, msg: Parameters[1]) => + chrome.tabs.sendMessage(tabId, msg), + bypassOverlay: async (tabId: number, enabled: boolean) => { + try { + await chrome.tabs.sendMessage(tabId, { + type: OVERLAY_AUTOMATION_BYPASS, + enabled, + }); + } catch { + // Content script may be unavailable on restricted pages. + } + }, + }; + // Message listeners stay up (cheap; fire only for record message types). + // Tab / webNavigation observation attaches lazily while a recording is + // active — see ensureBrowserObservationListeners in tools/record.ts. + attachRecordStepListener(); + attachRecordFinishListener(recordDeps); + attachRecordQueryListener(recordDeps); if (typeof chrome.notifications?.onClicked?.addListener === "function") { attachBorrowNotificationClickHandler({ onClicked: chrome.notifications.onClicked, diff --git a/apps/extension/src/entrypoints/content.ts b/apps/extension/src/entrypoints/content.ts index b9e9dee..d721d70 100644 --- a/apps/extension/src/entrypoints/content.ts +++ b/apps/extension/src/entrypoints/content.ts @@ -6,7 +6,13 @@ import { BorrowConfirmationOverlay } from "@/content/BorrowConfirmationOverlay"; import { ControlOverlay } from "@/content/ControlOverlay"; import { HelpRequestOverlay } from "@/content/HelpRequestOverlay"; import overlayCss from "@/content/overlay.css?inline"; -import { OverlayController } from "@/content/overlay-controller"; +import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller"; +import { RecordOverlay } from "@/content/RecordOverlay"; +import { + handleRecordContentMessage, + isRecordContentMessage, + type RecordCaptureController, +} from "@/content/record-capture"; import { HELP_RESPONSE, type HelpCancelMessage, @@ -24,6 +30,12 @@ import { type OverlayWhoAmIResponse, } from "@/lib/overlay-bridge"; import { sendInterrupt } from "@/lib/overlay-interrupt-client"; +import { + RECORD_FINISH, + RECORD_QUERY, + type RecordStartAck, + type RecordStopAck, +} from "@/lib/record-bridge"; import { SESSIONS_LIVE_FLAG_KEY } from "@/lib/sessions-live-flag"; import type { BorrowCancelMessage, @@ -45,6 +57,8 @@ export default defineContentScript({ const overlays = new OverlayController(); let activeHelpRespond: ((outcome: "continued" | "cancelled", note?: string) => void) | null = null; + let recordCapture: RecordCaptureController | null = null; + let activeRecordRequestId: string | null = null; let reactRoot: ReactDOM.Root | null = null; let overlayHost: HTMLElement | null = null; let hostLossReported = false; @@ -87,12 +101,13 @@ export default defineContentScript({ requests: overlayState.borrowRequests, }), React.createElement(ControlOverlay, { - visible: overlayState.controlVisible && overlayState.activeHelp === null, + visible: shouldShowAgentControlOverlay(overlayState), interrupting: overlayState.interrupting, automationBypass: overlayState.automationBypassCount > 0, onInterrupt: handleInterrupt, }), React.createElement(HelpRequestOverlay, { request: overlayState.activeHelp }), + React.createElement(RecordOverlay, { request: overlayState.activeRecord }), ), ), ); @@ -104,6 +119,9 @@ export default defineContentScript({ activeHelpRespond?.("cancelled"); activeHelpRespond = null; } + recordCapture?.dispose(); + recordCapture = null; + activeRecordRequestId = null; renderOverlay(); } @@ -140,6 +158,42 @@ export default defineContentScript({ _sender: chrome.runtime.MessageSender, sendResponse: (response: BorrowResponseMessage | HelpResponseMessage) => void, ) => { + if (isRecordContentMessage(message)) { + const needsAsync = handleRecordContentMessage( + message, + { + activeRequestId: activeRecordRequestId, + capture: recordCapture, + setActiveRequestId: (id) => { + activeRecordRequestId = id; + }, + setCapture: (capture) => { + recordCapture = capture; + }, + onStart: (requestId) => { + overlays.setAgentRecordRequest({ + id: requestId, + onFinish: () => { + void chrome.runtime.sendMessage({ + type: RECORD_FINISH, + requestId, + }); + }, + }); + renderOverlay(); + }, + onStop: () => { + overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined); + renderOverlay(); + }, + }, + sendResponse as unknown as + | ((response: RecordStartAck | RecordStopAck) => void) + | undefined, + ); + return needsAsync; + } + if ( message && typeof message === "object" && @@ -228,23 +282,59 @@ export default defineContentScript({ return false; }; - async function mountOverlayIfAgent(): Promise { + async function syncAgentOverlay(): Promise { if (!(await anySessionLive())) return; try { const reply = (await chrome.runtime.sendMessage({ kind: OVERLAY_MSG_WHO_AM_I, })) as OverlayWhoAmIResponse | undefined; if (!reply?.sessionId) return; + + // Query / re-arm recording *before* activating the control overlay so + // record start (navigate → content load) does not flash + // 「Agent 正在控制」before RecordOverlay mounts. + let recordQuery: { active?: boolean; requestId?: string } | undefined; + try { + recordQuery = (await chrome.runtime.sendMessage({ + type: RECORD_QUERY, + })) as { active?: boolean; requestId?: string } | undefined; + } catch (err) { + console.debug("[bsk overlay] record query failed", err); + } + + if ( + recordQuery?.active && + typeof recordQuery.requestId === "string" && + overlays.snapshot().activeRecord === null + ) { + const requestId = recordQuery.requestId; + overlays.setAgentRecordRequest({ + id: requestId, + onFinish: () => { + void chrome.runtime.sendMessage({ + type: RECORD_FINISH, + requestId, + }); + }, + }); + } + overlays.activateAgentSession(reply.sessionId); renderOverlay(); } catch (err) { - console.debug("[bsk overlay] who_am_i failed", err); + console.debug("[bsk overlay] syncAgentOverlay failed", err); } } + const onPageShow = (event: PageTransitionEvent) => { + if (event.persisted) void syncAgentOverlay(); + }; + ui.mount(); chrome.runtime.onMessage.addListener(onMessage); - void mountOverlayIfAgent(); + void syncAgentOverlay(); + + window.addEventListener("pageshow", onPageShow); const hostObserver = new MutationObserver(() => { const connected = overlayHost?.isConnected ?? false; @@ -254,6 +344,7 @@ export default defineContentScript({ remountInProgress = true; try { ui.mount(); + void syncAgentOverlay(); } finally { remountInProgress = false; } @@ -268,6 +359,11 @@ export default defineContentScript({ ctx.onInvalidated(() => { hostObserver.disconnect(); chrome.runtime.onMessage.removeListener(onMessage); + window.removeEventListener("pageshow", onPageShow); + // Restore history hooks / remove capture listeners before the CS unloads. + recordCapture?.dispose(); + recordCapture = null; + activeRecordRequestId = null; }); }, }); diff --git a/apps/extension/src/lib/__tests__/describe-target.test.ts b/apps/extension/src/lib/__tests__/describe-target.test.ts new file mode 100644 index 0000000..d192803 --- /dev/null +++ b/apps/extension/src/lib/__tests__/describe-target.test.ts @@ -0,0 +1,117 @@ +import { beforeEach, describe, expect, it } from "vitest"; +import { + describeEventTarget, + describeTarget, + isMeaningfulClickTarget, +} from "../describe-target"; + +describe("describeTarget", () => { + beforeEach(() => { + document.body.innerHTML = ""; + }); + + it("prefers aria-label and button role", () => { + document.body.innerHTML = ``; + const el = document.querySelector("button")!; + expect(describeTarget(el)).toEqual({ + tag: "button", + role: "button", + name: "发布", + }); + }); + + it("uses label text and name_attr for inputs", () => { + document.body.innerHTML = ` + + + `; + const el = document.querySelector("input")!; + expect(describeTarget(el)).toEqual({ + tag: "input", + role: "textbox", + name: "服务名称", + name_attr: "serviceName", + }); + }); + + it("never invents CSS selectors or @eN", () => { + document.body.innerHTML = `详情`; + const desc = describeTarget(document.querySelector("a")!); + expect(JSON.stringify(desc)).not.toMatch(/@e\d+|nth-of-type|#/); + expect(desc).toEqual({ tag: "a", role: "link", name: "详情" }); + }); + + it("prefers heading text inside SERP-style links", () => { + document.body.innerHTML = ` + +

Tencent 腾讯

+ https://www.tencent.com + + `; + expect(describeTarget(document.querySelector("a")!).name).toBe("Tencent 腾讯"); + }); +}); + +describe("LLM textbook click targets", () => { + beforeEach(() => { + document.body.innerHTML = ""; + }); + + it("rejects tag-only targets the LLM cannot find", () => { + expect(isMeaningfulClickTarget({ tag: "div" })).toBe(false); + }); + + it("treats a short visible name as the teaching signal", () => { + expect(isMeaningfulClickTarget({ tag: "div", name: "发布" })).toBe(true); + }); + + it("ignores anonymous layout chrome and lifts child clicks to the named button", () => { + document.body.innerHTML = ` +
background
+ + `; + expect(describeEventTarget(document.querySelector("#noise"))).toBeNull(); + expect(describeEventTarget(document.querySelector("#icon"))).toEqual({ + tag: "button", + role: "button", + name: "发布", + }); + }); + + it("does not attach layout sibling text as nearby_label on buttons", () => { + document.body.innerHTML = ` +
background
+ + `; + expect(describeTarget(document.querySelector("#pub")!)).toEqual({ + tag: "button", + role: "button", + name: "发布", + }); + }); + + it("keeps nearby_label for form fields from a preceding label-like sibling", () => { + document.body.innerHTML = ` +
服务名称
+ + `; + expect(describeTarget(document.querySelector("#svc")!)).toEqual( + expect.objectContaining({ + tag: "input", + nearby_label: "服务名称", + }), + ); + }); + + it("does not treat plain text blocks as clicks", () => { + document.body.innerHTML = `

说明文字

`; + expect(describeEventTarget(document.querySelector("#copy"))).toBeNull(); + }); + + it("records custom button-like controls by their visible label", () => { + document.body.innerHTML = `
发布
`; + expect(describeEventTarget(document.querySelector("#pub"))).toEqual( + expect.objectContaining({ tag: "div", name: "发布" }), + ); + }); +}); diff --git a/apps/extension/src/lib/__tests__/recording-step-buffer.test.ts b/apps/extension/src/lib/__tests__/recording-step-buffer.test.ts new file mode 100644 index 0000000..396253a --- /dev/null +++ b/apps/extension/src/lib/__tests__/recording-step-buffer.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "vitest"; +import { appendRecordedPayload, observeRecordedNavigation } from "../recording-step-buffer"; + +describe("recording-step-buffer", () => { + it("stores semantic click without summary", () => { + const buffer = { steps: [], pendingNavigation: false }; + appendRecordedPayload(buffer, { + op: "click", + target: { tag: "button", role: "button", name: "发布" }, + expects_navigation: true, + }); + expect(buffer.steps).toEqual([ + { + op: "click", + target: { tag: "button", role: "button", name: "发布" }, + }, + ]); + expect(buffer.pendingNavigation).toBe(true); + }); + + it("annotates navigated_to on action-caused navigation instead of wait_for_navigation", () => { + const buffer = { + steps: [ + { + op: "click" as const, + target: { tag: "button", role: "button", name: "发布" }, + }, + ], + currentUrl: "https://example.com/a", + pendingNavigation: true, + pendingNavigationDeadline: Date.now() + 5_000, + }; + observeRecordedNavigation(buffer, "https://example.com/b", true); + expect(buffer.steps).toEqual([ + { + op: "click", + target: { tag: "button", role: "button", name: "发布" }, + navigated_to: "https://example.com/b", + }, + ]); + expect(JSON.stringify(buffer.steps)).not.toContain("wait_for_navigation"); + }); + + it("annotates navigated_to onto a select that auto-submits a navigation", () => { + const buffer = { + steps: [ + { + op: "select" as const, + target: { tag: "select", role: "combobox", name: "分类" }, + values: ["tech"], + }, + ], + currentUrl: "https://example.com/list", + pendingNavigation: true, + pendingNavigationDeadline: Date.now() + 5_000, + }; + observeRecordedNavigation(buffer, "https://example.com/list?cat=tech", true); + expect(buffer.steps).toEqual([ + { + op: "select", + target: { tag: "select", role: "combobox", name: "分类" }, + values: ["tech"], + navigated_to: "https://example.com/list?cat=tech", + }, + ]); + }); + + it("emits navigate for uncaused URL changes", () => { + const buffer = { + steps: [], + currentUrl: "https://example.com/a", + pendingNavigation: false, + }; + observeRecordedNavigation(buffer, "https://example.com/b", false); + expect(buffer.steps[0]).toMatchObject({ + op: "navigate", + url: "https://example.com/b", + }); + }); +}); diff --git a/apps/extension/src/lib/__tests__/trace-reducer.test.ts b/apps/extension/src/lib/__tests__/trace-reducer.test.ts new file mode 100644 index 0000000..777a86c --- /dev/null +++ b/apps/extension/src/lib/__tests__/trace-reducer.test.ts @@ -0,0 +1,141 @@ +import { describe, expect, it } from "vitest"; +import type { DraftTraceStep } from "@/transport/types"; +import { reduceTraceSteps, resolveTraceStartUrl, shouldRecordPress } from "../trace-reducer"; + +describe("shouldRecordPress", () => { + it("keeps Enter and Escape", () => { + expect(shouldRecordPress("Enter")).toBe(true); + expect(shouldRecordPress("Escape")).toBe(true); + }); + + it("drops modifiers, clipboard shortcuts, and bare typing", () => { + expect(shouldRecordPress("Meta")).toBe(false); + expect(shouldRecordPress("c", ["meta"])).toBe(false); + expect(shouldRecordPress("a", ["ctrl"])).toBe(false); + expect(shouldRecordPress("x")).toBe(false); + expect(shouldRecordPress("中")).toBe(false); + }); +}); + +describe("reduceTraceSteps", () => { + it("builds steps with pages dictionary and page id references", () => { + const drafts: DraftTraceStep[] = [ + { + op: "navigate", + url: "https://example.com/search?q=hello&utm_source=x", + page_url: "https://example.com/search?q=hello&utm_source=x", + }, + { + op: "fill", + target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" }, + value: "browser skill", + page_url: "https://example.com/search?q=hello&utm_source=x", + }, + { + op: "press", + key: "Enter", + target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" }, + navigated_to: "https://example.com/results/42", + page_url: "https://example.com/search?q=hello&utm_source=x", + }, + { + op: "click", + target: { tag: "button", role: "button", name: "发布" }, + navigated_to: "https://example.com/p/99", + page_url: "https://example.com/results/42", + }, + { + op: "press", + key: "a", + page_url: "https://example.com/p/99", + }, + ]; + + const { pages, steps } = reduceTraceSteps( + drafts, + "https://example.com/search?q=hello&utm_source=x", + ); + expect(JSON.stringify(steps)).not.toContain("parameters"); + expect(JSON.stringify(steps)).not.toContain("intent"); + expect(JSON.stringify(steps)).not.toContain("summary"); + expect(steps.map((s) => s.op)).toEqual(["navigate", "fill", "press", "click"]); + expect(pages.map((p) => p.url)).toEqual([ + "https://example.com/search?q=hello&utm_source=x", + "https://example.com/results/42", + "https://example.com/p/99", + ]); + + expect(steps[0]).toMatchObject({ + id: 1, + op: "navigate", + page: "p1", + to: "https://example.com/search?q=hello&utm_source=x", + }); + + expect(steps[1]).toMatchObject({ + op: "fill", + page: "p1", + value: "browser skill", + }); + + expect(steps[2]).toMatchObject({ + op: "press", + key: "Enter", + page: "p1", + effect: { navigated_to: "p2" }, + }); + + expect(steps[3]).toMatchObject({ + op: "click", + page: "p2", + effect: { navigated_to: "p3" }, + }); + }); + + it("collapses consecutive navigations", () => { + const { steps } = reduceTraceSteps([ + { op: "navigate", url: "https://a.example/redirect1" }, + { op: "navigate", url: "https://a.example/final" }, + ]); + expect(steps).toHaveLength(1); + expect(steps[0]).toMatchObject({ + op: "navigate", + to: "https://a.example/final", + }); + }); + + it("maps select navigated_to onto effect.navigated_to (page id)", () => { + const { pages, steps } = reduceTraceSteps( + [ + { + op: "select", + target: { tag: "select", role: "combobox", name: "分类" }, + values: ["tech"], + labels: ["技术"], + navigated_to: "https://example.com/list?cat=tech", + page_url: "https://example.com/list", + }, + ], + "https://example.com/list", + ); + expect(pages.map((p) => p.url)).toEqual([ + "https://example.com/list", + "https://example.com/list?cat=tech", + ]); + expect(steps[0]).toMatchObject({ + op: "select", + page: "p1", + selection: [{ value: "tech", label: "技术" }], + effect: { navigated_to: "p2" }, + }); + }); + + it("resolveTraceStartUrl prefers explicit start URL", () => { + expect( + resolveTraceStartUrl( + [{ op: "navigate", url: "https://example.com/other" }], + "https://example.com/start", + ), + ).toBe("https://example.com/start"); + }); +}); diff --git a/apps/extension/src/lib/describe-target.ts b/apps/extension/src/lib/describe-target.ts new file mode 100644 index 0000000..f447a72 --- /dev/null +++ b/apps/extension/src/lib/describe-target.ts @@ -0,0 +1,342 @@ +/** + * Build a semantic TargetDescriptor for an interacted element. + * + * Trace steps are an LLM *textbook*: each click must say what to look for + * on screen (usually a short visible name). Tag-only noise like + * `{ "tag": "div" }` / “点击div” is useless and must not be recorded. + */ + +export interface TargetDescriptor { + role?: string; + name?: string; + tag: string; + name_attr?: string; + placeholder?: string; + nearby_label?: string; +} + +/** Max length for a label that is still a useful “find this on screen” hint. */ +const ACTIONABLE_LABEL_MAX = 48; + +function normalizeWhitespace(value: string): string { + return value.replace(/\s+/g, " ").trim(); +} + +function truncate(value: string, max = 80): string { + const trimmed = normalizeWhitespace(value); + if (trimmed.length <= max) return trimmed; + return `${trimmed.slice(0, max - 1)}…`; +} + +/** Short, human label an LLM can search for in a snapshot. */ +export function isActionableLabel(name: string): boolean { + const trimmed = normalizeWhitespace(name); + if (!trimmed) return false; + if (trimmed.length > ACTIONABLE_LABEL_MAX) return false; + if (/^https?:\/\//i.test(trimmed)) return false; + return true; +} + +function labelledByText(el: Element): string | null { + const ids = el.getAttribute("aria-labelledby"); + if (!ids) return null; + const parts = ids + .split(/\s+/) + .map((id) => document.getElementById(id)?.textContent ?? "") + .map(normalizeWhitespace) + .filter(Boolean); + return parts.length ? truncate(parts.join(" ")) : null; +} + +function associatedLabelText(el: Element): string | null { + if (!(el instanceof HTMLElement) || !el.id) return null; + const label = document.querySelector(`label[for="${CSS.escape(el.id)}"]`); + if (!label) return null; + const text = normalizeWhitespace(label.textContent ?? ""); + return text ? truncate(text) : null; +} + +function wrappingLabelText(el: Element): string | null { + const label = el.closest("label"); + if (!label) return null; + const clone = label.cloneNode(true) as HTMLElement; + for (const control of clone.querySelectorAll("input,textarea,select,button")) { + control.remove(); + } + const text = normalizeWhitespace(clone.textContent ?? ""); + return text ? truncate(text) : null; +} + +function placeholderOrTitle(el: Element): string | null { + if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) { + const ph = normalizeWhitespace(el.placeholder); + if (ph) return truncate(ph); + } + const title = normalizeWhitespace(el.getAttribute("title") ?? ""); + return title ? truncate(title) : null; +} + +function ownText(el: Element): string | null { + if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) { + return null; + } + const text = normalizeWhitespace(el.textContent ?? ""); + return text ? truncate(text) : null; +} + +/** Prefer a compact title inside links (e.g. SERP result `

`), not the whole blob. */ +function compactLinkName(el: Element): string | null { + const heading = el.querySelector("h1, h2, h3, h4"); + if (heading) { + const text = normalizeWhitespace(heading.textContent ?? ""); + if (text) return truncate(text); + } + const img = el.querySelector("img[alt]"); + if (img instanceof HTMLImageElement) { + const alt = normalizeWhitespace(img.alt); + if (alt) return truncate(alt); + } + const clone = el.cloneNode(true) as HTMLElement; + for (const junk of clone.querySelectorAll("cite, script, style, svg, noscript")) { + junk.remove(); + } + const text = normalizeWhitespace(clone.textContent ?? ""); + if (!text) return null; + const withoutUrl = text.replace(/https?:\/\/\S+/g, "").trim(); + return truncate(withoutUrl || text); +} + +/** Visible accessible name preference order for LLM textbooks. */ +export function accessibleName(el: Element): string | undefined { + const ariaLabel = normalizeWhitespace(el.getAttribute("aria-label") ?? ""); + if (ariaLabel) return truncate(ariaLabel); + + const labelledBy = labelledByText(el); + if (labelledBy) return labelledBy; + + const forLabel = associatedLabelText(el); + if (forLabel) return forLabel; + + const wrapLabel = wrappingLabelText(el); + if (wrapLabel) return wrapLabel; + + const alt = + el instanceof HTMLImageElement + ? normalizeWhitespace(el.alt) + : normalizeWhitespace(el.getAttribute("alt") ?? ""); + if (alt) return truncate(alt); + + const ph = placeholderOrTitle(el); + if (ph) return ph; + + const tag = el.tagName.toLowerCase(); + if (tag === "a" || el.getAttribute("role") === "link") { + const linkName = compactLinkName(el); + if (linkName) return linkName; + } + + const text = ownText(el); + if (text) return text; + + if (el instanceof HTMLInputElement || el instanceof HTMLButtonElement) { + const value = normalizeWhitespace(el.value); + if (value && (el instanceof HTMLButtonElement || el.type === "submit" || el.type === "button")) { + return truncate(value); + } + } + + return undefined; +} + +const INTERACTIVE_SELECTOR = [ + "a[href]", + "button", + "summary", + "select", + 'input:not([type="hidden"])', + "textarea", + '[role="button"]', + '[role="link"]', + '[role="tab"]', + '[role="menuitem"]', + '[role="menuitemcheckbox"]', + '[role="menuitemradio"]', + '[role="option"]', + '[role="checkbox"]', + '[role="radio"]', + '[role="switch"]', + '[role="treeitem"]', + '[role="combobox"]', + '[contenteditable]:not([contenteditable="false"])', +].join(","); + +const INTERACTIVE_ROLES = new Set([ + "button", + "link", + "tab", + "menuitem", + "menuitemcheckbox", + "menuitemradio", + "option", + "checkbox", + "radio", + "switch", + "treeitem", + "combobox", + "textbox", + "searchbox", + "slider", +]); + +function looksClickable(el: Element): boolean { + if (!(el instanceof HTMLElement)) return false; + if (el.matches(INTERACTIVE_SELECTOR)) return true; + const role = normalizeWhitespace(el.getAttribute("role") ?? "").toLowerCase(); + if (role && INTERACTIVE_ROLES.has(role)) return true; + if (el.tabIndex >= 0) return true; + try { + if (getComputedStyle(el).cursor === "pointer") return true; + } catch { + // happy-dom / detached nodes + } + const cls = typeof el.className === "string" ? el.className : ""; + if (/\b(btn|button|link|clickable|action)\b/i.test(cls)) return true; + return false; +} + +/** + * Resolve the element a click should describe for the LLM textbook. + * Never falls back to an anonymous layout `div`. + */ +export function resolveClickableElement(target: Element): Element | null { + const interactive = target.closest(INTERACTIVE_SELECTOR); + if (interactive instanceof Element) return interactive; + + let node: Element | null = target; + let depth = 0; + while (node && node !== document.body && node !== document.documentElement && depth < 8) { + const name = accessibleName(node); + if (name && isActionableLabel(name) && looksClickable(node)) { + return node; + } + node = node.parentElement; + depth += 1; + } + return null; +} + +/** + * Teachable clicks: the LLM must get a label it can later find via snapshot + * (visible name), or at least a form `name_attr` for checkbox/radio. + * Recording “点击div” with no name fails this bar. + */ +export function isMeaningfulClickTarget(target: TargetDescriptor): boolean { + const name = target.name?.trim(); + if (name && isActionableLabel(name)) return true; + if ( + (target.role === "checkbox" || target.role === "radio" || target.tag === "input") && + target.name_attr + ) { + return true; + } + return false; +} + +export function inferRole(el: Element): string | undefined { + const explicit = normalizeWhitespace(el.getAttribute("role") ?? "").toLowerCase(); + if (explicit) return explicit; + + const tag = el.tagName.toLowerCase(); + if (tag === "button") return "button"; + if (tag === "a" && el.hasAttribute("href")) return "link"; + if (tag === "select") return "combobox"; + if (tag === "textarea") return "textbox"; + if (tag === "img") return "img"; + if (tag === "summary") return "button"; + if (el instanceof HTMLElement && (el.isContentEditable || el.contentEditable === "true")) { + return "textbox"; + } + if (el instanceof HTMLInputElement) { + const type = el.type.toLowerCase(); + if (type === "checkbox") return "checkbox"; + if (type === "radio") return "radio"; + if (type === "submit" || type === "button" || type === "image" || type === "reset") { + return "button"; + } + if (type === "range") return "slider"; + return "textbox"; + } + return undefined; +} + +/** + * Nearby labels help the LLM name form fields. Only apply to fillable + * controls — never to buttons/links, where a previous layout sibling + * (e.g. a chrome `
`) would pollute the textbook target. + */ +function nearbyLabelText(el: Element): string | undefined { + if ( + !( + el instanceof HTMLInputElement || + el instanceof HTMLTextAreaElement || + el instanceof HTMLSelectElement + ) + ) { + return undefined; + } + if (el.id) { + const byFor = document.querySelector(`label[for="${CSS.escape(el.id)}"]`); + const text = byFor ? normalizeWhitespace(byFor.textContent ?? "") : ""; + if (text) return truncate(text, ACTIONABLE_LABEL_MAX); + } + const wrapped = el.closest("label"); + if (wrapped) { + const clone = wrapped.cloneNode(true) as HTMLLabelElement; + for (const control of clone.querySelectorAll("input, textarea, select, button")) { + control.remove(); + } + const text = normalizeWhitespace(clone.textContent ?? ""); + if (text) return truncate(text, ACTIONABLE_LABEL_MAX); + } + const prev = el.previousElementSibling; + if (prev && /^(LABEL|SPAN|DIV|P|DT)$/i.test(prev.tagName)) { + const text = normalizeWhitespace(prev.textContent ?? ""); + if (text && text.length <= ACTIONABLE_LABEL_MAX) return text; + } + return undefined; +} + +export function describeTarget(el: Element): TargetDescriptor { + const tag = el.tagName.toLowerCase(); + const role = inferRole(el); + const name = accessibleName(el); + const nameAttr = + el instanceof HTMLInputElement || + el instanceof HTMLTextAreaElement || + el instanceof HTMLSelectElement || + el instanceof HTMLButtonElement + ? normalizeWhitespace(el.name) || undefined + : normalizeWhitespace(el.getAttribute("name") ?? "") || undefined; + const placeholder = + el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement + ? normalizeWhitespace(el.placeholder) || undefined + : normalizeWhitespace(el.getAttribute("placeholder") ?? "") || undefined; + const nearby = nearbyLabelText(el); + + return { + tag, + ...(role ? { role } : {}), + ...(name ? { name } : {}), + ...(nameAttr ? { name_attr: nameAttr } : {}), + ...(placeholder ? { placeholder } : {}), + ...(nearby && nearby !== name ? { nearby_label: nearby } : {}), + }; +} + +export function describeEventTarget(target: EventTarget | null): TargetDescriptor | null { + if (!(target instanceof Element)) return null; + const clickable = resolveClickableElement(target); + if (!clickable) return null; + const desc = describeTarget(clickable); + return isMeaningfulClickTarget(desc) ? desc : null; +} diff --git a/apps/extension/src/lib/record-bridge.ts b/apps/extension/src/lib/record-bridge.ts new file mode 100644 index 0000000..cb5d446 --- /dev/null +++ b/apps/extension/src/lib/record-bridge.ts @@ -0,0 +1,111 @@ +/** + * Wire protocol for user-action recording, sent between the background + * service worker and a tab's content script. + */ + +import type { TargetDescriptor } from "./describe-target"; + +export const RECORD_START = "bsk-record-start"; +export const RECORD_STEP = "bsk-record-step"; +export const RECORD_STOP = "bsk-record-stop"; +export const RECORD_CANCEL = "bsk-record-cancel"; +export const RECORD_FINISH = "bsk-record-finish"; +export const RECORD_QUERY = "bsk-record-query"; + +export interface RecordStartAck { + ok: true; +} + +export type RecordStopAck = { ok: true } | { ok: false; error: string }; + +export interface RecordQueryMessage { + type: typeof RECORD_QUERY; +} + +export interface RecordQueryResponse { + active: boolean; + requestId?: string; +} + +export interface RecordStartMessage { + type: typeof RECORD_START; + requestId: string; +} + +export interface RecordStepPayload { + op: "click" | "fill" | "press" | "select" | "navigate"; + target?: TargetDescriptor; + value?: string; + key?: string; + modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">; + values?: string[]; + labels?: string[]; + url?: string; + redacted?: boolean; + /** Page URL when the step was captured. */ + page_url?: string; + /** Capture-only hint; never persisted unless converted to navigated_to. */ + expects_navigation?: boolean; + /** Whether an observed URL change was synchronously caused by the action. */ + navigation_caused_by_action?: boolean; +} + +export interface RecordStepMessage { + type: typeof RECORD_STEP; + requestId: string; + step: RecordStepPayload; +} + +export interface RecordStopMessage { + type: typeof RECORD_STOP; + requestId: string; +} + +export interface RecordCancelMessage { + type: typeof RECORD_CANCEL; + requestId: string; +} + +export interface RecordFinishMessage { + type: typeof RECORD_FINISH; + requestId: string; +} + +export function isRecordStartMessage(msg: unknown): msg is RecordStartMessage { + if (typeof msg !== "object" || msg === null) return false; + const m = msg as Record; + return m.type === RECORD_START && typeof m.requestId === "string"; +} + +export function isRecordStopMessage(msg: unknown): msg is RecordStopMessage { + if (typeof msg !== "object" || msg === null) return false; + const m = msg as Record; + return m.type === RECORD_STOP && typeof m.requestId === "string"; +} + +export function isRecordCancelMessage(msg: unknown): msg is RecordCancelMessage { + if (typeof msg !== "object" || msg === null) return false; + const m = msg as Record; + return m.type === RECORD_CANCEL && typeof m.requestId === "string"; +} + +export function isRecordFinishMessage(msg: unknown): msg is RecordFinishMessage { + if (typeof msg !== "object" || msg === null) return false; + const m = msg as Record; + return m.type === RECORD_FINISH && typeof m.requestId === "string"; +} + +export function isRecordQueryMessage(msg: unknown): msg is RecordQueryMessage { + if (typeof msg !== "object" || msg === null) return false; + const m = msg as Record; + return m.type === RECORD_QUERY; +} + +export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage { + if (typeof msg !== "object" || msg === null) return false; + const m = msg as Record; + if (m.type !== RECORD_STEP || typeof m.requestId !== "string") return false; + const step = m.step; + if (typeof step !== "object" || step === null) return false; + return typeof (step as RecordStepPayload).op === "string"; +} diff --git a/apps/extension/src/lib/recording-step-buffer.ts b/apps/extension/src/lib/recording-step-buffer.ts new file mode 100644 index 0000000..0c7c26e --- /dev/null +++ b/apps/extension/src/lib/recording-step-buffer.ts @@ -0,0 +1,128 @@ +import type { DraftTraceStep } from "@/transport/types"; +import type { RecordStepPayload } from "./record-bridge"; + +export interface RecordingStepBuffer { + steps: DraftTraceStep[]; + currentUrl?: string; + pendingNavigation: boolean; + pendingNavigationDeadline?: number; +} + +const NAVIGATION_TRIGGER_WINDOW_MS = 3_000; + +function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null { + const pageUrl = payload.page_url; + switch (payload.op) { + case "click": + return payload.target + ? { + op: "click", + target: payload.target, + ...(pageUrl ? { page_url: pageUrl } : {}), + } + : null; + case "fill": + return payload.target + ? { + op: "fill", + target: payload.target, + value: payload.value ?? "", + ...(payload.redacted ? { redacted: true } : {}), + ...(pageUrl ? { page_url: pageUrl } : {}), + } + : null; + case "press": + return payload.key + ? { + op: "press", + key: payload.key, + ...(payload.target ? { target: payload.target } : {}), + ...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}), + ...(pageUrl ? { page_url: pageUrl } : {}), + } + : null; + case "select": + return payload.target && payload.values + ? { + op: "select", + target: payload.target, + values: payload.values, + ...(payload.labels?.length ? { labels: payload.labels } : {}), + ...(pageUrl ? { page_url: pageUrl } : {}), + } + : null; + case "navigate": + return null; + } +} + +function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): boolean { + for (let i = buffer.steps.length - 1; i >= 0; i -= 1) { + const step = buffer.steps[i]; + if (!step) continue; + if (step.op === "click" || step.op === "press" || step.op === "select") { + buffer.steps[i] = { ...step, navigated_to: url }; + return true; + } + break; + } + return false; +} + +export function observeRecordedNavigation( + buffer: RecordingStepBuffer, + url: string, + causedByAction?: boolean, +): void { + if (!url || url === buffer.currentUrl) return; + buffer.currentUrl = url; + const pendingIsCurrent = + buffer.pendingNavigation && + (buffer.pendingNavigationDeadline === undefined || + buffer.pendingNavigationDeadline >= Date.now()); + + if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) { + buffer.pendingNavigation = false; + buffer.pendingNavigationDeadline = undefined; + if (!annotateLastStepNavigation(buffer, url)) { + buffer.steps.push({ + op: "navigate", + url, + page_url: url, + }); + } + return; + } + + buffer.pendingNavigation = false; + buffer.pendingNavigationDeadline = undefined; + buffer.steps.push({ + op: "navigate", + url, + page_url: url, + }); +} + +export function appendRecordedPayload( + buffer: RecordingStepBuffer, + payload: RecordStepPayload, +): void { + if (payload.op === "navigate") { + if (payload.url) { + observeRecordedNavigation(buffer, payload.url, payload.navigation_caused_by_action); + } + return; + } + const step = toDraftStep({ + ...payload, + page_url: payload.page_url ?? buffer.currentUrl, + }); + if (!step) return; + buffer.steps.push(step); + if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") { + buffer.pendingNavigation = payload.expects_navigation === true; + buffer.pendingNavigationDeadline = buffer.pendingNavigation + ? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS + : undefined; + } +} diff --git a/apps/extension/src/lib/trace-reducer.ts b/apps/extension/src/lib/trace-reducer.ts new file mode 100644 index 0000000..e3572ee --- /dev/null +++ b/apps/extension/src/lib/trace-reducer.ts @@ -0,0 +1,218 @@ +import type { DraftTraceStep, PageRef, SelectedOption, Step } from "@/transport/types"; + +const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]); +const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]); + +export function shouldRecordPress( + key: string, + modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">, +): boolean { + if (MODIFIER_ONLY_KEYS.has(key)) return false; + const mods = modifiers ?? []; + const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta"); + if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false; + if (key === "Enter" || key === "Escape") return true; + // Drop bare character typing — FillSession already records the value. + if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false; + return false; +} + +function shouldIncludeDraft(step: DraftTraceStep): boolean { + if (step.op === "fill" && !(step.value ?? "").trim() && !step.redacted) return false; + if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false; + return true; +} + +/** Collapse consecutive navigations to the last hop. */ +function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] { + const out: DraftTraceStep[] = []; + for (const step of steps) { + const prev = out[out.length - 1]; + if (step.op === "navigate" && prev?.op === "navigate") { + out[out.length - 1] = step; + continue; + } + out.push(step); + } + return out; +} + +function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] { + const urls: string[] = []; + if (startUrl) urls.push(startUrl); + for (const step of steps) { + if (step.op === "navigate") { + urls.push(step.url); + continue; + } + if ("page_url" in step && step.page_url) urls.push(step.page_url); + if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to); + } + const seen = new Set(); + const unique: string[] = []; + for (const url of urls) { + if (!url || seen.has(url)) continue; + seen.add(url); + unique.push(url); + } + return unique; +} + +function buildPageRegistry( + steps: DraftTraceStep[], + startUrl?: string, +): { pages: PageRef[]; urlToId: Map } { + const urls = collectUrls(steps, startUrl); + const urlToId = new Map(); + const pages = urls.map((url, index) => { + const id = `p${index + 1}`; + urlToId.set(url, id); + return { id, url }; + }); + return { pages, urlToId }; +} + +function pageIdFor( + url: string | undefined, + urlToId: Map, + fallbackUrl?: string, +): string { + if (url && urlToId.has(url)) return urlToId.get(url)!; + if (fallbackUrl && urlToId.has(fallbackUrl)) return urlToId.get(fallbackUrl)!; + return urlToId.values().next().value ?? "p1"; +} + +function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined { + if (step.op === "navigate") return step.page_url ?? step.url; + if ("page_url" in step && step.page_url) return step.page_url; + return fallbackUrl; +} + +function effectForNavigation( + navigatedTo: string | undefined, + urlToId: Map, +): Step["effect"] { + if (!navigatedTo) return undefined; + const pageId = urlToId.get(navigatedTo); + if (!pageId) return undefined; + return { navigated_to: pageId }; +} + +function withEffect(step: Step, effect: Step["effect"]): Step { + if (!effect) return step; + return { ...step, effect }; +} + +function toSelection(values: string[], labels?: string[]): SelectedOption[] { + return values.map((value, index) => ({ + value, + ...(labels?.[index] ? { label: labels[index] } : {}), + })); +} + +function toV2Step( + step: DraftTraceStep, + id: number, + urlToId: Map, + fallbackUrl?: string, +): Step | null { + if (!shouldIncludeDraft(step)) return null; + + const pageUrl = pageUrlForDraft(step, fallbackUrl); + const page = pageIdFor(pageUrl, urlToId, fallbackUrl); + + switch (step.op) { + case "navigate": + return { + op: "navigate", + id, + page: pageIdFor(step.url, urlToId, fallbackUrl), + to: step.url, + }; + case "click": + return withEffect( + { + op: "click", + id, + page, + target: step.target, + }, + effectForNavigation(step.navigated_to, urlToId), + ); + case "fill": + return { + op: "fill", + id, + page, + target: step.target, + value: step.value, + ...(step.redacted ? { redacted: true } : {}), + }; + case "press": + return withEffect( + { + op: "press", + id, + page, + key: step.key, + ...(step.target ? { target: step.target } : {}), + ...(step.modifiers?.length ? { modifiers: step.modifiers } : {}), + }, + effectForNavigation(step.navigated_to, urlToId), + ); + case "select": + return withEffect( + { + op: "select", + id, + page, + target: step.target, + selection: toSelection(step.values, step.labels), + }, + effectForNavigation(step.navigated_to, urlToId), + ); + } +} + +export interface ReducedTrace { + pages: PageRef[]; + steps: Step[]; +} + +/** + * Compile capture drafts into record-only trace v2 steps. + * Variable inputs are NOT classified here — executing agents infer that at run time. + */ +export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): ReducedTrace { + const collapsed = collapseNavigations(steps); + const { pages, urlToId } = buildPageRegistry(collapsed, startUrl); + const out: Step[] = []; + let id = 1; + let lastUrl = startUrl; + for (const draft of collapsed) { + if (draft.op === "navigate") lastUrl = draft.url; + else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to; + else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url; + const step = toV2Step(draft, id, urlToId, lastUrl); + if (!step) continue; + out.push(step); + id += 1; + } + return { pages, steps: out }; +} + +export function resolveTraceStartUrl( + drafts: DraftTraceStep[], + startUrl?: string, + pages?: PageRef[], +): string { + if (startUrl) return startUrl; + const navigate = drafts.find((step): step is Extract => { + return step.op === "navigate"; + }); + if (navigate) return navigate.url; + for (const draft of drafts) { + if ("page_url" in draft && draft.page_url) return draft.page_url; + } + return pages?.[0]?.url ?? "about:blank"; +} diff --git a/apps/extension/src/tools/__tests__/record-observation.test.ts b/apps/extension/src/tools/__tests__/record-observation.test.ts new file mode 100644 index 0000000..b63ad9d --- /dev/null +++ b/apps/extension/src/tools/__tests__/record-observation.test.ts @@ -0,0 +1,52 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { + ensureBrowserObservationListeners, + isBrowserObservationAttachedForTests, + releaseBrowserObservationListenersIfIdle, + resetBrowserObservationForTests, + setBrowserObservationAttachForTests, +} from "../record"; + +describe("browser observation lifecycle", () => { + afterEach(() => { + resetBrowserObservationForTests(); + }); + + it("does not attach listeners until ensure is called", () => { + const tab = vi.fn(() => () => undefined); + const nav = vi.fn(() => () => undefined); + setBrowserObservationAttachForTests(tab, nav); + + expect(isBrowserObservationAttachedForTests()).toBe(false); + expect(tab).not.toHaveBeenCalled(); + expect(nav).not.toHaveBeenCalled(); + }); + + it("attaches once on ensure and detaches when idle", () => { + const detachTab = vi.fn(); + const detachNav = vi.fn(); + const tab = vi.fn(() => detachTab); + const nav = vi.fn(() => detachNav); + setBrowserObservationAttachForTests(tab, nav); + + ensureBrowserObservationListeners({ + tabsApi: { get: vi.fn(), query: vi.fn(), create: vi.fn() } as never, + sendToTab: vi.fn(), + }); + expect(isBrowserObservationAttachedForTests()).toBe(true); + expect(tab).toHaveBeenCalledTimes(1); + expect(nav).toHaveBeenCalledTimes(1); + + // Idempotent. + ensureBrowserObservationListeners({ + tabsApi: { get: vi.fn(), query: vi.fn(), create: vi.fn() } as never, + sendToTab: vi.fn(), + }); + expect(tab).toHaveBeenCalledTimes(1); + + releaseBrowserObservationListenersIfIdle(); + expect(isBrowserObservationAttachedForTests()).toBe(false); + expect(detachTab).toHaveBeenCalledTimes(1); + expect(detachNav).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/extension/src/tools/dispatcher.ts b/apps/extension/src/tools/dispatcher.ts index 669727f..5091141 100644 --- a/apps/extension/src/tools/dispatcher.ts +++ b/apps/extension/src/tools/dispatcher.ts @@ -13,6 +13,9 @@ import type { NetworkParams, PressParams, ProtocolFrame, + RecordAwaitParams, + RecordStartParams, + RecordStopParams, ReloadParams, RequestFrame, RequestHelpParams, @@ -42,6 +45,7 @@ import { handleScreenshot, handleSnapshot, } from "./observation"; +import { handleRecordAwait, handleRecordStart, handleRecordStop } from "./record"; import { handleSessionStart, handleSessionStop, @@ -396,6 +400,45 @@ export class ToolDispatcher { notificationCopy: this.helpNotificationCopy?.(), signal, }); + case "tool.record_start": + return handleRecordStart(this.sessions, req.params as RecordStartParams, { + tabsApi: chromeTabsApi, + sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg), + bypassOverlay: async (tabId, enabled) => { + try { + await chrome.tabs.sendMessage(tabId, { + type: OVERLAY_AUTOMATION_BYPASS, + enabled, + }); + } catch { + // Content script may be unavailable on restricted pages. + } + }, + ...(this.cdp ? { cdp: this.cdp } : {}), + signal, + }); + case "tool.record_stop": + return handleRecordStop(this.sessions, req.params as RecordStopParams, { + tabsApi: chromeTabsApi, + sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg), + bypassOverlay: async (tabId, enabled) => { + try { + await chrome.tabs.sendMessage(tabId, { + type: OVERLAY_AUTOMATION_BYPASS, + enabled, + }); + } catch { + // Content script may be unavailable on restricted pages. + } + }, + signal, + }); + case "tool.record_await": + return handleRecordAwait(this.sessions, req.params as RecordAwaitParams, { + tabsApi: chromeTabsApi, + sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg), + signal, + }); default: return { code: "unknown_method", diff --git a/apps/extension/src/tools/record.ts b/apps/extension/src/tools/record.ts new file mode 100644 index 0000000..76defdd --- /dev/null +++ b/apps/extension/src/tools/record.ts @@ -0,0 +1,825 @@ +// `tool.record_start` / `tool.record_stop` / `tool.record_await` — capture +// user actions in the Agent Window via the content script and return a +// semantic (LLM textbook) trace. + +import { + isRecordFinishMessage, + isRecordQueryMessage, + isRecordStepMessage, + RECORD_CANCEL, + RECORD_START, + RECORD_STEP, + RECORD_STOP, + type RecordCancelMessage, + type RecordFinishMessage, + type RecordQueryResponse, + type RecordStartAck, + type RecordStartMessage, + type RecordStopMessage, +} from "@/lib/record-bridge"; +import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording-step-buffer"; +import { reduceTraceSteps, resolveTraceStartUrl } from "@/lib/trace-reducer"; +import type { SessionManager } from "@/session-manager/manager"; +import type { + DraftTraceStep, + RecordAwaitParams, + RecordAwaitResult, + RecordStartParams, + RecordStartResult, + RecordStopParams, + RecordStopResult, + RpcError, + Trace, +} from "@/transport/types"; +import { handleNavigate } from "./navigation"; +import { + type CdpRunner, + type ChromeTabsApi, + chromeTabsApi, + isRpcError, + lookupSession, + resolveTargetTab, +} from "./shared"; + +interface ActiveRecording { + requestId: string; + tabId: number; + agentWindowId: number; + startUrl?: string; + purpose?: string; + steps: DraftTraceStep[]; + startedAt: string; + finishPromise: Promise; + resolveFinish: (trace: Trace) => void; + rejectFinish: (err: Error) => void; + settled: boolean; + finishing: boolean; + currentUrl?: string; + pendingNavigation: boolean; + pendingNavigationDeadline?: number; +} + +const recordings = new Map(); + +const RECORD_START_RETRIES = 3; +const RECORD_START_RETRY_DELAY_MS = 500; +const RECORD_REARM_DEBOUNCE_MS = 150; +const RECORD_REARM_MAX_ATTEMPTS = 12; +const RECORD_REARM_RETRY_DELAY_MS = 400; + +const rearmTimers = new Map>(); + +function makeRequestId(tabId: number): string { + return `rec-${tabId}-${Date.now().toString(36)}`; +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** Pages where MV3 content scripts cannot attach (Agent Window boots here). */ +function isContentScriptRestrictedUrl(url: string | undefined): boolean { + if (!url) return true; + const lower = url.toLowerCase(); + return ( + lower === "about:blank" || + lower.startsWith("about:") || + lower.startsWith("chrome://") || + lower.startsWith("chrome-extension://") || + lower.startsWith("edge://") || + lower.startsWith("devtools://") || + lower.startsWith("devtools:") || + lower.startsWith("https://chrome.google.com/webstore") + ); +} + +async function waitForTabReady( + tabId: number, + tabsApi: ChromeTabsApi, + timeoutMs = 10_000, +): Promise { + try { + const tab = await tabsApi.get(tabId); + if (tab.status === "complete") return; + } catch { + return; + } + + await new Promise((resolve, reject) => { + const timer = setTimeout(() => { + chrome.tabs.onUpdated.removeListener(listener); + reject(new Error("tab load timeout")); + }, timeoutMs); + const listener = (updatedTabId: number, info: chrome.tabs.TabChangeInfo) => { + if (updatedTabId !== tabId || info.status !== "complete") return; + clearTimeout(timer); + chrome.tabs.onUpdated.removeListener(listener); + resolve(); + }; + chrome.tabs.onUpdated.addListener(listener); + }); +} + +function isRecordStartAck(response: unknown): response is RecordStartAck { + return ( + typeof response === "object" && + response !== null && + "ok" in response && + (response as RecordStartAck).ok === true + ); +} + +async function sendRecordStartWithAck( + tabId: number, + msg: RecordStartMessage, + sendToTab: RecordDeps["sendToTab"], +): Promise { + let lastError: unknown; + for (let attempt = 0; attempt < RECORD_START_RETRIES; attempt += 1) { + try { + const response = await sendToTab(tabId, msg); + if (isRecordStartAck(response)) return; + lastError = new Error("content script did not ack RECORD_START"); + } catch (err) { + lastError = err; + } + if (attempt + 1 < RECORD_START_RETRIES) { + await sleep(RECORD_START_RETRY_DELAY_MS); + } + } + throw lastError ?? new Error("failed to start recording in content script"); +} + +function buildTrace(recording: ActiveRecording): Trace { + const { pages, steps } = reduceTraceSteps(recording.steps, recording.startUrl); + const startUrl = resolveTraceStartUrl(recording.steps, recording.startUrl, pages); + return { + ...(recording.purpose ? { purpose: recording.purpose } : {}), + recorded_at: new Date().toISOString(), + started_at: recording.startedAt, + entry: { start_url: startUrl }, + pages, + steps, + }; +} + +export interface RecordDeps { + tabsApi: ChromeTabsApi; + sendToTab( + tabId: number, + msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage, + ): Promise; + bypassOverlay?: (tabId: number, enabled: boolean) => Promise; + cdp?: CdpRunner; + signal?: AbortSignal; +} + +let defaultDeps: RecordDeps | null = null; +function getDefaultDeps(): RecordDeps { + if (!defaultDeps) { + defaultDeps = { + tabsApi: chromeTabsApi, + sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg), + }; + } + return defaultDeps; +} + +/** Disposer for lazily attached tab / webNavigation observers. */ +let detachBrowserObservation: (() => void) | null = null; + +type AttachObservation = (deps: RecordDeps) => () => void; + +// Deferred wrappers so we do not capture attach* before their declarations. +let attachTabObservation: AttachObservation = (deps) => attachRecordTabListener(deps); +let attachNavObservation: AttachObservation = (deps) => attachRecordNavigationListener(deps); + +/** Test seam: swap real chrome listeners for fakes. */ +export function setBrowserObservationAttachForTests( + tab: AttachObservation | null, + nav: AttachObservation | null, +): void { + attachTabObservation = tab ?? ((deps) => attachRecordTabListener(deps)); + attachNavObservation = nav ?? ((deps) => attachRecordNavigationListener(deps)); +} + +export function isBrowserObservationAttachedForTests(): boolean { + return detachBrowserObservation !== null; +} + +export function resetBrowserObservationForTests(): void { + detachBrowserObservation?.(); + detachBrowserObservation = null; + recordings.clear(); + attachTabObservation = (deps) => attachRecordTabListener(deps); + attachNavObservation = (deps) => attachRecordNavigationListener(deps); +} + +/** + * Attach tab/webNavigation listeners while any recording is active. + * Must run before navigate-on-start so rearm observes the destination load. + */ +export function ensureBrowserObservationListeners(deps: RecordDeps = getDefaultDeps()): void { + if (detachBrowserObservation) return; + const detachTab = attachTabObservation(deps); + const detachNav = attachNavObservation(deps); + detachBrowserObservation = () => { + detachTab(); + detachNav(); + }; +} + +/** Detach when the recordings map is empty. */ +export function releaseBrowserObservationListenersIfIdle(): void { + if (recordings.size > 0) return; + if (!detachBrowserObservation) return; + detachBrowserObservation(); + detachBrowserObservation = null; +} + +export function attachRecordStepListener(): () => void { + const listener = ( + message: unknown, + _sender: chrome.runtime.MessageSender, + _sendResponse: () => void, + ) => { + if (!isRecordStepMessage(message)) return false; + for (const recording of recordings.values()) { + if (recording.requestId !== message.requestId) continue; + appendRecordedPayload(recording, message.step); + return false; + } + return false; + }; + chrome.runtime.onMessage.addListener(listener); + return () => chrome.runtime.onMessage.removeListener(listener); +} + +export function attachRecordFinishListener(deps: RecordDeps = getDefaultDeps()): () => void { + const listener = ( + message: unknown, + sender: chrome.runtime.MessageSender, + _sendResponse: () => void, + ) => { + if (!isRecordFinishMessage(message)) return false; + const tabId = sender.tab?.id; + if (tabId === undefined) return false; + void finishRecordingByRequest(message.requestId, tabId, deps); + return false; + }; + chrome.runtime.onMessage.addListener(listener); + return () => chrome.runtime.onMessage.removeListener(listener); +} + +function findRecordingByTabId(tabId: number): ActiveRecording | null { + for (const recording of recordings.values()) { + if (recording.tabId === tabId && !recording.settled) return recording; + } + return null; +} + +async function findRecordingForTab( + tabId: number, + deps: RecordDeps, +): Promise { + const direct = findRecordingByTabId(tabId); + if (direct) return direct; + + try { + const tab = await deps.tabsApi.get(tabId); + const windowId = tab.windowId; + if (typeof windowId !== "number") return null; + for (const recording of recordings.values()) { + if (!recording.settled && recording.agentWindowId === windowId) return recording; + } + } catch { + return null; + } + return null; +} + +function clearRearmTimer(tabId: number): void { + const timer = rearmTimers.get(tabId); + if (timer) { + clearTimeout(timer); + rearmTimers.delete(tabId); + } +} + +async function clearRearmTimersForRecording( + recording: ActiveRecording, + deps: RecordDeps, +): Promise { + clearRearmTimer(recording.tabId); + try { + const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId }); + for (const tab of tabs) { + if (typeof tab.id === "number") clearRearmTimer(tab.id); + } + } catch { + // Best-effort cleanup. + } +} + +async function stopRecordingOnAllAgentTabs( + recording: ActiveRecording, + deps: RecordDeps, +): Promise { + const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId }; + let tabIds = [recording.tabId]; + try { + const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId }); + tabIds = [ + ...new Set([ + recording.tabId, + ...tabs.flatMap((tab) => (typeof tab.id === "number" ? [tab.id] : [])), + ]), + ]; + } catch { + // Fall back to the current recording tab. + } + + for (const tabId of tabIds) { + try { + const response = await deps.sendToTab(tabId, stopMsg); + if (tabId === recording.tabId && !isRecordStartAck(response)) { + throw new Error("content script did not confirm recorded steps"); + } + } catch { + if (tabId === recording.tabId) { + throw new Error("failed to flush recorded steps"); + } + } + if (deps.bypassOverlay) { + try { + await deps.bypassOverlay(tabId, false); + } catch { + // Best-effort cleanup. + } + } + } +} + +async function rearmRecording( + recording: ActiveRecording, + targetTabId: number, + deps: RecordDeps, +): Promise { + // Do NOT toggle automation-bypass here: each retry used to increment the + // content-script counter, and a single stop decrement left the ControlOverlay + // stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay + // already hides the control chrome while activeRecord is set. + for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) { + const startMsg: RecordStartMessage = { type: RECORD_START, requestId: recording.requestId }; + try { + await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab); + recording.tabId = targetTabId; + return true; + } catch { + if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) { + await sleep(RECORD_REARM_RETRY_DELAY_MS); + } + } + } + return false; +} + +function scheduleRearmForTab(tabId: number, deps: RecordDeps): void { + const existing = rearmTimers.get(tabId); + if (existing) clearTimeout(existing); + rearmTimers.set( + tabId, + setTimeout(() => { + rearmTimers.delete(tabId); + void (async () => { + const current = await findRecordingForTab(tabId, deps); + if (current) await rearmRecording(current, tabId, deps); + })(); + }, RECORD_REARM_DEBOUNCE_MS), + ); +} + +export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): () => void { + const onCreated = (tab: chrome.tabs.Tab) => { + const tabId = tab.id; + const windowId = tab.windowId; + if (tabId === undefined || windowId === undefined) return; + for (const recording of recordings.values()) { + if (recording.settled || recording.agentWindowId !== windowId) continue; + scheduleRearmForTab(tabId, deps); + return; + } + }; + + const onActivated = (activeInfo: chrome.tabs.TabActiveInfo) => { + scheduleRearmForTab(activeInfo.tabId, deps); + }; + + chrome.tabs.onCreated.addListener(onCreated); + chrome.tabs.onActivated.addListener(onActivated); + return () => { + chrome.tabs.onCreated.removeListener(onCreated); + chrome.tabs.onActivated.removeListener(onActivated); + }; +} + +export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps()): () => void { + const observeMainFrame = (tabId: number, url?: string, causedByAction?: boolean) => { + void (async () => { + const recording = await findRecordingForTab(tabId, deps); + if (recording && url) { + observeRecordedNavigation(recording, url, causedByAction); + } + })(); + }; + const onMainFrameComplete = (tabId: number, url?: string) => { + void (async () => { + observeMainFrame(tabId, url); + scheduleRearmForTab(tabId, deps); + })(); + }; + + if (chrome.webNavigation?.onCompleted) { + const completedListener = ( + details: chrome.webNavigation.WebNavigationFramedCallbackDetails, + ) => { + if (details.frameId !== 0) return; + onMainFrameComplete(details.tabId, details.url); + }; + const committedListener = ( + details: chrome.webNavigation.WebNavigationTransitionCallbackDetails, + ) => { + if (details.frameId !== 0) return; + const causedByAction = + details.transitionType === "link" || details.transitionType === "form_submit" + ? true + : details.transitionType === "typed" || + details.transitionType === "auto_bookmark" || + details.transitionType === "generated" || + details.transitionType === "keyword" || + details.transitionType === "keyword_generated" || + details.transitionType === "reload" + ? false + : undefined; + observeMainFrame(details.tabId, details.url, causedByAction); + }; + chrome.webNavigation.onCompleted.addListener(completedListener); + chrome.webNavigation.onCommitted?.addListener(committedListener); + return () => { + chrome.webNavigation.onCompleted.removeListener(completedListener); + chrome.webNavigation.onCommitted?.removeListener(committedListener); + }; + } + + const listener = (tabId: number, info: chrome.tabs.TabChangeInfo) => { + if (info.status !== "complete") return; + onMainFrameComplete(tabId, info.url); + }; + chrome.tabs.onUpdated.addListener(listener); + return () => chrome.tabs.onUpdated.removeListener(listener); +} + +export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()): () => void { + const listener = ( + message: unknown, + sender: chrome.runtime.MessageSender, + sendResponse: (response: RecordQueryResponse) => void, + ) => { + if (!isRecordQueryMessage(message)) return false; + const tabId = sender.tab?.id; + if (tabId === undefined) { + sendResponse({ active: false }); + return false; + } + void (async () => { + const recording = await findRecordingForTab(tabId, deps); + if (!recording) { + sendResponse({ active: false }); + return; + } + await rearmRecording(recording, tabId, deps); + sendResponse({ active: true, requestId: recording.requestId }); + })(); + return true; + }; + chrome.runtime.onMessage.addListener(listener); + return () => chrome.runtime.onMessage.removeListener(listener); +} + +async function finishRecordingByRequest( + requestId: string, + tabId: number, + deps: RecordDeps, +): Promise { + for (const [sessionId, recording] of recordings) { + if (recording.requestId !== requestId || recording.settled) continue; + const match = await findRecordingForTab(tabId, deps); + if (match !== recording) continue; + await finishRecording(sessionId, deps); + return; + } +} + +async function finishRecording(sessionId: string, deps: RecordDeps): Promise { + const recording = recordings.get(sessionId); + if (!recording || recording.settled || recording.finishing) return null; + recording.finishing = true; + + await clearRearmTimersForRecording(recording, deps); + try { + await stopRecordingOnAllAgentTabs(recording, deps); + } catch { + recording.finishing = false; + return null; + } + + recording.settled = true; + recordings.delete(sessionId); + releaseBrowserObservationListenersIfIdle(); + const trace = buildTrace(recording); + recording.resolveFinish(trace); + return trace; +} + +export async function handleRecordStart( + manager: SessionManager, + params: RecordStartParams, + deps: RecordDeps = getDefaultDeps(), +): Promise { + const ctxOrErr = lookupSession(manager, params, "record_start"); + if (isRpcError(ctxOrErr)) return ctxOrErr; + const ctx = ctxOrErr; + if (recordings.has(params.session_id)) { + return { + code: "protocol_error", + message: `session ${params.session_id} is already recording`, + }; + } + const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi); + if (isRpcError(target)) return target; + + // Register the recording *before* navigate so content-script syncAgentOverlay + // on the destination page can RECORD_QUERY → rearm → show RecordOverlay + // instead of flashing ControlOverlay ("Agent 正在控制"). + const requestId = makeRequestId(target.tabId); + let resolveFinish!: (trace: Trace) => void; + let rejectFinish!: (err: Error) => void; + const finishPromise = new Promise((resolve, reject) => { + resolveFinish = resolve; + rejectFinish = reject; + }); + const provisionalUrl = params.url; + recordings.set(params.session_id, { + requestId, + tabId: target.tabId, + agentWindowId: ctx.agentWindowId, + startUrl: provisionalUrl, + ...(params.purpose ? { purpose: params.purpose } : {}), + steps: [], + startedAt: new Date().toISOString(), + finishPromise, + resolveFinish, + rejectFinish, + settled: false, + finishing: false, + currentUrl: provisionalUrl, + pendingNavigation: false, + pendingNavigationDeadline: undefined, + }); + // Observe navigations for the whole recording lifetime; attach before + // optional navigate so the destination load can rearm capture. + ensureBrowserObservationListeners(deps); + + const abortPending = async (notifyContent: boolean) => { + recordings.delete(params.session_id); + releaseBrowserObservationListenersIfIdle(); + if (notifyContent) { + try { + await deps.sendToTab(target.tabId, { + type: RECORD_CANCEL, + requestId, + }); + } catch { + // Content may never have received RECORD_START. + } + } + if (deps.bypassOverlay) { + try { + await deps.bypassOverlay(target.tabId, false); + } catch { + // Ignore cleanup errors. + } + } + }; + + const cancelledError = (): RpcError => ({ + code: "cancelled", + message: "record_start aborted", + }); + + /** Dispatcher may race-cancel while we still hold a provisional recording. */ + const abortIfCancelled = async (notifyContent: boolean): Promise => { + if (!deps.signal?.aborted) return null; + await abortPending(notifyContent); + return cancelledError(); + }; + + { + const cancelled = await abortIfCancelled(false); + if (cancelled) return cancelled; + } + + if (params.url && deps.cdp) { + const nav = await handleNavigate( + manager, + { + session_id: params.session_id, + url: params.url, + tab_id: target.tabId, + }, + { cdp: deps.cdp, tabsApi: deps.tabsApi, signal: deps.signal }, + ); + if (isRpcError(nav)) { + await abortPending(false); + return nav; + } + { + const cancelled = await abortIfCancelled(false); + if (cancelled) return cancelled; + } + try { + await waitForTabReady(target.tabId, deps.tabsApi); + } catch { + // Proceed with retries even if the tab never reports complete. + } + { + const cancelled = await abortIfCancelled(false); + if (cancelled) return cancelled; + } + } + + let startUrl: string | undefined; + try { + const tab = await deps.tabsApi.get(target.tabId); + startUrl = tab.url; + } catch { + startUrl = params.url; + } + + const active = recordings.get(params.session_id); + if (!active) { + // Cleared by a concurrent abort / session teardown. + return cancelledError(); + } + active.startUrl = startUrl; + active.currentUrl = startUrl; + + if (isContentScriptRestrictedUrl(startUrl)) { + await abortPending(false); + return { + code: "invalid_params", + message: params.url + ? `cannot record on restricted URL (${startUrl}); use an http(s) page` + : "Agent Window starts on about:blank where the content script cannot inject — pass --url https://… before recording", + }; + } + + { + const cancelled = await abortIfCancelled(false); + if (cancelled) return cancelled; + } + + if (deps.bypassOverlay) { + try { + // Single ref for the initial race before RecordOverlay mounts; rearm must + // not stack additional refs (see rearmRecording). Cleared on stop. + await deps.bypassOverlay(target.tabId, true); + } catch { + // Best-effort; activeRecord also hides the control overlay. + } + } + + { + const cancelled = await abortIfCancelled(true); + if (cancelled) return cancelled; + } + + const startMsg: RecordStartMessage = { type: RECORD_START, requestId }; + + try { + await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab); + } catch { + await abortPending(true); + return { + code: "protocol_error", + message: + "failed to start recording in content script — reload the BrowserSkill extension, then retry with --url https://…", + }; + } + + { + const cancelled = await abortIfCancelled(true); + if (cancelled) return cancelled; + } + + return { tab_id: target.tabId, recording: true }; +} + +export async function handleRecordStop( + manager: SessionManager, + params: RecordStopParams, + deps: RecordDeps = getDefaultDeps(), +): Promise { + const ctxOrErr = lookupSession(manager, params, "record_stop"); + if (isRpcError(ctxOrErr)) return ctxOrErr; + + const recording = recordings.get(params.session_id); + if (!recording) { + return { + code: "not_found", + message: `no active recording for session ${params.session_id}`, + }; + } + + const trace = await finishRecording(params.session_id, deps); + if (!trace) { + return { + code: "protocol_error", + message: `failed to flush recorded steps for session ${params.session_id}; the recording is still active — retry \`bsk record stop\``, + }; + } + return { trace }; +} + +export async function handleRecordAwait( + manager: SessionManager, + params: RecordAwaitParams, + deps: RecordDeps = getDefaultDeps(), +): Promise { + const ctxOrErr = lookupSession(manager, params, "record_await"); + if (isRpcError(ctxOrErr)) return ctxOrErr; + + const recording = recordings.get(params.session_id); + if (!recording) { + return { + code: "not_found", + message: `no active recording for session ${params.session_id}`, + }; + } + + if (deps.signal?.aborted) { + return { code: "cancelled", message: "record_await aborted" }; + } + + const outcome = await new Promise<{ trace: Trace } | { error: RpcError }>((resolve) => { + let settled = false; + const finish = (result: { trace: Trace } | { error: RpcError }) => { + if (settled) return; + settled = true; + if (timer) clearTimeout(timer); + deps.signal?.removeEventListener("abort", onAbort); + resolve(result); + }; + const onAbort = () => finish({ error: { code: "cancelled", message: "record_await aborted" } }); + const timer = + params.timeout_ms === undefined + ? undefined + : setTimeout( + () => + finish({ + error: { + code: "timeout", + message: `record_await timed out after ${params.timeout_ms}ms`, + }, + }), + params.timeout_ms, + ); + deps.signal?.addEventListener("abort", onAbort, { once: true }); + void recording.finishPromise.then( + (trace) => finish({ trace }), + () => + finish({ + error: { code: "cancelled", message: "recording was cleared" }, + }), + ); + }); + return "trace" in outcome ? { trace: outcome.trace } : outcome.error; +} + +export function clearRecordingForSession(sessionId: string): void { + const recording = recordings.get(sessionId); + if (!recording) { + recordings.delete(sessionId); + releaseBrowserObservationListenersIfIdle(); + return; + } + void clearRearmTimersForRecording(recording, getDefaultDeps()); + if (!recording.settled) { + recording.settled = true; + recording.rejectFinish(new Error("recording cleared")); + } + recordings.delete(sessionId); + releaseBrowserObservationListenersIfIdle(); +} + +export type { RecordFinishMessage }; diff --git a/apps/extension/src/tools/session.ts b/apps/extension/src/tools/session.ts index 6c14d7e..c83a2cd 100644 --- a/apps/extension/src/tools/session.ts +++ b/apps/extension/src/tools/session.ts @@ -1,5 +1,6 @@ import type { SessionManager } from "@/session-manager/manager"; import type { RpcError } from "@/transport/types"; +import { clearRecordingForSession } from "./record"; import { returnBorrowedTab, type TabManagementDeps } from "./tabs"; export interface SessionStartParams { @@ -156,6 +157,8 @@ export async function handleSessionStop( // Step 2: clear the per-session RefStore (review M6/M7 parity). ctx.refStore.clear(); + clearRecordingForSession(params.session_id); + // Step 3: detach CDP sessions this session opened (no-op if none). await deps.cdp?.detachSession(params.session_id); diff --git a/apps/extension/src/transport/types.ts b/apps/extension/src/transport/types.ts index 2727491..bcff0e3 100644 --- a/apps/extension/src/transport/types.ts +++ b/apps/extension/src/transport/types.ts @@ -537,3 +537,135 @@ export interface RequestHelpResult { tab_id: number; resolved_targets?: ResolvedTarget[]; } + +// -------------------------------------------------------------------------- +// Semantic record payloads — mirror bsk-protocol record.rs (Trace v2) +// -------------------------------------------------------------------------- + +export interface TargetDescriptor { + role?: string; + name?: string; + tag: string; + name_attr?: string; + placeholder?: string; + nearby_label?: string; +} + +export interface TraceEntry { + start_url: string; +} + +export interface PageRef { + id: string; + url: string; + title?: string; +} + +export interface SelectedOption { + value: string; + label?: string; +} + +export interface StepEffect { + navigated_to: string; +} + +export interface StepCommon { + id: number; + page: string; + effect?: StepEffect; +} + +/** Capture/buffer draft before v2 reduction. */ +export type DraftTraceStep = + | { + op: "click"; + target: TargetDescriptor; + navigated_to?: string; + page_url?: string; + } + | { + op: "fill"; + target: TargetDescriptor; + value: string; + redacted?: boolean; + page_url?: string; + } + | { + op: "press"; + key: string; + target?: TargetDescriptor; + modifiers?: KeyModifier[]; + navigated_to?: string; + page_url?: string; + } + | { + op: "select"; + target: TargetDescriptor; + values: string[]; + labels?: string[]; + navigated_to?: string; + page_url?: string; + } + | { + op: "navigate"; + url: string; + page_url?: string; + }; + +/** Exported record-only step (trace v2). */ +export type Step = + | ({ op: "navigate" } & StepCommon & { to: string }) + | ({ op: "click" } & StepCommon & { target: TargetDescriptor }) + | ({ op: "fill" } & StepCommon & { + target: TargetDescriptor; + value: string; + redacted?: boolean; + }) + | ({ op: "select" } & StepCommon & { + target: TargetDescriptor; + selection: SelectedOption[]; + }) + | ({ op: "press" } & StepCommon & { + key: string; + modifiers?: KeyModifier[]; + target?: TargetDescriptor; + }); + +export interface Trace { + recorded_at: string; + started_at?: string; + purpose?: string; + entry: TraceEntry; + pages: PageRef[]; + steps: Step[]; +} + +export interface RecordStartParams { + session_id: string; + tab_id?: number; + url?: string; + purpose?: string; +} + +export interface RecordStartResult { + tab_id: number; + recording: boolean; +} + +export interface RecordStopParams { + session_id: string; +} + +export interface RecordStopResult { + trace: Trace; +} + +export interface RecordAwaitParams { + session_id: string; + timeout_ms?: number; +} + +export interface RecordAwaitResult { + trace: Trace; +} diff --git a/apps/extension/wxt.config.ts b/apps/extension/wxt.config.ts index 9eee8da..9e91051 100644 --- a/apps/extension/wxt.config.ts +++ b/apps/extension/wxt.config.ts @@ -25,7 +25,7 @@ export default defineConfig({ name: "BrowserSkill", description: "Let AI agents use your logged-in browser in a separate Agent Window—without interrupting your work. Powered by the bsk CLI.", - permissions: ["alarms", "debugger", "notifications", "tabs", "storage", "windows"], + permissions: ["alarms", "debugger", "notifications", "tabs", "storage", "webNavigation", "windows"], host_permissions: [""], icons: { 16: "icon/logo.png", diff --git a/crates/bsk-cli/skill/SKILL.md b/crates/bsk-cli/skill/SKILL.md index 5edf9ea..d68b40f 100644 --- a/crates/bsk-cli/skill/SKILL.md +++ b/crates/bsk-cli/skill/SKILL.md @@ -46,6 +46,21 @@ Optional: `bsk session start --browser ` when multiple bro Emergency cleanup: `bsk session stop --all` or the Agent Window overlay **Stop all**. +## Stop when the goal is met + +Every task is a **bounded goal**, not open-ended browsing. The goal may come from the user's request, a recorded `trace.json`, or both. + +1. **Define success first** — one concrete, observable condition derived from the user's words, `purpose`, or the last meaningful step in a trace (e.g. "form submitted", "item added to cart", "playback started"). +2. **Take the shortest path** — snapshot → act → at most one check. Do not wander, re-try unrelated actions, or stack exploratory steps. +3. **Stop as soon as success is reached** — run `bsk session stop ` immediately unless the user explicitly asked to keep the session open (e.g. "don't close yet", "keep browsing"). +4. **No post-success work** — once the goal is met, do not click, refresh, navigate, re-search, switch tabs, or "double-check" that it worked. Further verification is a new task. +5. **When blocked, pause — do not brute-force** — if the page requires human input (login, captcha, OTP, payment confirmation) or an action fails twice with no progress, call `bsk request-help` instead of retrying blindly. See **Ask the human for help** below. +6. **When unsure** — at most one extra `bsk snapshot`. If success looks met, stop. If not, ask the user; do not keep clicking. + +**With a trace:** replay steps in order using `target` role/name/tag and raw `value`/`selection` fields. After the last step (or when its `effect.navigated_to` / success hint is satisfied), apply rules 3–4 immediately. The trace guides execution; it does not extend control beyond the goal. + +**Without a trace:** the user's request *is* the success condition. Satisfying it ends the task — same stop rules apply. + ## Core interaction loop Write operations only affect tabs in the **Agent Window** (or tabs you **borrowed** into it). @@ -191,6 +206,23 @@ acts. The result `outcome` is one of: `note` carries any text the user typed back. `resolved_targets` reports which refs/selectors matched a live element. +### Recording — `bsk record` + +Capture the user's own actions in the Agent Window to a `trace.json`, for later LLM-driven automation: + +```bash +bsk record start --url https://… [--purpose "publish a wiki doc"] [--output trace.json] +# `--url` is required on a fresh session (about:blank cannot host the content script). +# Blocks until the user clicks Finish in the recording panel, then writes ./trace.json and closes the window. + +bsk record stop [--output trace.json] # terminal fallback if the browser panel is unavailable +``` + +- The trace is a **record-only action log** (a `pages[]` dictionary + `navigate`/`click`/`fill`/`select`/`press` steps with `target` descriptors). It records *what the user did*; deciding which inputs are variable is left to the executing agent. +- `--purpose` is optional context metadata; it does **not** change what gets captured. +- There is **no** `bsk replay` — to redo a flow, read the trace and reuse the existing `session` / `snapshot` / `@eN` / `click` / `fill` tools. Follow **Stop when the goal is met**. +- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text. + ## Error handling ### Exit codes (`echo $?` after `bsk …`) @@ -221,8 +253,9 @@ Always **`bsk session stop `** in a `finally`-style path so the Agent Window 1. **No token theft** — do not `bsk evaluate` on sensitive sites to read `localStorage`, cookies, or auth headers for exfiltration. 2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks. 3. **No skip stop** — always `bsk session stop `; never assume idle timeout will clean up. -4. **No observe escalation before snapshot** — use `bsk snapshot` first; only use `bsk get-html` or `bsk screenshot` when the snapshot is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot ref — never skip snapshot just to grab a visual. -5. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces. +4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open. +5. **No observe escalation before snapshot** — use `bsk snapshot` first; only use `bsk get-html` or `bsk screenshot` when the snapshot is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot ref — never skip snapshot just to grab a visual. +6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces. --- diff --git a/crates/bsk-cli/src/cli/mod.rs b/crates/bsk-cli/src/cli/mod.rs index cada994..9e8e48a 100644 --- a/crates/bsk-cli/src/cli/mod.rs +++ b/crates/bsk-cli/src/cli/mod.rs @@ -19,6 +19,8 @@ pub mod interaction; pub mod logs; pub mod navigate; pub mod network; +pub mod record; +pub mod record_state; pub mod render_error; pub mod screenshot; pub mod session; @@ -39,6 +41,7 @@ use crate::cli::install_skill::InstallSkillArgs; use crate::cli::interaction::{ClickArgs, FillArgs, PressArgs, SelectArgs}; use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs}; use crate::cli::network::NetworkArgs; +use crate::cli::record::RecordCmd; use crate::cli::screenshot::ScreenshotArgs; use crate::cli::session::SessionCmd; use crate::cli::snapshot::SnapshotArgs; @@ -169,6 +172,9 @@ pub enum Command { /// Ask the human to complete an in-page step (captcha / login / confirm). #[command(name = "request-help")] RequestHelp(RequestHelpArgs), + + /// Record user actions into a semantic `trace.json` textbook for LLMs. + Record(RecordCmd), } #[derive(Debug, Clone, Args, Default)] diff --git a/crates/bsk-cli/src/cli/record.rs b/crates/bsk-cli/src/cli/record.rs new file mode 100644 index 0000000..38a93fb --- /dev/null +++ b/crates/bsk-cli/src/cli/record.rs @@ -0,0 +1,256 @@ +//! `bsk record start|stop` — capture user actions in the Agent Window. + +use std::fs; +use std::path::PathBuf; +use std::time::Duration; + +use anyhow::Context; +use bsk_protocol::Method; +use bsk_protocol::tools::{ + RecordAwaitParams, RecordAwaitResult, RecordStartParams, RecordStartResult, RecordStopParams, + RecordStopResult, Trace, +}; +use clap::{Args, Subcommand}; + +use crate::cli::TOOL_IPC_TIMEOUT; +use crate::cli::business_rpc; +use crate::cli::ensure_daemon::ensure_daemon; +use crate::cli::error::{CliError, Format}; +use crate::cli::record_state; +use crate::cli::session::{start_session, stop_session}; + +/// Max wait for the user to click 结束 in the browser (24 hours). +const RECORD_AWAIT_TIMEOUT_MS: u32 = 86_400_000; + +/// `bsk record …` subcommand tree. +#[derive(Debug, Clone, Args)] +pub struct RecordCmd { + #[command(subcommand)] + pub sub: RecordSub, +} + +#[derive(Debug, Clone, Subcommand)] +pub enum RecordSub { + /// Open the Agent Window, record user actions, and block until finished in the browser. + Start(RecordStartArgs), + /// Stop recording from the terminal (fallback), write trace JSON, and close the window. + /// Works even while `record start` is blocked in `record_await` (daemon forwards + /// `tool.record_stop` without the per-session busy gate). + Stop(RecordStopArgs), +} + +#[derive(Debug, Clone, Args)] +pub struct RecordStartArgs { + #[arg(long = "tab-id")] + pub tab_id: Option, + + /// Navigate to this http(s) URL before recording (required for a fresh + /// session — Agent Window starts on about:blank, which cannot host the + /// content script). + #[arg(long)] + pub url: Option, + + /// Optional goal text stored on the exported trace for LLM context. + #[arg(long)] + pub purpose: Option, + + /// Output path for the trace JSON (default `./trace.json`). + #[arg(long, default_value = "trace.json")] + pub output: PathBuf, +} + +#[derive(Debug, Clone, Args)] +pub struct RecordStopArgs { + /// Output path for the trace JSON (default `./trace.json`). + #[arg(long, default_value = "trace.json")] + pub output: PathBuf, +} + +pub fn dispatch(cmd: RecordCmd, format: Format) -> Result<(), CliError> { + match cmd.sub { + RecordSub::Start(args) => dispatch_start(args, format), + RecordSub::Stop(args) => dispatch_stop(args, format), + } +} + +fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError> { + if record_state::read().is_ok() { + return Err(CliError::Local(anyhow::anyhow!( + "a recording is already in progress; run `bsk record stop` first" + ))); + } + + let info = ensure_daemon().context("ensure daemon is running")?; + let session = start_session(info.sock_path.clone(), None)?; + + let start_params = RecordStartParams { + session_id: session.session_id.clone(), + tab_id: args.tab_id, + url: args.url, + purpose: args.purpose.clone(), + }; + let start_result = business_rpc::call::( + info.sock_path.clone(), + "record-start", + Method::ToolRecordStart, + Some(start_params), + TOOL_IPC_TIMEOUT, + ); + + let start_result = match start_result { + Ok(result) => result, + Err(err) => { + let _ = stop_session(info.sock_path, &session.session_id); + return Err(err); + } + }; + + if let Err(err) = record_state::write(&session.session_id) { + let _ = stop_session(info.sock_path.clone(), &session.session_id); + return Err(CliError::Local(err)); + } + + if format == Format::Human { + println!( + "recording on tab={} — click 结束 in the browser when done", + start_result.tab_id + ); + } + + let await_params = RecordAwaitParams { + session_id: session.session_id.clone(), + timeout_ms: Some(RECORD_AWAIT_TIMEOUT_MS), + }; + let await_result = business_rpc::call::( + info.sock_path.clone(), + "record-await", + Method::ToolRecordAwait, + Some(await_params), + record_await_ipc_timeout(RECORD_AWAIT_TIMEOUT_MS), + ); + + // Keep write/render inside a Result so `?` cannot skip session teardown. + let run_result: Result<(), CliError> = match await_result { + Ok(result) => (|| { + write_trace_file(&args.output, &result.trace)?; + render_finish(&result.trace, &args.output, format) + })(), + Err(err) => Err(err), + }; + + let session_stop_result = stop_session(info.sock_path, &session.session_id); + record_state::clear(); + + run_result?; + session_stop_result?; + Ok(()) +} + +fn dispatch_stop(args: RecordStopArgs, format: Format) -> Result<(), CliError> { + let state = record_state::read().map_err(CliError::Local)?; + let info = ensure_daemon().context("ensure daemon is running")?; + let session_id = state.session_id.clone(); + + let run_result: Result<(), CliError> = (|| { + let params = RecordStopParams { + session_id: session_id.clone(), + }; + let result = business_rpc::call::( + info.sock_path.clone(), + "record-stop", + Method::ToolRecordStop, + Some(params), + TOOL_IPC_TIMEOUT, + )?; + write_trace_file(&args.output, &result.trace)?; + render_stop(&result, &args.output, format) + })(); + + let session_stop_result = stop_session(info.sock_path, &session_id); + record_state::clear(); + + run_result?; + session_stop_result?; + Ok(()) +} + +fn record_await_ipc_timeout(timeout_ms: u32) -> Duration { + Duration::from_millis(u64::from(timeout_ms)) + .checked_add(Duration::from_secs(15)) + .unwrap_or(Duration::from_secs(u64::from(timeout_ms / 1_000) + 15)) +} + +fn write_trace_file(output: &PathBuf, trace: &Trace) -> Result<(), CliError> { + let json = serde_json::to_string_pretty(trace) + .context("serialize trace JSON") + .map_err(CliError::Local)?; + if let Some(parent) = output.parent() { + if !parent.as_os_str().is_empty() { + fs::create_dir_all(parent) + .with_context(|| format!("create output directory {}", parent.display())) + .map_err(CliError::Local)?; + } + } + fs::write(output, format!("{json}\n")) + .with_context(|| format!("write trace to {}", output.display())) + .map_err(CliError::Local)?; + Ok(()) +} + +fn render_finish(trace: &Trace, output: &PathBuf, format: Format) -> Result<(), CliError> { + match format { + Format::Json => { + println!( + "{}", + serde_json::to_string(&serde_json::json!({ + "output": output, + "trace": trace, + "window_closed": true, + })) + .map_err(|e| CliError::Local(anyhow::anyhow!(e)))? + ); + } + Format::Human => { + println!("saved {} steps to {}", trace.steps.len(), output.display()); + } + } + Ok(()) +} + +fn render_stop( + result: &RecordStopResult, + output: &PathBuf, + format: Format, +) -> Result<(), CliError> { + render_finish(&result.trace, output, format) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_output_is_trace_json() { + let args = RecordStopArgs { + output: PathBuf::from("trace.json"), + }; + assert_eq!(args.output, PathBuf::from("trace.json")); + } + + #[test] + fn start_args_default_output_is_trace_json() { + let args = RecordStartArgs { + tab_id: None, + url: None, + purpose: None, + output: PathBuf::from("trace.json"), + }; + assert_eq!(args.output, PathBuf::from("trace.json")); + } + + #[test] + fn record_await_ipc_timeout_covers_long_wait() { + let got = record_await_ipc_timeout(RECORD_AWAIT_TIMEOUT_MS); + assert!(got >= Duration::from_secs(86_400)); + } +} diff --git a/crates/bsk-cli/src/cli/record_state.rs b/crates/bsk-cli/src/cli/record_state.rs new file mode 100644 index 0000000..c56a686 --- /dev/null +++ b/crates/bsk-cli/src/cli/record_state.rs @@ -0,0 +1,117 @@ +//! Persist the session id between `bsk record start` and `bsk record stop`. + +use std::fs; +use std::time::{SystemTime, UNIX_EPOCH}; + +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; + +use crate::daemon::paths; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct RecordSessionState { + pub session_id: String, + pub started_at: String, +} + +pub fn write(session_id: &str) -> Result<()> { + let path = paths::record_session_path()?; + paths::ensure_bsk_home()?; + if path.exists() { + return Err(anyhow::anyhow!( + "a recording is already in progress; run `bsk record stop` first" + )); + } + let state = RecordSessionState { + session_id: session_id.to_string(), + started_at: started_at_unix_ms(), + }; + let json = serde_json::to_string_pretty(&state).context("serialize record session state")?; + fs::write(&path, format!("{json}\n")).with_context(|| format!("write {}", path.display()))?; + Ok(()) +} + +pub fn read() -> Result { + let path = paths::record_session_path()?; + if !path.exists() { + return Err(anyhow::anyhow!( + "no recording in progress; run `bsk record start` first" + )); + } + let raw = fs::read_to_string(&path).with_context(|| format!("read {}", path.display()))?; + serde_json::from_str(&raw).with_context(|| format!("parse {}", path.display())) +} + +pub fn clear() { + if let Ok(path) = paths::record_session_path() { + let _ = fs::remove_file(path); + } +} + +fn started_at_unix_ms() -> String { + let ms = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_millis(); + format!("{ms}") +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::Mutex; + + fn env_guard() -> &'static Mutex<()> { + static GUARD: Mutex<()> = Mutex::new(()); + &GUARD + } + + fn with_temp_home(f: F) { + let _lock = env_guard().lock().unwrap_or_else(|e| e.into_inner()); + let tmp = tempfile::tempdir().unwrap(); + unsafe { + std::env::set_var(crate::daemon::paths::BSK_HOME_ENV, tmp.path()); + } + f(); + clear(); + unsafe { + std::env::remove_var(crate::daemon::paths::BSK_HOME_ENV); + } + } + + #[test] + fn write_read_round_trips() { + with_temp_home(|| { + write("abcd").unwrap(); + let state = read().unwrap(); + assert_eq!(state.session_id, "abcd"); + assert!(!state.started_at.is_empty()); + }); + } + + #[test] + fn write_rejects_duplicate() { + with_temp_home(|| { + write("abcd").unwrap(); + let err = write("efgh").unwrap_err(); + assert!(err.to_string().contains("already in progress")); + }); + } + + #[test] + fn read_errors_when_missing() { + with_temp_home(|| { + let err = read().unwrap_err(); + assert!(err.to_string().contains("no recording in progress")); + }); + } + + #[test] + fn clear_removes_state_file() { + with_temp_home(|| { + write("abcd").unwrap(); + clear(); + assert!(read().is_err()); + }); + } +} diff --git a/crates/bsk-cli/src/cli/session.rs b/crates/bsk-cli/src/cli/session.rs index 0f1d94a..1fb1d36 100644 --- a/crates/bsk-cli/src/cli/session.rs +++ b/crates/bsk-cli/src/cli/session.rs @@ -63,11 +63,11 @@ struct StartParams { } #[derive(Debug, Deserialize)] -struct StartReply { - session_id: String, - browser_instance_id: String, +pub struct StartReply { + pub session_id: String, + pub browser_instance_id: String, #[serde(default)] - agent_window_id: Option, + pub agent_window_id: Option, } #[derive(Debug, Serialize)] @@ -78,21 +78,21 @@ struct StopParams { } #[derive(Debug, Deserialize)] -struct StopReply { - stopped: Vec, +pub struct StopReply { + pub stopped: Vec, #[serde(default)] - failed: Vec, + pub failed: Vec, #[serde(default)] - returned_tab_ids: Vec, + pub returned_tab_ids: Vec, #[serde(default)] - return_failures: Vec, + pub return_failures: Vec, } #[derive(Debug, Deserialize, Serialize)] -struct StopFailure { - session_id: String, - code: ErrorCode, - message: String, +pub struct StopFailure { + pub session_id: String, + pub code: ErrorCode, + pub message: String, } #[derive(Debug, Deserialize)] @@ -113,14 +113,7 @@ pub fn dispatch(cmd: SessionCmd, format: Format) -> Result<(), CliError> { } fn run_start(sock: PathBuf, args: SessionStartArgs, format: Format) -> Result<(), CliError> { - let result: Result = call( - sock, - Method::SessionStart, - Some(StartParams { - browser_instance_id: args.browser, - }), - Duration::from_secs(30), - ); + let result = start_session(sock, args.browser); match result { Ok(reply) => match format { Format::Json => { @@ -143,6 +136,31 @@ fn run_start(sock: PathBuf, args: SessionStartArgs, format: Format) -> Result<() Ok(()) } +/// Start a session and open the Agent Window. Used by `session start` and `record start`. +pub fn start_session(sock: PathBuf, browser: Option) -> Result { + call( + sock, + Method::SessionStart, + Some(StartParams { + browser_instance_id: browser, + }), + Duration::from_secs(30), + ) +} + +/// Stop a single session by id. +pub fn stop_session(sock: PathBuf, session_id: &str) -> Result { + call( + sock, + Method::SessionStop, + Some(StopParams { + session_id: Some(session_id.to_string()), + all: false, + }), + SESSION_STOP_IPC_TIMEOUT, + ) +} + /// Render a `session.start` failure (review I3). /// /// The previous implementation hand-rolled `eprintln!("error: ...")` diff --git a/crates/bsk-cli/src/daemon/ipc.rs b/crates/bsk-cli/src/daemon/ipc.rs index 654d96c..d273bcf 100644 --- a/crates/bsk-cli/src/daemon/ipc.rs +++ b/crates/bsk-cli/src/daemon/ipc.rs @@ -242,7 +242,10 @@ pub fn full_handler(status: DaemonStatus, state: Arc) -> RpcHandler | Method::ToolSelect | Method::ToolEvaluate | Method::ToolWaitForNavigation - | Method::ToolRequestHelp => { + | Method::ToolRequestHelp + | Method::ToolRecordStart + | Method::ToolRecordStop + | Method::ToolRecordAwait => { handle_tool_dispatch(&state, rpc_id, method, params).await } Method::ToolWaitMs => handle_wait_ms(&state.abort_registry, rpc_id, params).await, @@ -319,10 +322,19 @@ async fn handle_tool_dispatch( } }; let entry = inflight_guard.entry(); - let outcome = state - .tool_queues - .dispatch(&session_id, method, params, timeout, Some(entry)) - .await; + // `record_stop` must reach the extension while `record_await` holds the + // serial busy lock — finishing the recording unblocks await. + let outcome = if method == Method::ToolRecordStop { + state + .tool_queues + .dispatch_unlocked(&session_id, method, params, timeout, Some(entry)) + .await + } else { + state + .tool_queues + .dispatch(&session_id, method, params, timeout, Some(entry)) + .await + }; drop(inflight_guard); match outcome { Ok(v) => ResponseBody::Ok(v), diff --git a/crates/bsk-cli/src/daemon/paths.rs b/crates/bsk-cli/src/daemon/paths.rs index 1cf2bb5..f3ec24d 100644 --- a/crates/bsk-cli/src/daemon/paths.rs +++ b/crates/bsk-cli/src/daemon/paths.rs @@ -95,6 +95,11 @@ pub fn sock_path() -> Result { Ok(bsk_home()?.join("run").join("daemon.sock")) } +/// Path to the in-progress recording session state (`record-session.json`). +pub fn record_session_path() -> Result { + Ok(bsk_home()?.join("record-session.json")) +} + /// Windows named-pipe name. Include the resolved `BSK_HOME` path in the /// token so test homes and custom installs do not share a predictable /// per-username pipe. @@ -163,6 +168,10 @@ mod tests { assert_eq!(log_path().unwrap(), home.join("daemon.log")); assert_eq!(update_check_path().unwrap(), home.join("update-check.json")); assert_eq!(sock_path().unwrap(), home.join("run").join("daemon.sock")); + assert_eq!( + record_session_path().unwrap(), + home.join("record-session.json") + ); }); } } diff --git a/crates/bsk-cli/src/daemon/queue.rs b/crates/bsk-cli/src/daemon/queue.rs index 919c207..3fac0eb 100644 --- a/crates/bsk-cli/src/daemon/queue.rs +++ b/crates/bsk-cli/src/daemon/queue.rs @@ -221,6 +221,13 @@ impl ToolQueueRegistry { guard.get(sid).is_some_and(|entry| entry.accepting) } + #[cfg(test)] + fn force_busy_for_tests(&self, sid: &SessionId, busy: bool) { + let guard = self.queues.lock().expect("tool queue registry poisoned"); + let entry = guard.get(sid).expect("queue must exist"); + entry.state.lock().expect("queue state poisoned").busy = busy; + } + /// Spawn the worker task for a session id. Idempotent: spawning the /// same id twice replaces the previous queue (the previous worker /// closes when its sender drops). @@ -320,6 +327,42 @@ impl ToolQueueRegistry { outcome } + /// Forward a tool RPC to the extension without taking the + /// per-session busy lock. Used by `tool.record_stop` so a second + /// CLI can finish an in-flight `tool.record_await` (extension + /// dispatch already runs concurrent `void dispatch(msg)` handlers). + pub async fn dispatch_unlocked( + &self, + sid: &SessionId, + method: Method, + params: Value, + timeout: Duration, + inflight: Option>, + ) -> Result { + { + let guard = self.queues.lock().expect("tool queue registry poisoned"); + let Some(entry) = guard.get(sid) else { + return Err(DispatchError::SessionNotFound); + }; + if !entry.accepting { + return Err(DispatchError::SessionStopping); + } + } + let (respond_tx, _respond_rx) = oneshot::channel(); + let job = ToolJob { + method, + params, + timeout, + respond: respond_tx, + inflight, + session_id: sid.clone(), + }; + match forward_one(sid, &self.browsers, &self.sessions, &job).await { + Ok(v) => Ok(v), + Err(err) => Err(DispatchError::Rpc(err)), + } + } + /// Stop accepting new jobs for `sid`, enqueue one final control RPC, /// and wait for it behind any already-queued tools. Used by /// `session.stop` so extension teardown cannot race an in-flight @@ -789,3 +832,50 @@ mod tool_job_session_id_tests { assert_eq!(job.session_id.0, "sess-A"); } } + +#[cfg(test)] +mod dispatch_unlocked_tests { + use super::*; + use crate::daemon::browsers::BrowserRegistry; + use crate::daemon::sessions::{SessionId, SessionRegistry}; + use bsk_protocol::Method; + use serde_json::json; + use std::sync::Arc; + + #[tokio::test] + async fn unlocked_dispatch_skips_session_busy_gate() { + let browsers = Arc::new(BrowserRegistry::new()); + let sessions = Arc::new(SessionRegistry::new()); + let queues = ToolQueueRegistry::new(browsers, sessions); + let sid = SessionId("s-record".into()); + queues.spawn(sid.clone()); + queues.force_busy_for_tests(&sid, true); + + let busy = queues + .dispatch( + &sid, + Method::ToolTabList, + json!({ "session_id": "s-record" }), + Duration::from_secs(1), + None, + ) + .await; + assert!(matches!(busy, Err(DispatchError::SessionBusy))); + + let unlocked = queues + .dispatch_unlocked( + &sid, + Method::ToolRecordStop, + json!({ "session_id": "s-record" }), + Duration::from_secs(1), + None, + ) + .await; + assert!( + !matches!(unlocked, Err(DispatchError::SessionBusy)), + "record_stop must not be rejected as SessionBusy; got {unlocked:?}" + ); + // No session row → forward_one returns NotFound as Rpc. + assert!(matches!(unlocked, Err(DispatchError::Rpc(_)))); + } +} diff --git a/crates/bsk-cli/src/main.rs b/crates/bsk-cli/src/main.rs index dd720b9..cdc9e73 100644 --- a/crates/bsk-cli/src/main.rs +++ b/crates/bsk-cli/src/main.rs @@ -97,6 +97,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> { Command::WaitForNavigation(args) => cli::waits::dispatch_wait_for_navigation(args, format), Command::WaitMs(args) => cli::waits::dispatch_wait_ms(args, format), Command::RequestHelp(args) => cli::human_loop::dispatch(args, format), + Command::Record(cmd) => cli::record::dispatch(cmd, format), } } diff --git a/crates/bsk-protocol/schema/tool_record_await_params.json b/crates/bsk-protocol/schema/tool_record_await_params.json new file mode 100644 index 0000000..7a8350a --- /dev/null +++ b/crates/bsk-protocol/schema/tool_record_await_params.json @@ -0,0 +1,21 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "RecordAwaitParams", + "type": "object", + "required": [ + "session_id" + ], + "properties": { + "session_id": { + "type": "string" + }, + "timeout_ms": { + "type": [ + "integer", + "null" + ], + "format": "uint32", + "minimum": 0.0 + } + } +} diff --git a/crates/bsk-protocol/schema/tool_record_await_result.json b/crates/bsk-protocol/schema/tool_record_await_result.json new file mode 100644 index 0000000..dad5f66 --- /dev/null +++ b/crates/bsk-protocol/schema/tool_record_await_result.json @@ -0,0 +1,417 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "RecordAwaitResult", + "type": "object", + "required": [ + "trace" + ], + "properties": { + "trace": { + "$ref": "#/definitions/Trace" + } + }, + "definitions": { + "KeyModifier": { + "description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).", + "type": "string", + "enum": [ + "alt", + "ctrl", + "meta", + "shift" + ] + }, + "PageRef": { + "description": "Page context dictionary entry — referenced by steps via `page` id.", + "type": "object", + "required": [ + "id", + "url" + ], + "properties": { + "id": { + "type": "string" + }, + "title": { + "type": [ + "string", + "null" + ] + }, + "url": { + "type": "string" + } + } + }, + "SelectedOption": { + "description": "One selected option (`select` op).", + "type": "object", + "required": [ + "value" + ], + "properties": { + "label": { + "type": [ + "string", + "null" + ] + }, + "value": { + "type": "string" + } + } + }, + "Step": { + "description": "One recorded user action — discriminated union by `op`.", + "oneOf": [ + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "op", + "page", + "to" + ], + "properties": { + "effect": { + "anyOf": [ + { + "$ref": "#/definitions/StepEffect" + }, + { + "type": "null" + } + ] + }, + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "navigate" + ] + }, + "page": { + "description": "Reference into `pages[]`.", + "type": "string" + }, + "to": { + "type": "string" + } + } + }, + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "op", + "page", + "target" + ], + "properties": { + "effect": { + "anyOf": [ + { + "$ref": "#/definitions/StepEffect" + }, + { + "type": "null" + } + ] + }, + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "click" + ] + }, + "page": { + "description": "Reference into `pages[]`.", + "type": "string" + }, + "target": { + "$ref": "#/definitions/TargetDescriptor" + } + } + }, + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "op", + "page", + "target", + "value" + ], + "properties": { + "effect": { + "anyOf": [ + { + "$ref": "#/definitions/StepEffect" + }, + { + "type": "null" + } + ] + }, + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "fill" + ] + }, + "page": { + "description": "Reference into `pages[]`.", + "type": "string" + }, + "redacted": { + "type": [ + "boolean", + "null" + ] + }, + "target": { + "$ref": "#/definitions/TargetDescriptor" + }, + "value": { + "type": "string" + } + } + }, + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "op", + "page", + "selection", + "target" + ], + "properties": { + "effect": { + "anyOf": [ + { + "$ref": "#/definitions/StepEffect" + }, + { + "type": "null" + } + ] + }, + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "select" + ] + }, + "page": { + "description": "Reference into `pages[]`.", + "type": "string" + }, + "selection": { + "type": "array", + "items": { + "$ref": "#/definitions/SelectedOption" + } + }, + "target": { + "$ref": "#/definitions/TargetDescriptor" + } + } + }, + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "key", + "op", + "page" + ], + "properties": { + "effect": { + "anyOf": [ + { + "$ref": "#/definitions/StepEffect" + }, + { + "type": "null" + } + ] + }, + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "key": { + "type": "string" + }, + "modifiers": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/KeyModifier" + } + }, + "op": { + "type": "string", + "enum": [ + "press" + ] + }, + "page": { + "description": "Reference into `pages[]`.", + "type": "string" + }, + "target": { + "anyOf": [ + { + "$ref": "#/definitions/TargetDescriptor" + }, + { + "type": "null" + } + ] + } + } + } + ] + }, + "StepEffect": { + "description": "Observed navigation after a step (objective fact only).", + "type": "object", + "required": [ + "navigated_to" + ], + "properties": { + "navigated_to": { + "description": "Reference into `pages[]` for the destination page.", + "type": "string" + } + } + }, + "TargetDescriptor": { + "description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.", + "type": "object", + "required": [ + "tag" + ], + "properties": { + "name": { + "type": [ + "string", + "null" + ] + }, + "name_attr": { + "type": [ + "string", + "null" + ] + }, + "nearby_label": { + "type": [ + "string", + "null" + ] + }, + "placeholder": { + "type": [ + "string", + "null" + ] + }, + "role": { + "type": [ + "string", + "null" + ] + }, + "tag": { + "type": "string" + } + } + }, + "Trace": { + "description": "Persisted user-action trace exported by `tool.record_stop` / `await`.", + "type": "object", + "required": [ + "entry", + "pages", + "recorded_at", + "steps" + ], + "properties": { + "entry": { + "$ref": "#/definitions/TraceEntry" + }, + "pages": { + "type": "array", + "items": { + "$ref": "#/definitions/PageRef" + } + }, + "purpose": { + "description": "Optional user-provided goal from `--purpose` (metadata only).", + "type": [ + "string", + "null" + ] + }, + "recorded_at": { + "description": "RFC 3339 timestamp when recording stopped.", + "type": "string" + }, + "started_at": { + "type": [ + "string", + "null" + ] + }, + "steps": { + "type": "array", + "items": { + "$ref": "#/definitions/Step" + } + } + } + }, + "TraceEntry": { + "description": "Recording entry point — first URL the flow starts from.", + "type": "object", + "required": [ + "start_url" + ], + "properties": { + "start_url": { + "type": "string" + } + } + } + } +} diff --git a/crates/bsk-protocol/schema/tool_record_start_params.json b/crates/bsk-protocol/schema/tool_record_start_params.json new file mode 100644 index 0000000..859ebf8 --- /dev/null +++ b/crates/bsk-protocol/schema/tool_record_start_params.json @@ -0,0 +1,32 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "RecordStartParams", + "type": "object", + "required": [ + "session_id" + ], + "properties": { + "purpose": { + "type": [ + "string", + "null" + ] + }, + "session_id": { + "type": "string" + }, + "tab_id": { + "type": [ + "integer", + "null" + ], + "format": "int64" + }, + "url": { + "type": [ + "string", + "null" + ] + } + } +} diff --git a/crates/bsk-protocol/schema/tool_record_start_result.json b/crates/bsk-protocol/schema/tool_record_start_result.json new file mode 100644 index 0000000..5c04ceb --- /dev/null +++ b/crates/bsk-protocol/schema/tool_record_start_result.json @@ -0,0 +1,18 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "RecordStartResult", + "type": "object", + "required": [ + "recording", + "tab_id" + ], + "properties": { + "recording": { + "type": "boolean" + }, + "tab_id": { + "type": "integer", + "format": "int64" + } + } +} diff --git a/crates/bsk-protocol/schema/tool_record_stop_params.json b/crates/bsk-protocol/schema/tool_record_stop_params.json new file mode 100644 index 0000000..2132963 --- /dev/null +++ b/crates/bsk-protocol/schema/tool_record_stop_params.json @@ -0,0 +1,13 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "RecordStopParams", + "type": "object", + "required": [ + "session_id" + ], + "properties": { + "session_id": { + "type": "string" + } + } +} diff --git a/crates/bsk-protocol/schema/tool_record_stop_result.json b/crates/bsk-protocol/schema/tool_record_stop_result.json new file mode 100644 index 0000000..e3da733 --- /dev/null +++ b/crates/bsk-protocol/schema/tool_record_stop_result.json @@ -0,0 +1,417 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "RecordStopResult", + "type": "object", + "required": [ + "trace" + ], + "properties": { + "trace": { + "$ref": "#/definitions/Trace" + } + }, + "definitions": { + "KeyModifier": { + "description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).", + "type": "string", + "enum": [ + "alt", + "ctrl", + "meta", + "shift" + ] + }, + "PageRef": { + "description": "Page context dictionary entry — referenced by steps via `page` id.", + "type": "object", + "required": [ + "id", + "url" + ], + "properties": { + "id": { + "type": "string" + }, + "title": { + "type": [ + "string", + "null" + ] + }, + "url": { + "type": "string" + } + } + }, + "SelectedOption": { + "description": "One selected option (`select` op).", + "type": "object", + "required": [ + "value" + ], + "properties": { + "label": { + "type": [ + "string", + "null" + ] + }, + "value": { + "type": "string" + } + } + }, + "Step": { + "description": "One recorded user action — discriminated union by `op`.", + "oneOf": [ + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "op", + "page", + "to" + ], + "properties": { + "effect": { + "anyOf": [ + { + "$ref": "#/definitions/StepEffect" + }, + { + "type": "null" + } + ] + }, + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "navigate" + ] + }, + "page": { + "description": "Reference into `pages[]`.", + "type": "string" + }, + "to": { + "type": "string" + } + } + }, + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "op", + "page", + "target" + ], + "properties": { + "effect": { + "anyOf": [ + { + "$ref": "#/definitions/StepEffect" + }, + { + "type": "null" + } + ] + }, + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "click" + ] + }, + "page": { + "description": "Reference into `pages[]`.", + "type": "string" + }, + "target": { + "$ref": "#/definitions/TargetDescriptor" + } + } + }, + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "op", + "page", + "target", + "value" + ], + "properties": { + "effect": { + "anyOf": [ + { + "$ref": "#/definitions/StepEffect" + }, + { + "type": "null" + } + ] + }, + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "fill" + ] + }, + "page": { + "description": "Reference into `pages[]`.", + "type": "string" + }, + "redacted": { + "type": [ + "boolean", + "null" + ] + }, + "target": { + "$ref": "#/definitions/TargetDescriptor" + }, + "value": { + "type": "string" + } + } + }, + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "op", + "page", + "selection", + "target" + ], + "properties": { + "effect": { + "anyOf": [ + { + "$ref": "#/definitions/StepEffect" + }, + { + "type": "null" + } + ] + }, + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "select" + ] + }, + "page": { + "description": "Reference into `pages[]`.", + "type": "string" + }, + "selection": { + "type": "array", + "items": { + "$ref": "#/definitions/SelectedOption" + } + }, + "target": { + "$ref": "#/definitions/TargetDescriptor" + } + } + }, + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "key", + "op", + "page" + ], + "properties": { + "effect": { + "anyOf": [ + { + "$ref": "#/definitions/StepEffect" + }, + { + "type": "null" + } + ] + }, + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "key": { + "type": "string" + }, + "modifiers": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/KeyModifier" + } + }, + "op": { + "type": "string", + "enum": [ + "press" + ] + }, + "page": { + "description": "Reference into `pages[]`.", + "type": "string" + }, + "target": { + "anyOf": [ + { + "$ref": "#/definitions/TargetDescriptor" + }, + { + "type": "null" + } + ] + } + } + } + ] + }, + "StepEffect": { + "description": "Observed navigation after a step (objective fact only).", + "type": "object", + "required": [ + "navigated_to" + ], + "properties": { + "navigated_to": { + "description": "Reference into `pages[]` for the destination page.", + "type": "string" + } + } + }, + "TargetDescriptor": { + "description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.", + "type": "object", + "required": [ + "tag" + ], + "properties": { + "name": { + "type": [ + "string", + "null" + ] + }, + "name_attr": { + "type": [ + "string", + "null" + ] + }, + "nearby_label": { + "type": [ + "string", + "null" + ] + }, + "placeholder": { + "type": [ + "string", + "null" + ] + }, + "role": { + "type": [ + "string", + "null" + ] + }, + "tag": { + "type": "string" + } + } + }, + "Trace": { + "description": "Persisted user-action trace exported by `tool.record_stop` / `await`.", + "type": "object", + "required": [ + "entry", + "pages", + "recorded_at", + "steps" + ], + "properties": { + "entry": { + "$ref": "#/definitions/TraceEntry" + }, + "pages": { + "type": "array", + "items": { + "$ref": "#/definitions/PageRef" + } + }, + "purpose": { + "description": "Optional user-provided goal from `--purpose` (metadata only).", + "type": [ + "string", + "null" + ] + }, + "recorded_at": { + "description": "RFC 3339 timestamp when recording stopped.", + "type": "string" + }, + "started_at": { + "type": [ + "string", + "null" + ] + }, + "steps": { + "type": "array", + "items": { + "$ref": "#/definitions/Step" + } + } + } + }, + "TraceEntry": { + "description": "Recording entry point — first URL the flow starts from.", + "type": "object", + "required": [ + "start_url" + ], + "properties": { + "start_url": { + "type": "string" + } + } + } + } +} diff --git a/crates/bsk-protocol/schema/trace.json b/crates/bsk-protocol/schema/trace.json new file mode 100644 index 0000000..a1898e6 --- /dev/null +++ b/crates/bsk-protocol/schema/trace.json @@ -0,0 +1,406 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Trace", + "description": "Persisted user-action trace exported by `tool.record_stop` / `await`.", + "type": "object", + "required": [ + "entry", + "pages", + "recorded_at", + "steps" + ], + "properties": { + "entry": { + "$ref": "#/definitions/TraceEntry" + }, + "pages": { + "type": "array", + "items": { + "$ref": "#/definitions/PageRef" + } + }, + "purpose": { + "description": "Optional user-provided goal from `--purpose` (metadata only).", + "type": [ + "string", + "null" + ] + }, + "recorded_at": { + "description": "RFC 3339 timestamp when recording stopped.", + "type": "string" + }, + "started_at": { + "type": [ + "string", + "null" + ] + }, + "steps": { + "type": "array", + "items": { + "$ref": "#/definitions/Step" + } + } + }, + "definitions": { + "KeyModifier": { + "description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).", + "type": "string", + "enum": [ + "alt", + "ctrl", + "meta", + "shift" + ] + }, + "PageRef": { + "description": "Page context dictionary entry — referenced by steps via `page` id.", + "type": "object", + "required": [ + "id", + "url" + ], + "properties": { + "id": { + "type": "string" + }, + "title": { + "type": [ + "string", + "null" + ] + }, + "url": { + "type": "string" + } + } + }, + "SelectedOption": { + "description": "One selected option (`select` op).", + "type": "object", + "required": [ + "value" + ], + "properties": { + "label": { + "type": [ + "string", + "null" + ] + }, + "value": { + "type": "string" + } + } + }, + "Step": { + "description": "One recorded user action — discriminated union by `op`.", + "oneOf": [ + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "op", + "page", + "to" + ], + "properties": { + "effect": { + "anyOf": [ + { + "$ref": "#/definitions/StepEffect" + }, + { + "type": "null" + } + ] + }, + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "navigate" + ] + }, + "page": { + "description": "Reference into `pages[]`.", + "type": "string" + }, + "to": { + "type": "string" + } + } + }, + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "op", + "page", + "target" + ], + "properties": { + "effect": { + "anyOf": [ + { + "$ref": "#/definitions/StepEffect" + }, + { + "type": "null" + } + ] + }, + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "click" + ] + }, + "page": { + "description": "Reference into `pages[]`.", + "type": "string" + }, + "target": { + "$ref": "#/definitions/TargetDescriptor" + } + } + }, + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "op", + "page", + "target", + "value" + ], + "properties": { + "effect": { + "anyOf": [ + { + "$ref": "#/definitions/StepEffect" + }, + { + "type": "null" + } + ] + }, + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "fill" + ] + }, + "page": { + "description": "Reference into `pages[]`.", + "type": "string" + }, + "redacted": { + "type": [ + "boolean", + "null" + ] + }, + "target": { + "$ref": "#/definitions/TargetDescriptor" + }, + "value": { + "type": "string" + } + } + }, + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "op", + "page", + "selection", + "target" + ], + "properties": { + "effect": { + "anyOf": [ + { + "$ref": "#/definitions/StepEffect" + }, + { + "type": "null" + } + ] + }, + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "select" + ] + }, + "page": { + "description": "Reference into `pages[]`.", + "type": "string" + }, + "selection": { + "type": "array", + "items": { + "$ref": "#/definitions/SelectedOption" + } + }, + "target": { + "$ref": "#/definitions/TargetDescriptor" + } + } + }, + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "key", + "op", + "page" + ], + "properties": { + "effect": { + "anyOf": [ + { + "$ref": "#/definitions/StepEffect" + }, + { + "type": "null" + } + ] + }, + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "key": { + "type": "string" + }, + "modifiers": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/KeyModifier" + } + }, + "op": { + "type": "string", + "enum": [ + "press" + ] + }, + "page": { + "description": "Reference into `pages[]`.", + "type": "string" + }, + "target": { + "anyOf": [ + { + "$ref": "#/definitions/TargetDescriptor" + }, + { + "type": "null" + } + ] + } + } + } + ] + }, + "StepEffect": { + "description": "Observed navigation after a step (objective fact only).", + "type": "object", + "required": [ + "navigated_to" + ], + "properties": { + "navigated_to": { + "description": "Reference into `pages[]` for the destination page.", + "type": "string" + } + } + }, + "TargetDescriptor": { + "description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.", + "type": "object", + "required": [ + "tag" + ], + "properties": { + "name": { + "type": [ + "string", + "null" + ] + }, + "name_attr": { + "type": [ + "string", + "null" + ] + }, + "nearby_label": { + "type": [ + "string", + "null" + ] + }, + "placeholder": { + "type": [ + "string", + "null" + ] + }, + "role": { + "type": [ + "string", + "null" + ] + }, + "tag": { + "type": "string" + } + } + }, + "TraceEntry": { + "description": "Recording entry point — first URL the flow starts from.", + "type": "object", + "required": [ + "start_url" + ], + "properties": { + "start_url": { + "type": "string" + } + } + } + } +} diff --git a/crates/bsk-protocol/schema/trace_step.json b/crates/bsk-protocol/schema/trace_step.json new file mode 100644 index 0000000..046925e --- /dev/null +++ b/crates/bsk-protocol/schema/trace_step.json @@ -0,0 +1,328 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Step", + "description": "One recorded user action — discriminated union by `op`.", + "oneOf": [ + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "op", + "page", + "to" + ], + "properties": { + "effect": { + "anyOf": [ + { + "$ref": "#/definitions/StepEffect" + }, + { + "type": "null" + } + ] + }, + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "navigate" + ] + }, + "page": { + "description": "Reference into `pages[]`.", + "type": "string" + }, + "to": { + "type": "string" + } + } + }, + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "op", + "page", + "target" + ], + "properties": { + "effect": { + "anyOf": [ + { + "$ref": "#/definitions/StepEffect" + }, + { + "type": "null" + } + ] + }, + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "click" + ] + }, + "page": { + "description": "Reference into `pages[]`.", + "type": "string" + }, + "target": { + "$ref": "#/definitions/TargetDescriptor" + } + } + }, + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "op", + "page", + "target", + "value" + ], + "properties": { + "effect": { + "anyOf": [ + { + "$ref": "#/definitions/StepEffect" + }, + { + "type": "null" + } + ] + }, + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "fill" + ] + }, + "page": { + "description": "Reference into `pages[]`.", + "type": "string" + }, + "redacted": { + "type": [ + "boolean", + "null" + ] + }, + "target": { + "$ref": "#/definitions/TargetDescriptor" + }, + "value": { + "type": "string" + } + } + }, + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "op", + "page", + "selection", + "target" + ], + "properties": { + "effect": { + "anyOf": [ + { + "$ref": "#/definitions/StepEffect" + }, + { + "type": "null" + } + ] + }, + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "op": { + "type": "string", + "enum": [ + "select" + ] + }, + "page": { + "description": "Reference into `pages[]`.", + "type": "string" + }, + "selection": { + "type": "array", + "items": { + "$ref": "#/definitions/SelectedOption" + } + }, + "target": { + "$ref": "#/definitions/TargetDescriptor" + } + } + }, + { + "description": "Fields shared by every step variant (flattened in JSON).", + "type": "object", + "required": [ + "id", + "key", + "op", + "page" + ], + "properties": { + "effect": { + "anyOf": [ + { + "$ref": "#/definitions/StepEffect" + }, + { + "type": "null" + } + ] + }, + "id": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + }, + "key": { + "type": "string" + }, + "modifiers": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/KeyModifier" + } + }, + "op": { + "type": "string", + "enum": [ + "press" + ] + }, + "page": { + "description": "Reference into `pages[]`.", + "type": "string" + }, + "target": { + "anyOf": [ + { + "$ref": "#/definitions/TargetDescriptor" + }, + { + "type": "null" + } + ] + } + } + } + ], + "definitions": { + "KeyModifier": { + "description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).", + "type": "string", + "enum": [ + "alt", + "ctrl", + "meta", + "shift" + ] + }, + "SelectedOption": { + "description": "One selected option (`select` op).", + "type": "object", + "required": [ + "value" + ], + "properties": { + "label": { + "type": [ + "string", + "null" + ] + }, + "value": { + "type": "string" + } + } + }, + "StepEffect": { + "description": "Observed navigation after a step (objective fact only).", + "type": "object", + "required": [ + "navigated_to" + ], + "properties": { + "navigated_to": { + "description": "Reference into `pages[]` for the destination page.", + "type": "string" + } + } + }, + "TargetDescriptor": { + "description": "Stable semantic handle for an interacted element.\n\n`name` and `nearby_label` are **untrusted page text**.", + "type": "object", + "required": [ + "tag" + ], + "properties": { + "name": { + "type": [ + "string", + "null" + ] + }, + "name_attr": { + "type": [ + "string", + "null" + ] + }, + "nearby_label": { + "type": [ + "string", + "null" + ] + }, + "placeholder": { + "type": [ + "string", + "null" + ] + }, + "role": { + "type": [ + "string", + "null" + ] + }, + "tag": { + "type": "string" + } + } + } + } +} diff --git a/crates/bsk-protocol/src/bin/dump-schema.rs b/crates/bsk-protocol/src/bin/dump-schema.rs index d20efa3..f226e44 100644 --- a/crates/bsk-protocol/src/bin/dump-schema.rs +++ b/crates/bsk-protocol/src/bin/dump-schema.rs @@ -100,4 +100,13 @@ fn main() { dump!(WaitMsResult, "tool_wait_ms_result"); dump!(RequestHelpParams, "tool_request_help_params"); dump!(RequestHelpResult, "tool_request_help_result"); + + dump!(Trace, "trace"); + dump!(Step, "trace_step"); + dump!(RecordStartParams, "tool_record_start_params"); + dump!(RecordStartResult, "tool_record_start_result"); + dump!(RecordStopParams, "tool_record_stop_params"); + dump!(RecordStopResult, "tool_record_stop_result"); + dump!(RecordAwaitParams, "tool_record_await_params"); + dump!(RecordAwaitResult, "tool_record_await_result"); } diff --git a/crates/bsk-protocol/src/method.rs b/crates/bsk-protocol/src/method.rs index e54a66a..d4148c7 100644 --- a/crates/bsk-protocol/src/method.rs +++ b/crates/bsk-protocol/src/method.rs @@ -74,6 +74,12 @@ pub enum Method { ToolWaitMs, #[serde(rename = "tool.request_help")] ToolRequestHelp, + #[serde(rename = "tool.record_start")] + ToolRecordStart, + #[serde(rename = "tool.record_stop")] + ToolRecordStop, + #[serde(rename = "tool.record_await")] + ToolRecordAwait, #[serde(rename = "cancel")] Cancel, @@ -121,9 +127,16 @@ impl Method { | Method::ToolFill | Method::ToolPress | Method::ToolSelect - | Method::ToolEvaluate => true, + | Method::ToolEvaluate + // May navigate via optional `url` and changes Agent Window + // chrome; gate behind pending-interrupt like other writes. + | Method::ToolRecordStart => true, // Read-only tool calls — transparent. + // + // `record_stop` / `record_await` observe / finish a recording + // without driving new automation gestures, so they stay + // ungated (teardown after interrupt must still work). Method::ToolTabList | Method::ToolSnapshot | Method::ToolGetHtml @@ -132,7 +145,9 @@ impl Method { | Method::ToolNetwork | Method::ToolWaitForNavigation | Method::ToolWaitMs - | Method::ToolRequestHelp => false, + | Method::ToolRequestHelp + | Method::ToolRecordStop + | Method::ToolRecordAwait => false, // Session lifecycle — not gated. Method::SessionStart @@ -218,6 +233,13 @@ mod tests { assert!(Method::ToolPress.is_mutating()); assert!(Method::ToolSelect.is_mutating()); assert!(Method::ToolEvaluate.is_mutating()); + assert!(Method::ToolRecordStart.is_mutating()); + } + + #[test] + fn is_mutating_classifies_record_stop_await_as_non_mutating() { + assert!(!Method::ToolRecordStop.is_mutating()); + assert!(!Method::ToolRecordAwait.is_mutating()); } #[test] diff --git a/crates/bsk-protocol/src/tools/mod.rs b/crates/bsk-protocol/src/tools/mod.rs index 1103cd0..8a0c319 100644 --- a/crates/bsk-protocol/src/tools/mod.rs +++ b/crates/bsk-protocol/src/tools/mod.rs @@ -1,4 +1,4 @@ -//! Typed params/results for the 21 `tool.*` RPC methods (§7). +//! Typed params/results for the `tool.*` RPC methods (§7). pub mod console; pub mod dialog; @@ -7,6 +7,7 @@ pub mod interaction; pub mod navigation; pub mod network; pub mod observation; +pub mod record; pub mod script; pub mod session; pub mod tabs; @@ -19,6 +20,7 @@ pub use interaction::*; pub use navigation::*; pub use network::*; pub use observation::*; +pub use record::*; pub use script::*; pub use session::*; pub use tabs::*; diff --git a/crates/bsk-protocol/src/tools/record.rs b/crates/bsk-protocol/src/tools/record.rs new file mode 100644 index 0000000..21ee1e6 --- /dev/null +++ b/crates/bsk-protocol/src/tools/record.rs @@ -0,0 +1,458 @@ +//! Semantic user-action recording (`tool.record_start` / `stop` / `await`). +//! +//! Trace v2 is a **record-only** log of user actions: what was clicked, filled, +//! selected, and where navigation occurred. No LLM runs during recording, so +//! variable-vs-constant classification is **not** stored — executing agents +//! infer that at run time from raw values and control names. + +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +use super::interaction::KeyModifier; + +// --------------------------------------------------------------------------- +// Target +// --------------------------------------------------------------------------- + +/// Stable semantic handle for an interacted element. +/// +/// `name` and `nearby_label` are **untrusted page text**. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct TargetDescriptor { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub role: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + pub tag: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name_attr: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub placeholder: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub nearby_label: Option, +} + +// --------------------------------------------------------------------------- +// Trace envelope +// --------------------------------------------------------------------------- + +/// Recording entry point — first URL the flow starts from. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct TraceEntry { + pub start_url: String, +} + +/// Page context dictionary entry — referenced by steps via `page` id. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct PageRef { + pub id: String, + pub url: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub title: Option, +} + +/// One selected option (`select` op). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct SelectedOption { + pub value: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub label: Option, +} + +/// Observed navigation after a step (objective fact only). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct StepEffect { + /// Reference into `pages[]` for the destination page. + pub navigated_to: String, +} + +/// Fields shared by every step variant (flattened in JSON). +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct StepCommon { + pub id: u32, + /// Reference into `pages[]`. + pub page: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub effect: Option, +} + +// --------------------------------------------------------------------------- +// Step op-specific payloads +// --------------------------------------------------------------------------- + +/// One recorded user action — discriminated union by `op`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "op", rename_all = "snake_case")] +pub enum Step { + Navigate { + #[serde(flatten)] + common: StepCommon, + to: String, + }, + Click { + #[serde(flatten)] + common: StepCommon, + target: TargetDescriptor, + }, + Fill { + #[serde(flatten)] + common: StepCommon, + target: TargetDescriptor, + value: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + redacted: Option, + }, + Select { + #[serde(flatten)] + common: StepCommon, + target: TargetDescriptor, + selection: Vec, + }, + Press { + #[serde(flatten)] + common: StepCommon, + key: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + modifiers: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + target: Option, + }, +} + +// --------------------------------------------------------------------------- +// Trace root +// --------------------------------------------------------------------------- + +/// Persisted user-action trace exported by `tool.record_stop` / `await`. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct Trace { + /// RFC 3339 timestamp when recording stopped. + pub recorded_at: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub started_at: Option, + /// Optional user-provided goal from `--purpose` (metadata only). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub purpose: Option, + pub entry: TraceEntry, + pub pages: Vec, + pub steps: Vec, +} + +// --------------------------------------------------------------------------- +// RPC params / results +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct RecordStartParams { + pub session_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tab_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub url: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub purpose: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct RecordStartResult { + pub tab_id: i64, + pub recording: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct RecordStopParams { + pub session_id: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct RecordStopResult { + pub trace: Trace, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct RecordAwaitParams { + pub session_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub timeout_ms: Option, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)] +pub struct RecordAwaitResult { + pub trace: Trace, +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn sample_common(id: u32) -> StepCommon { + StepCommon { + id, + page: "p1".into(), + effect: None, + } + } + + fn sample_target() -> TargetDescriptor { + TargetDescriptor { + role: Some("button".into()), + name: Some("发布".into()), + tag: "button".into(), + name_attr: None, + placeholder: None, + nearby_label: None, + } + } + + fn sample_trace() -> Trace { + Trace { + recorded_at: "2026-07-17T09:01:10Z".into(), + started_at: Some("2026-07-17T09:00:00Z".into()), + purpose: Some("发布一篇文章".into()), + entry: TraceEntry { + start_url: "https://x.com/editor".into(), + }, + pages: vec![ + PageRef { + id: "p1".into(), + url: "https://x.com/editor".into(), + title: Some("写文章".into()), + }, + PageRef { + id: "p2".into(), + url: "https://x.com/p/99".into(), + title: None, + }, + ], + steps: vec![ + Step::Fill { + common: sample_common(1), + target: TargetDescriptor { + role: Some("textbox".into()), + name: Some("标题".into()), + tag: "input".into(), + name_attr: None, + placeholder: None, + nearby_label: None, + }, + value: "我的第一篇文章".into(), + redacted: None, + }, + Step::Select { + common: sample_common(3), + target: TargetDescriptor { + role: Some("combobox".into()), + name: Some("分类".into()), + tag: "select".into(), + name_attr: None, + placeholder: None, + nearby_label: None, + }, + selection: vec![SelectedOption { + value: "tech".into(), + label: Some("技术分享".into()), + }], + }, + Step::Click { + common: StepCommon { + effect: Some(StepEffect { + navigated_to: "p2".into(), + }), + ..sample_common(4) + }, + target: sample_target(), + }, + ], + } + } + + #[test] + fn step_click_round_trips() { + let step = Step::Click { + common: sample_common(1), + target: sample_target(), + }; + let v = serde_json::to_value(&step).unwrap(); + assert_eq!(v.get("op").and_then(|v| v.as_str()), Some("click")); + assert_eq!(v.get("page").and_then(|v| v.as_str()), Some("p1")); + assert!(v.get("key").is_none()); + let round: Step = serde_json::from_value(v).unwrap(); + assert_eq!(round, step); + } + + #[test] + fn step_fill_with_raw_value_round_trips() { + let step = Step::Fill { + common: sample_common(2), + target: TargetDescriptor { + role: Some("textbox".into()), + name: Some("服务名称".into()), + tag: "input".into(), + name_attr: Some("serviceName".into()), + placeholder: None, + nearby_label: None, + }, + value: "my-svc".into(), + redacted: None, + }; + let v = serde_json::to_value(&step).unwrap(); + assert_eq!(v["op"], "fill"); + assert_eq!(v["value"], "my-svc"); + let round: Step = serde_json::from_value(v).unwrap(); + assert_eq!(round, step); + } + + #[test] + fn step_fill_password_is_redacted() { + let step = Step::Fill { + common: sample_common(1), + target: TargetDescriptor { + role: Some("textbox".into()), + name: Some("密码".into()), + tag: "input".into(), + name_attr: None, + placeholder: None, + nearby_label: None, + }, + value: "***".into(), + redacted: Some(true), + }; + let v = serde_json::to_value(&step).unwrap(); + assert_eq!(v["value"], "***"); + assert_eq!(v["redacted"], true); + } + + #[test] + fn step_select_uses_object_array() { + let step = Step::Select { + common: sample_common(3), + target: TargetDescriptor { + role: Some("combobox".into()), + name: Some("分类".into()), + tag: "select".into(), + name_attr: None, + placeholder: None, + nearby_label: None, + }, + selection: vec![SelectedOption { + value: "tech".into(), + label: Some("技术分享".into()), + }], + }; + let v = serde_json::to_value(&step).unwrap(); + assert_eq!(v["selection"][0]["value"], "tech"); + assert_eq!(v["selection"][0]["label"], "技术分享"); + let round: Step = serde_json::from_value(v).unwrap(); + assert_eq!(round, step); + } + + #[test] + fn step_navigate_round_trips() { + let step = Step::Navigate { + common: sample_common(1), + to: "https://example.com".into(), + }; + let v = serde_json::to_value(&step).unwrap(); + assert_eq!(v["op"], "navigate"); + assert_eq!(v["to"], "https://example.com"); + let round: Step = serde_json::from_value(v).unwrap(); + assert_eq!(round, step); + } + + #[test] + fn step_press_with_modifiers_round_trips() { + let step = Step::Press { + common: StepCommon { + effect: Some(StepEffect { + navigated_to: "p2".into(), + }), + ..sample_common(2) + }, + key: "Enter".into(), + modifiers: Some(vec![KeyModifier::Ctrl, KeyModifier::Shift]), + target: Some(sample_target()), + }; + let v = serde_json::to_value(&step).unwrap(); + assert_eq!(v["key"], "Enter"); + assert_eq!(v["modifiers"], json!(["ctrl", "shift"])); + assert_eq!(v["effect"]["navigated_to"], "p2"); + let round: Step = serde_json::from_value(v).unwrap(); + assert_eq!(round, step); + } + + #[test] + fn trace_record_only_round_trips() { + let trace = sample_trace(); + let v = serde_json::to_value(&trace).unwrap(); + assert!(v.get("version").is_none()); + assert_eq!( + v.get("purpose").and_then(|v| v.as_str()), + Some("发布一篇文章") + ); + assert!(v.get("parameters").is_none()); + assert!(v.get("site").is_none()); + assert!(v.get("goal").is_none()); + assert_eq!( + v.get("entry") + .and_then(|e| e.get("start_url")) + .and_then(|u| u.as_str()), + Some("https://x.com/editor") + ); + let round: Trace = serde_json::from_value(v).unwrap(); + assert_eq!(round, trace); + } + + #[test] + fn discriminated_union_click_ignores_extra_key_field() { + let bad = json!({ + "op": "click", + "id": 1, + "page": "p1", + "target": { "tag": "button", "name": "OK" }, + "key": "Enter" + }); + let step: Step = serde_json::from_value(bad).unwrap(); + assert!(matches!(step, Step::Click { .. })); + let v = serde_json::to_value(&step).unwrap(); + assert!(v.get("key").is_none()); + } + + #[test] + fn extension_trace_deserializes() { + let v = json!({ + "recorded_at": "2026-07-21T08:00:00Z", + "started_at": "2026-07-21T07:59:00Z", + "purpose": "demo", + "entry": { "start_url": "https://example.com/editor" }, + "pages": [ + { "id": "p1", "url": "https://example.com/editor" }, + { "id": "p2", "url": "https://example.com/p/99" } + ], + "steps": [ + { + "op": "fill", + "id": 1, + "page": "p1", + "target": { "tag": "input", "role": "textbox", "name": "标题" }, + "value": "hello" + }, + { + "op": "click", + "id": 2, + "page": "p1", + "target": { "tag": "button", "role": "button", "name": "发布" }, + "effect": { "navigated_to": "p2" } + } + ] + }); + let trace: Trace = serde_json::from_value(v).unwrap(); + assert_eq!(trace.entry.start_url, "https://example.com/editor"); + assert_eq!(trace.pages.len(), 2); + assert_eq!(trace.steps.len(), 2); + } +} diff --git a/packages/i18n/src/locales/en-US/extension.json b/packages/i18n/src/locales/en-US/extension.json index aa275ae..f5c548c 100644 --- a/packages/i18n/src/locales/en-US/extension.json +++ b/packages/i18n/src/locales/en-US/extension.json @@ -68,5 +68,9 @@ "expand": "Expand", "dragHandle": "Drag to move", "notificationTitle": "BrowserSkill: Agent needs your help" + }, + "recordOverlay": { + "recording": "Recording your actions", + "finish": "Finish" } } diff --git a/packages/i18n/src/locales/zh-CN/extension.json b/packages/i18n/src/locales/zh-CN/extension.json index b9e6415..d76d90c 100644 --- a/packages/i18n/src/locales/zh-CN/extension.json +++ b/packages/i18n/src/locales/zh-CN/extension.json @@ -68,5 +68,9 @@ "expand": "展开", "dragHandle": "拖动以移动", "notificationTitle": "BrowserSkill：Agent 需要你的帮助" + }, + "recordOverlay": { + "recording": "正在录制用户操作", + "finish": "结束" } } diff --git a/skill/SKILL.md b/skill/SKILL.md index 5edf9ea..d68b40f 100644 --- a/skill/SKILL.md +++ b/skill/SKILL.md @@ -46,6 +46,21 @@ Optional: `bsk session start --browser ` when multiple bro Emergency cleanup: `bsk session stop --all` or the Agent Window overlay **Stop all**. +## Stop when the goal is met + +Every task is a **bounded goal**, not open-ended browsing. The goal may come from the user's request, a recorded `trace.json`, or both. + +1. **Define success first** — one concrete, observable condition derived from the user's words, `purpose`, or the last meaningful step in a trace (e.g. "form submitted", "item added to cart", "playback started"). +2. **Take the shortest path** — snapshot → act → at most one check. Do not wander, re-try unrelated actions, or stack exploratory steps. +3. **Stop as soon as success is reached** — run `bsk session stop ` immediately unless the user explicitly asked to keep the session open (e.g. "don't close yet", "keep browsing"). +4. **No post-success work** — once the goal is met, do not click, refresh, navigate, re-search, switch tabs, or "double-check" that it worked. Further verification is a new task. +5. **When blocked, pause — do not brute-force** — if the page requires human input (login, captcha, OTP, payment confirmation) or an action fails twice with no progress, call `bsk request-help` instead of retrying blindly. See **Ask the human for help** below. +6. **When unsure** — at most one extra `bsk snapshot`. If success looks met, stop. If not, ask the user; do not keep clicking. + +**With a trace:** replay steps in order using `target` role/name/tag and raw `value`/`selection` fields. After the last step (or when its `effect.navigated_to` / success hint is satisfied), apply rules 3–4 immediately. The trace guides execution; it does not extend control beyond the goal. + +**Without a trace:** the user's request *is* the success condition. Satisfying it ends the task — same stop rules apply. + ## Core interaction loop Write operations only affect tabs in the **Agent Window** (or tabs you **borrowed** into it). @@ -191,6 +206,23 @@ acts. The result `outcome` is one of: `note` carries any text the user typed back. `resolved_targets` reports which refs/selectors matched a live element. +### Recording — `bsk record` + +Capture the user's own actions in the Agent Window to a `trace.json`, for later LLM-driven automation: + +```bash +bsk record start --url https://… [--purpose "publish a wiki doc"] [--output trace.json] +# `--url` is required on a fresh session (about:blank cannot host the content script). +# Blocks until the user clicks Finish in the recording panel, then writes ./trace.json and closes the window. + +bsk record stop [--output trace.json] # terminal fallback if the browser panel is unavailable +``` + +- The trace is a **record-only action log** (a `pages[]` dictionary + `navigate`/`click`/`fill`/`select`/`press` steps with `target` descriptors). It records *what the user did*; deciding which inputs are variable is left to the executing agent. +- `--purpose` is optional context metadata; it does **not** change what gets captured. +- There is **no** `bsk replay` — to redo a flow, read the trace and reuse the existing `session` / `snapshot` / `@eN` / `click` / `fill` tools. Follow **Stop when the goal is met**. +- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text. + ## Error handling ### Exit codes (`echo $?` after `bsk …`) @@ -221,8 +253,9 @@ Always **`bsk session stop `** in a `finally`-style path so the Agent Window 1. **No token theft** — do not `bsk evaluate` on sensitive sites to read `localStorage`, cookies, or auth headers for exfiltration. 2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks. 3. **No skip stop** — always `bsk session stop `; never assume idle timeout will clean up. -4. **No observe escalation before snapshot** — use `bsk snapshot` first; only use `bsk get-html` or `bsk screenshot` when the snapshot is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot ref — never skip snapshot just to grab a visual. -5. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces. +4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open. +5. **No observe escalation before snapshot** — use `bsk snapshot` first; only use `bsk get-html` or `bsk screenshot` when the snapshot is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot ref — never skip snapshot just to grab a visual. +6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces. ---