feat: add bsk record for semantic user-action traces

Capture user actions in the Agent Window and export a record-only
trace (pages[] + navigate/click/fill/select/press steps with target
descriptors). Adds the record overlay UI aligned with the control
overlay, and wires record start/stop/await through CLI, daemon,
protocol, and extension.
This commit is contained in:
“haonan”
2026-07-22 11:27:30 +08:00
parent ccf3c32140
commit ac028a7a18
47 changed files with 6038 additions and 46 deletions
+6
View File
@@ -19,6 +19,8 @@ pub mod interaction;
pub mod logs;
pub mod navigate;
pub mod network;
pub mod record;
pub mod record_state;
pub mod render_error;
pub mod screenshot;
pub mod session;
@@ -39,6 +41,7 @@ use crate::cli::install_skill::InstallSkillArgs;
use crate::cli::interaction::{ClickArgs, FillArgs, PressArgs, SelectArgs};
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
use crate::cli::network::NetworkArgs;
use crate::cli::record::RecordCmd;
use crate::cli::screenshot::ScreenshotArgs;
use crate::cli::session::SessionCmd;
use crate::cli::snapshot::SnapshotArgs;
@@ -169,6 +172,9 @@ pub enum Command {
/// Ask the human to complete an in-page step (captcha / login / confirm).
#[command(name = "request-help")]
RequestHelp(RequestHelpArgs),
/// Record user actions into a semantic `trace.json` textbook for LLMs.
Record(RecordCmd),
}
#[derive(Debug, Clone, Args, Default)]
+256
View File
@@ -0,0 +1,256 @@
//! `bsk record start|stop` — capture user actions in the Agent Window.
use std::fs;
use std::path::PathBuf;
use std::time::Duration;
use anyhow::Context;
use bsk_protocol::Method;
use bsk_protocol::tools::{
RecordAwaitParams, RecordAwaitResult, RecordStartParams, RecordStartResult, RecordStopParams,
RecordStopResult, Trace,
};
use clap::{Args, Subcommand};
use crate::cli::TOOL_IPC_TIMEOUT;
use crate::cli::business_rpc;
use crate::cli::ensure_daemon::ensure_daemon;
use crate::cli::error::{CliError, Format};
use crate::cli::record_state;
use crate::cli::session::{start_session, stop_session};
/// Max wait for the user to click 结束 in the browser (24 hours).
const RECORD_AWAIT_TIMEOUT_MS: u32 = 86_400_000;
/// `bsk record …` subcommand tree.
#[derive(Debug, Clone, Args)]
pub struct RecordCmd {
#[command(subcommand)]
pub sub: RecordSub,
}
#[derive(Debug, Clone, Subcommand)]
pub enum RecordSub {
/// Open the Agent Window, record user actions, and block until finished in the browser.
Start(RecordStartArgs),
/// Stop recording from the terminal (fallback), write trace JSON, and close the window.
/// Works even while `record start` is blocked in `record_await` (daemon forwards
/// `tool.record_stop` without the per-session busy gate).
Stop(RecordStopArgs),
}
#[derive(Debug, Clone, Args)]
pub struct RecordStartArgs {
#[arg(long = "tab-id")]
pub tab_id: Option<i64>,
/// Navigate to this http(s) URL before recording (required for a fresh
/// session — Agent Window starts on about:blank, which cannot host the
/// content script).
#[arg(long)]
pub url: Option<String>,
/// Optional goal text stored on the exported trace for LLM context.
#[arg(long)]
pub purpose: Option<String>,
/// Output path for the trace JSON (default `./trace.json`).
#[arg(long, default_value = "trace.json")]
pub output: PathBuf,
}
#[derive(Debug, Clone, Args)]
pub struct RecordStopArgs {
/// Output path for the trace JSON (default `./trace.json`).
#[arg(long, default_value = "trace.json")]
pub output: PathBuf,
}
pub fn dispatch(cmd: RecordCmd, format: Format) -> Result<(), CliError> {
match cmd.sub {
RecordSub::Start(args) => dispatch_start(args, format),
RecordSub::Stop(args) => dispatch_stop(args, format),
}
}
fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError> {
if record_state::read().is_ok() {
return Err(CliError::Local(anyhow::anyhow!(
"a recording is already in progress; run `bsk record stop` first"
)));
}
let info = ensure_daemon().context("ensure daemon is running")?;
let session = start_session(info.sock_path.clone(), None)?;
let start_params = RecordStartParams {
session_id: session.session_id.clone(),
tab_id: args.tab_id,
url: args.url,
purpose: args.purpose.clone(),
};
let start_result = business_rpc::call::<RecordStartParams, RecordStartResult>(
info.sock_path.clone(),
"record-start",
Method::ToolRecordStart,
Some(start_params),
TOOL_IPC_TIMEOUT,
);
let start_result = match start_result {
Ok(result) => result,
Err(err) => {
let _ = stop_session(info.sock_path, &session.session_id);
return Err(err);
}
};
if let Err(err) = record_state::write(&session.session_id) {
let _ = stop_session(info.sock_path.clone(), &session.session_id);
return Err(CliError::Local(err));
}
if format == Format::Human {
println!(
"recording on tab={} — click 结束 in the browser when done",
start_result.tab_id
);
}
let await_params = RecordAwaitParams {
session_id: session.session_id.clone(),
timeout_ms: Some(RECORD_AWAIT_TIMEOUT_MS),
};
let await_result = business_rpc::call::<RecordAwaitParams, RecordAwaitResult>(
info.sock_path.clone(),
"record-await",
Method::ToolRecordAwait,
Some(await_params),
record_await_ipc_timeout(RECORD_AWAIT_TIMEOUT_MS),
);
// Keep write/render inside a Result so `?` cannot skip session teardown.
let run_result: Result<(), CliError> = match await_result {
Ok(result) => (|| {
write_trace_file(&args.output, &result.trace)?;
render_finish(&result.trace, &args.output, format)
})(),
Err(err) => Err(err),
};
let session_stop_result = stop_session(info.sock_path, &session.session_id);
record_state::clear();
run_result?;
session_stop_result?;
Ok(())
}
fn dispatch_stop(args: RecordStopArgs, format: Format) -> Result<(), CliError> {
let state = record_state::read().map_err(CliError::Local)?;
let info = ensure_daemon().context("ensure daemon is running")?;
let session_id = state.session_id.clone();
let run_result: Result<(), CliError> = (|| {
let params = RecordStopParams {
session_id: session_id.clone(),
};
let result = business_rpc::call::<RecordStopParams, RecordStopResult>(
info.sock_path.clone(),
"record-stop",
Method::ToolRecordStop,
Some(params),
TOOL_IPC_TIMEOUT,
)?;
write_trace_file(&args.output, &result.trace)?;
render_stop(&result, &args.output, format)
})();
let session_stop_result = stop_session(info.sock_path, &session_id);
record_state::clear();
run_result?;
session_stop_result?;
Ok(())
}
fn record_await_ipc_timeout(timeout_ms: u32) -> Duration {
Duration::from_millis(u64::from(timeout_ms))
.checked_add(Duration::from_secs(15))
.unwrap_or(Duration::from_secs(u64::from(timeout_ms / 1_000) + 15))
}
fn write_trace_file(output: &PathBuf, trace: &Trace) -> Result<(), CliError> {
let json = serde_json::to_string_pretty(trace)
.context("serialize trace JSON")
.map_err(CliError::Local)?;
if let Some(parent) = output.parent() {
if !parent.as_os_str().is_empty() {
fs::create_dir_all(parent)
.with_context(|| format!("create output directory {}", parent.display()))
.map_err(CliError::Local)?;
}
}
fs::write(output, format!("{json}\n"))
.with_context(|| format!("write trace to {}", output.display()))
.map_err(CliError::Local)?;
Ok(())
}
fn render_finish(trace: &Trace, output: &PathBuf, format: Format) -> Result<(), CliError> {
match format {
Format::Json => {
println!(
"{}",
serde_json::to_string(&serde_json::json!({
"output": output,
"trace": trace,
"window_closed": true,
}))
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
);
}
Format::Human => {
println!("saved {} steps to {}", trace.steps.len(), output.display());
}
}
Ok(())
}
fn render_stop(
result: &RecordStopResult,
output: &PathBuf,
format: Format,
) -> Result<(), CliError> {
render_finish(&result.trace, output, format)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn default_output_is_trace_json() {
let args = RecordStopArgs {
output: PathBuf::from("trace.json"),
};
assert_eq!(args.output, PathBuf::from("trace.json"));
}
#[test]
fn start_args_default_output_is_trace_json() {
let args = RecordStartArgs {
tab_id: None,
url: None,
purpose: None,
output: PathBuf::from("trace.json"),
};
assert_eq!(args.output, PathBuf::from("trace.json"));
}
#[test]
fn record_await_ipc_timeout_covers_long_wait() {
let got = record_await_ipc_timeout(RECORD_AWAIT_TIMEOUT_MS);
assert!(got >= Duration::from_secs(86_400));
}
}
+117
View File
@@ -0,0 +1,117 @@
//! Persist the session id between `bsk record start` and `bsk record stop`.
use std::fs;
use std::time::{SystemTime, UNIX_EPOCH};
use anyhow::{Context, Result};
use serde::{Deserialize, Serialize};
use crate::daemon::paths;
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct RecordSessionState {
pub session_id: String,
pub started_at: String,
}
pub fn write(session_id: &str) -> Result<()> {
let path = paths::record_session_path()?;
paths::ensure_bsk_home()?;
if path.exists() {
return Err(anyhow::anyhow!(
"a recording is already in progress; run `bsk record stop` first"
));
}
let state = RecordSessionState {
session_id: session_id.to_string(),
started_at: started_at_unix_ms(),
};
let json = serde_json::to_string_pretty(&state).context("serialize record session state")?;
fs::write(&path, format!("{json}\n")).with_context(|| format!("write {}", path.display()))?;
Ok(())
}
pub fn read() -> Result<RecordSessionState> {
let path = paths::record_session_path()?;
if !path.exists() {
return Err(anyhow::anyhow!(
"no recording in progress; run `bsk record start` first"
));
}
let raw = fs::read_to_string(&path).with_context(|| format!("read {}", path.display()))?;
serde_json::from_str(&raw).with_context(|| format!("parse {}", path.display()))
}
pub fn clear() {
if let Ok(path) = paths::record_session_path() {
let _ = fs::remove_file(path);
}
}
fn started_at_unix_ms() -> String {
let ms = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_millis();
format!("{ms}")
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Mutex;
fn env_guard() -> &'static Mutex<()> {
static GUARD: Mutex<()> = Mutex::new(());
&GUARD
}
fn with_temp_home<F: FnOnce()>(f: F) {
let _lock = env_guard().lock().unwrap_or_else(|e| e.into_inner());
let tmp = tempfile::tempdir().unwrap();
unsafe {
std::env::set_var(crate::daemon::paths::BSK_HOME_ENV, tmp.path());
}
f();
clear();
unsafe {
std::env::remove_var(crate::daemon::paths::BSK_HOME_ENV);
}
}
#[test]
fn write_read_round_trips() {
with_temp_home(|| {
write("abcd").unwrap();
let state = read().unwrap();
assert_eq!(state.session_id, "abcd");
assert!(!state.started_at.is_empty());
});
}
#[test]
fn write_rejects_duplicate() {
with_temp_home(|| {
write("abcd").unwrap();
let err = write("efgh").unwrap_err();
assert!(err.to_string().contains("already in progress"));
});
}
#[test]
fn read_errors_when_missing() {
with_temp_home(|| {
let err = read().unwrap_err();
assert!(err.to_string().contains("no recording in progress"));
});
}
#[test]
fn clear_removes_state_file() {
with_temp_home(|| {
write("abcd").unwrap();
clear();
assert!(read().is_err());
});
}
}
+39 -21
View File
@@ -63,11 +63,11 @@ struct StartParams {
}
#[derive(Debug, Deserialize)]
struct StartReply {
session_id: String,
browser_instance_id: String,
pub struct StartReply {
pub session_id: String,
pub browser_instance_id: String,
#[serde(default)]
agent_window_id: Option<i64>,
pub agent_window_id: Option<i64>,
}
#[derive(Debug, Serialize)]
@@ -78,21 +78,21 @@ struct StopParams {
}
#[derive(Debug, Deserialize)]
struct StopReply {
stopped: Vec<String>,
pub struct StopReply {
pub stopped: Vec<String>,
#[serde(default)]
failed: Vec<StopFailure>,
pub failed: Vec<StopFailure>,
#[serde(default)]
returned_tab_ids: Vec<i64>,
pub returned_tab_ids: Vec<i64>,
#[serde(default)]
return_failures: Vec<ReturnFailure>,
pub return_failures: Vec<ReturnFailure>,
}
#[derive(Debug, Deserialize, Serialize)]
struct StopFailure {
session_id: String,
code: ErrorCode,
message: String,
pub struct StopFailure {
pub session_id: String,
pub code: ErrorCode,
pub message: String,
}
#[derive(Debug, Deserialize)]
@@ -113,14 +113,7 @@ pub fn dispatch(cmd: SessionCmd, format: Format) -> Result<(), CliError> {
}
fn run_start(sock: PathBuf, args: SessionStartArgs, format: Format) -> Result<(), CliError> {
let result: Result<StartReply, CliError> = call(
sock,
Method::SessionStart,
Some(StartParams {
browser_instance_id: args.browser,
}),
Duration::from_secs(30),
);
let result = start_session(sock, args.browser);
match result {
Ok(reply) => match format {
Format::Json => {
@@ -143,6 +136,31 @@ fn run_start(sock: PathBuf, args: SessionStartArgs, format: Format) -> Result<()
Ok(())
}
/// Start a session and open the Agent Window. Used by `session start` and `record start`.
pub fn start_session(sock: PathBuf, browser: Option<String>) -> Result<StartReply, CliError> {
call(
sock,
Method::SessionStart,
Some(StartParams {
browser_instance_id: browser,
}),
Duration::from_secs(30),
)
}
/// Stop a single session by id.
pub fn stop_session(sock: PathBuf, session_id: &str) -> Result<StopReply, CliError> {
call(
sock,
Method::SessionStop,
Some(StopParams {
session_id: Some(session_id.to_string()),
all: false,
}),
SESSION_STOP_IPC_TIMEOUT,
)
}
/// Render a `session.start` failure (review I3).
///
/// The previous implementation hand-rolled `eprintln!("error: ...")`