feat(vom): update vom algorithm, fix overlay problem with background state machine

This commit is contained in:
Ljy-0827
2026-07-27 13:46:33 +08:00
parent e14ae9cea0
commit 4fb2e331a6
36 changed files with 2276 additions and 452 deletions
+5
View File
@@ -19,6 +19,7 @@ pub mod interaction;
pub mod logs;
pub mod navigate;
pub mod network;
pub mod observe;
pub mod record;
pub mod record_state;
pub mod render_error;
@@ -41,6 +42,7 @@ use crate::cli::install_skill::InstallSkillArgs;
use crate::cli::interaction::{ClickArgs, FillArgs, PressArgs, SelectArgs};
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
use crate::cli::network::NetworkArgs;
use crate::cli::observe::ObserveArgs;
use crate::cli::record::RecordCmd;
use crate::cli::screenshot::ScreenshotArgs;
use crate::cli::session::SessionCmd;
@@ -122,6 +124,9 @@ pub enum Command {
/// Produce an indented aria-snapshot with @eN refs.
Snapshot(SnapshotArgs),
/// Produce a semantic VOM observation with perception probes.
Observe(ObserveArgs),
/// Read buffered console/log/exception messages.
Console(ConsoleArgs),
+79
View File
@@ -0,0 +1,79 @@
//! `bsk observe` — semantic VOM observation with bounded perception probes.
use std::path::PathBuf;
use anyhow::Context;
use bsk_protocol::Method;
use bsk_protocol::tools::{ObserveParams, ObserveResult};
use clap::Args;
use crate::cli::TOOL_IPC_TIMEOUT;
use crate::cli::dialogs::print_dialog_summaries;
use crate::cli::ensure_daemon::ensure_daemon;
use crate::cli::error::{CliError, Format};
#[derive(Debug, Clone, Args)]
pub struct ObserveArgs {
/// Session id (must be active).
#[arg(long)]
pub session: String,
/// Target tab. Defaults to the Agent Window's active tab.
#[arg(long = "tab-id")]
pub tab_id: Option<i64>,
/// Cap on VOM tree depth before truncating.
#[arg(long = "max-depth")]
pub max_depth: Option<u32>,
/// Soft cap on rendered tokens (~4 chars/token).
#[arg(long = "max-tokens")]
pub max_tokens: Option<u32>,
}
pub fn dispatch(args: ObserveArgs, format: Format) -> Result<(), CliError> {
let info = ensure_daemon().context("ensure daemon is running")?;
run(info.sock_path, args, format)
}
fn run(sock: PathBuf, args: ObserveArgs, format: Format) -> Result<(), CliError> {
let params = ObserveParams {
session_id: args.session.clone(),
tab_id: args.tab_id,
max_depth: args.max_depth,
max_tokens: args.max_tokens,
};
let reply: ObserveResult = call(sock, params)?;
match format {
Format::Json => {
let json = serde_json::to_string_pretty(&reply)
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?;
println!("{json}");
}
Format::Human => {
if reply.text.is_empty() {
println!("(empty observation — page may still be loading)");
} else {
println!("{}", reply.text);
}
if reply.truncated {
eprintln!(
"warning: observation truncated (refs={}, tab={}). Increase --max-depth / --max-tokens if needed.",
reply.ref_count, reply.tab_id
);
}
print_dialog_summaries(&reply.dialogs);
}
}
Ok(())
}
fn call(sock: PathBuf, params: ObserveParams) -> Result<ObserveResult, CliError> {
crate::cli::business_rpc::call::<ObserveParams, ObserveResult>(
sock,
"observe",
Method::ToolObserve,
Some(params),
TOOL_IPC_TIMEOUT,
)
}
+9 -10
View File
@@ -231,6 +231,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
| Method::ToolConsole
| Method::ToolNetwork
| Method::ToolSnapshot
| Method::ToolObserve
| Method::ToolGetHtml
| Method::ToolNavigate
| Method::ToolNavigateBack
@@ -291,16 +292,14 @@ async fn handle_tool_dispatch(
};
// Pre-flight: if the user has clicked the agent-window mask's
// stop button, every session carries a one-shot "pending
// interrupt" marker. The marker is consumed by the next
// *mutating* tool call (which is rejected with `UserAborted`).
// Read-only tools (snapshot / get_html / waits / tab_list) and
// session-lifecycle RPCs pass through transparently — gating
// them would prevent the agent from observing page state
// before asking the user, or from cleanly tearing down the
// session. Classification lives on `Method::is_mutating()` so
// adding a new tool variant requires an explicit
// classification call.
if method.is_mutating() && state.session_interrupts.try_consume(&session_id) {
// interrupt" marker. The marker is consumed by the next method
// that dispatches browser/page input (which is rejected with
// `UserAborted`). Passive reads and control-plane RPCs pass through
// transparently — gating them would prevent the agent from observing
// page state before asking the user, or from cleanly tearing down the
// session. Classification lives on `Method::effect()` so adding a new
// tool variant requires an explicit classification call.
if method.requires_interrupt_gate() && state.session_interrupts.try_consume(&session_id) {
return ResponseBody::Err(RpcError {
code: ErrorCode::UserAborted,
message: "tool dispatch rejected: pending user interrupt. The user explicitly requested to stop. Ask the user how to proceed before issuing further actions.".into(),
@@ -2,13 +2,13 @@
//!
//! Holds a single-use marker per `SessionId` indicating that the
//! user has clicked the agent-window mask's stop button. The next
//! mutating `tool.*` call for that session is rejected with
//! `ErrorCode::UserAborted`; read-only tools and session-lifecycle
//! browser-input-dispatching `tool.*` call for that session is rejected
//! with `ErrorCode::UserAborted`; passive reads and session-lifecycle
//! RPCs pass through transparently and do not consume the marker.
//!
//! The marker is single-use and has no expiry: it sits in the
//! registry until consumed by a mutating call or until the session
//! is torn down. This lets the user's interrupt survive an LLM
//! registry until consumed by an input-dispatching call or until the
//! session is torn down. This lets the user's interrupt survive an LLM
//! thinking phase of arbitrary length — the v1 time-window
//! mechanism dropped interrupts whenever the LLM took longer to
//! respond than the window allowed.
+2 -2
View File
@@ -46,8 +46,8 @@ pub struct DaemonState {
/// Per-session "pending interrupt" signal. The WS event handler
/// `mark`s the session when the user clicks the agent-window
/// mask's stop button; the IPC tool-dispatch handler
/// `try_consume`s on the way in so the next mutating tool call
/// is rejected with `UserAborted`. Independent of
/// `try_consume`s on the way in so the next browser-input-dispatching
/// tool call is rejected with `UserAborted`. Independent of
/// `SessionRegistry` because the signal is a transient runtime
/// control state.
pub session_interrupts: Arc<SessionInterruptRegistry>,
+1
View File
@@ -82,6 +82,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
Command::Tab(cmd) => cli::tab::dispatch(cmd, format),
Command::Screenshot(args) => cli::screenshot::dispatch(args, format),
Command::Snapshot(args) => cli::snapshot::dispatch(args, format),
Command::Observe(args) => cli::observe::dispatch(args, format),
Command::Console(args) => cli::console::dispatch(args, format),
Command::Network(args) => cli::network::dispatch(args, format),
Command::GetHtml(args) => cli::get_html::dispatch(args, format),