mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(vom): update vom algorithm, fix overlay problem with background state machine
This commit is contained in:
@@ -19,6 +19,7 @@ pub mod interaction;
|
||||
pub mod logs;
|
||||
pub mod navigate;
|
||||
pub mod network;
|
||||
pub mod observe;
|
||||
pub mod record;
|
||||
pub mod record_state;
|
||||
pub mod render_error;
|
||||
@@ -41,6 +42,7 @@ use crate::cli::install_skill::InstallSkillArgs;
|
||||
use crate::cli::interaction::{ClickArgs, FillArgs, PressArgs, SelectArgs};
|
||||
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
|
||||
use crate::cli::network::NetworkArgs;
|
||||
use crate::cli::observe::ObserveArgs;
|
||||
use crate::cli::record::RecordCmd;
|
||||
use crate::cli::screenshot::ScreenshotArgs;
|
||||
use crate::cli::session::SessionCmd;
|
||||
@@ -122,6 +124,9 @@ pub enum Command {
|
||||
/// Produce an indented aria-snapshot with @eN refs.
|
||||
Snapshot(SnapshotArgs),
|
||||
|
||||
/// Produce a semantic VOM observation with perception probes.
|
||||
Observe(ObserveArgs),
|
||||
|
||||
/// Read buffered console/log/exception messages.
|
||||
Console(ConsoleArgs),
|
||||
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
//! `bsk observe` — semantic VOM observation with bounded perception probes.
|
||||
|
||||
use std::path::PathBuf;
|
||||
|
||||
use anyhow::Context;
|
||||
use bsk_protocol::Method;
|
||||
use bsk_protocol::tools::{ObserveParams, ObserveResult};
|
||||
use clap::Args;
|
||||
|
||||
use crate::cli::TOOL_IPC_TIMEOUT;
|
||||
use crate::cli::dialogs::print_dialog_summaries;
|
||||
use crate::cli::ensure_daemon::ensure_daemon;
|
||||
use crate::cli::error::{CliError, Format};
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct ObserveArgs {
|
||||
/// Session id (must be active).
|
||||
#[arg(long)]
|
||||
pub session: String,
|
||||
|
||||
/// Target tab. Defaults to the Agent Window's active tab.
|
||||
#[arg(long = "tab-id")]
|
||||
pub tab_id: Option<i64>,
|
||||
|
||||
/// Cap on VOM tree depth before truncating.
|
||||
#[arg(long = "max-depth")]
|
||||
pub max_depth: Option<u32>,
|
||||
|
||||
/// Soft cap on rendered tokens (~4 chars/token).
|
||||
#[arg(long = "max-tokens")]
|
||||
pub max_tokens: Option<u32>,
|
||||
}
|
||||
|
||||
pub fn dispatch(args: ObserveArgs, format: Format) -> Result<(), CliError> {
|
||||
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||
run(info.sock_path, args, format)
|
||||
}
|
||||
|
||||
fn run(sock: PathBuf, args: ObserveArgs, format: Format) -> Result<(), CliError> {
|
||||
let params = ObserveParams {
|
||||
session_id: args.session.clone(),
|
||||
tab_id: args.tab_id,
|
||||
max_depth: args.max_depth,
|
||||
max_tokens: args.max_tokens,
|
||||
};
|
||||
let reply: ObserveResult = call(sock, params)?;
|
||||
match format {
|
||||
Format::Json => {
|
||||
let json = serde_json::to_string_pretty(&reply)
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?;
|
||||
println!("{json}");
|
||||
}
|
||||
Format::Human => {
|
||||
if reply.text.is_empty() {
|
||||
println!("(empty observation — page may still be loading)");
|
||||
} else {
|
||||
println!("{}", reply.text);
|
||||
}
|
||||
if reply.truncated {
|
||||
eprintln!(
|
||||
"warning: observation truncated (refs={}, tab={}). Increase --max-depth / --max-tokens if needed.",
|
||||
reply.ref_count, reply.tab_id
|
||||
);
|
||||
}
|
||||
print_dialog_summaries(&reply.dialogs);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn call(sock: PathBuf, params: ObserveParams) -> Result<ObserveResult, CliError> {
|
||||
crate::cli::business_rpc::call::<ObserveParams, ObserveResult>(
|
||||
sock,
|
||||
"observe",
|
||||
Method::ToolObserve,
|
||||
Some(params),
|
||||
TOOL_IPC_TIMEOUT,
|
||||
)
|
||||
}
|
||||
@@ -231,6 +231,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
|
||||
| Method::ToolConsole
|
||||
| Method::ToolNetwork
|
||||
| Method::ToolSnapshot
|
||||
| Method::ToolObserve
|
||||
| Method::ToolGetHtml
|
||||
| Method::ToolNavigate
|
||||
| Method::ToolNavigateBack
|
||||
@@ -291,16 +292,14 @@ async fn handle_tool_dispatch(
|
||||
};
|
||||
// Pre-flight: if the user has clicked the agent-window mask's
|
||||
// stop button, every session carries a one-shot "pending
|
||||
// interrupt" marker. The marker is consumed by the next
|
||||
// *mutating* tool call (which is rejected with `UserAborted`).
|
||||
// Read-only tools (snapshot / get_html / waits / tab_list) and
|
||||
// session-lifecycle RPCs pass through transparently — gating
|
||||
// them would prevent the agent from observing page state
|
||||
// before asking the user, or from cleanly tearing down the
|
||||
// session. Classification lives on `Method::is_mutating()` so
|
||||
// adding a new tool variant requires an explicit
|
||||
// classification call.
|
||||
if method.is_mutating() && state.session_interrupts.try_consume(&session_id) {
|
||||
// interrupt" marker. The marker is consumed by the next method
|
||||
// that dispatches browser/page input (which is rejected with
|
||||
// `UserAborted`). Passive reads and control-plane RPCs pass through
|
||||
// transparently — gating them would prevent the agent from observing
|
||||
// page state before asking the user, or from cleanly tearing down the
|
||||
// session. Classification lives on `Method::effect()` so adding a new
|
||||
// tool variant requires an explicit classification call.
|
||||
if method.requires_interrupt_gate() && state.session_interrupts.try_consume(&session_id) {
|
||||
return ResponseBody::Err(RpcError {
|
||||
code: ErrorCode::UserAborted,
|
||||
message: "tool dispatch rejected: pending user interrupt. The user explicitly requested to stop. Ask the user how to proceed before issuing further actions.".into(),
|
||||
|
||||
@@ -2,13 +2,13 @@
|
||||
//!
|
||||
//! Holds a single-use marker per `SessionId` indicating that the
|
||||
//! user has clicked the agent-window mask's stop button. The next
|
||||
//! mutating `tool.*` call for that session is rejected with
|
||||
//! `ErrorCode::UserAborted`; read-only tools and session-lifecycle
|
||||
//! browser-input-dispatching `tool.*` call for that session is rejected
|
||||
//! with `ErrorCode::UserAborted`; passive reads and session-lifecycle
|
||||
//! RPCs pass through transparently and do not consume the marker.
|
||||
//!
|
||||
//! The marker is single-use and has no expiry: it sits in the
|
||||
//! registry until consumed by a mutating call or until the session
|
||||
//! is torn down. This lets the user's interrupt survive an LLM
|
||||
//! registry until consumed by an input-dispatching call or until the
|
||||
//! session is torn down. This lets the user's interrupt survive an LLM
|
||||
//! thinking phase of arbitrary length — the v1 time-window
|
||||
//! mechanism dropped interrupts whenever the LLM took longer to
|
||||
//! respond than the window allowed.
|
||||
|
||||
@@ -46,8 +46,8 @@ pub struct DaemonState {
|
||||
/// Per-session "pending interrupt" signal. The WS event handler
|
||||
/// `mark`s the session when the user clicks the agent-window
|
||||
/// mask's stop button; the IPC tool-dispatch handler
|
||||
/// `try_consume`s on the way in so the next mutating tool call
|
||||
/// is rejected with `UserAborted`. Independent of
|
||||
/// `try_consume`s on the way in so the next browser-input-dispatching
|
||||
/// tool call is rejected with `UserAborted`. Independent of
|
||||
/// `SessionRegistry` because the signal is a transient runtime
|
||||
/// control state.
|
||||
pub session_interrupts: Arc<SessionInterruptRegistry>,
|
||||
|
||||
@@ -82,6 +82,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
|
||||
Command::Tab(cmd) => cli::tab::dispatch(cmd, format),
|
||||
Command::Screenshot(args) => cli::screenshot::dispatch(args, format),
|
||||
Command::Snapshot(args) => cli::snapshot::dispatch(args, format),
|
||||
Command::Observe(args) => cli::observe::dispatch(args, format),
|
||||
Command::Console(args) => cli::console::dispatch(args, format),
|
||||
Command::Network(args) => cli::network::dispatch(args, format),
|
||||
Command::GetHtml(args) => cli::get_html::dispatch(args, format),
|
||||
|
||||
Reference in New Issue
Block a user