feat(extension): add bsk hover and adjust vom to support

This commit is contained in:
Ljy-0827
2026-08-06 12:19:45 +08:00
parent 0f264c5df1
commit 91225e5fdb
31 changed files with 1051 additions and 47 deletions
+11 -7
View File
@@ -67,22 +67,25 @@ Write operations only affect tabs in the **Agent Window** (or tabs you **borrowe
```
bsk navigate <url> --session <id>
bsk snapshot --session <id> → aria tree with @e1, @e2, … refs
bsk observe --session <id> → semantic VOM view; may reveal hover/focus surfaces
bsk click @e3 --session <id> → or bsk fill, bsk select, bsk press
bsk snapshot --session <id> → again after navigation / DOM change
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
bsk observe --session <id> → again after navigation / DOM change
```
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
## Observation priority
Start with `bsk snapshot` to understand page structure, text, controls, and element refs. Use `bsk observe` when semantic VOM output or conditional hover/focus surfaces would materially help. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
1. `bsk snapshot` — strict static page understanding and interaction planning
2. `bsk observe` — semantic VOM observation; may run bounded perception probes such as hover-surface discovery
1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
2. `bsk snapshot` — strict static accessibility tree fallback
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
@@ -186,6 +189,7 @@ bsk emulate --session <id> --off
| Command | Summary |
|---------|---------|
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
+87 -2
View File
@@ -10,8 +10,8 @@ use std::time::Duration;
use anyhow::Context;
use bsk_protocol::Method;
use bsk_protocol::tools::{
ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, PressParams,
PressResult, SelectParams, SelectResult,
ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier,
MouseButton, PressParams, PressResult, SelectParams, SelectResult,
};
use clap::{Args, ValueEnum};
@@ -189,6 +189,91 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
Ok(())
}
// ---------------------------------------------------------------------------
// bsk hover
// ---------------------------------------------------------------------------
#[derive(Debug, Clone, Args)]
pub struct HoverArgs {
/// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used.
pub target: Option<String>,
/// Force-treat the value as a snapshot ref (overrides positional detection).
#[arg(long = "ref")]
pub ref_: Option<String>,
/// Force-treat the value as a CSS selector.
#[arg(long = "selector")]
pub selector: Option<String>,
#[arg(long)]
pub session: String,
#[arg(long = "tab-id")]
pub tab_id: Option<i64>,
/// Comma-separated modifiers (`alt,ctrl,shift,meta`).
#[arg(long, default_value = "")]
pub modifiers: String,
/// Wait after moving the mouse so hover-triggered UI can settle.
#[arg(long = "settle", default_value = "200ms", value_parser = parse_timeout_ms)]
pub settle: u32,
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
pub timeout: u32,
}
pub fn dispatch_hover(args: HoverArgs, format: Format) -> Result<(), CliError> {
let info = ensure_daemon().context("ensure daemon is running")?;
let (ref_, selector) = split_target(
args.target.clone(),
args.ref_.clone(),
args.selector.clone(),
)?;
let modifiers = parse_modifiers(&args.modifiers)
.map_err(|e| CliError::Local(anyhow::anyhow!("--modifiers: {e}")))?;
let params = HoverParams {
session_id: args.session.clone(),
ref_,
selector,
tab_id: args.tab_id,
modifiers: if modifiers.is_empty() {
None
} else {
Some(modifiers)
},
settle_ms: Some(args.settle),
timeout_ms: Some(args.timeout),
};
let reply: HoverResult = call(
info.sock_path,
Method::ToolHover,
params,
"hover-1",
args.timeout,
)?;
match format {
Format::Json => {
println!(
"{}",
serde_json::to_string_pretty(&reply)
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
);
}
Format::Human => {
let target =
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref());
println!(
"hover ok tab={} target={target} at=({}, {})",
reply.tab_id, reply.x, reply.y
);
print_dialog_summaries(&reply.dialogs);
}
}
Ok(())
}
// ---------------------------------------------------------------------------
// bsk fill
// ---------------------------------------------------------------------------
+4 -1
View File
@@ -42,7 +42,7 @@ use crate::cli::evaluate::EvaluateArgs;
use crate::cli::get_html::GetHtmlArgs;
use crate::cli::human_loop::RequestHelpArgs;
use crate::cli::install_skill::InstallSkillArgs;
use crate::cli::interaction::{ClickArgs, FillArgs, PressArgs, SelectArgs};
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs};
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
use crate::cli::network::NetworkArgs;
use crate::cli::observe::ObserveArgs;
@@ -164,6 +164,9 @@ pub enum Command {
/// Click a snapshot ref or CSS selector.
Click(ClickArgs),
/// Hover a snapshot ref or CSS selector.
Hover(HoverArgs),
/// Fill an input / textarea / contenteditable.
Fill(FillArgs),
+1
View File
@@ -242,6 +242,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
| Method::ToolNavigateForward
| Method::ToolReload
| Method::ToolClick
| Method::ToolHover
| Method::ToolFill
| Method::ToolPress
| Method::ToolSelect
+1
View File
@@ -93,6 +93,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
Command::NavigateForward(args) => cli::navigate::dispatch_navigate_forward(args, format),
Command::Reload(args) => cli::navigate::dispatch_reload(args, format),
Command::Click(args) => cli::interaction::dispatch_click(args, format),
Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
Command::Press(args) => cli::interaction::dispatch_press(args, format),
Command::Select(args) => cli::interaction::dispatch_select(args, format),
+18
View File
@@ -187,6 +187,24 @@ fn parses_click_count_alias() {
assert_eq!(args.click_count, 2);
}
#[test]
fn parses_hover_with_settle() {
let cli = parse(&[
"bsk",
"hover",
"@e1",
"--session",
"s1",
"--settle",
"300ms",
]);
let Command::Hover(args) = cli.command else {
panic!("expected hover command");
};
assert_eq!(args.target.as_deref(), Some("@e1"));
assert_eq!(args.settle, 300);
}
#[test]
fn rejects_zero_click_count() {
assert!(