mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(extension): add bsk hover and adjust vom to support
This commit is contained in:
@@ -67,22 +67,25 @@ Write operations only affect tabs in the **Agent Window** (or tabs you **borrowe
|
||||
|
||||
```
|
||||
bsk navigate <url> --session <id>
|
||||
bsk snapshot --session <id> → aria tree with @e1, @e2, … refs
|
||||
bsk observe --session <id> → semantic VOM view; may reveal hover/focus surfaces
|
||||
bsk click @e3 --session <id> → or bsk fill, bsk select, bsk press
|
||||
bsk snapshot --session <id> → again after navigation / DOM change
|
||||
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
||||
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
||||
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
||||
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
||||
bsk observe --session <id> → again after navigation / DOM change
|
||||
```
|
||||
|
||||
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
||||
|
||||
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
||||
|
||||
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
||||
|
||||
## Observation priority
|
||||
|
||||
Start with `bsk snapshot` to understand page structure, text, controls, and element refs. Use `bsk observe` when semantic VOM output or conditional hover/focus surfaces would materially help. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||
|
||||
1. `bsk snapshot` — strict static page understanding and interaction planning
|
||||
2. `bsk observe` — semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
||||
1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
||||
2. `bsk snapshot` — strict static accessibility tree fallback
|
||||
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
|
||||
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
|
||||
|
||||
@@ -186,6 +189,7 @@ bsk emulate --session <id> --off
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
||||
|
||||
@@ -10,8 +10,8 @@ use std::time::Duration;
|
||||
use anyhow::Context;
|
||||
use bsk_protocol::Method;
|
||||
use bsk_protocol::tools::{
|
||||
ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, PressParams,
|
||||
PressResult, SelectParams, SelectResult,
|
||||
ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier,
|
||||
MouseButton, PressParams, PressResult, SelectParams, SelectResult,
|
||||
};
|
||||
use clap::{Args, ValueEnum};
|
||||
|
||||
@@ -189,6 +189,91 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// bsk hover
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct HoverArgs {
|
||||
/// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used.
|
||||
pub target: Option<String>,
|
||||
|
||||
/// Force-treat the value as a snapshot ref (overrides positional detection).
|
||||
#[arg(long = "ref")]
|
||||
pub ref_: Option<String>,
|
||||
|
||||
/// Force-treat the value as a CSS selector.
|
||||
#[arg(long = "selector")]
|
||||
pub selector: Option<String>,
|
||||
|
||||
#[arg(long)]
|
||||
pub session: String,
|
||||
|
||||
#[arg(long = "tab-id")]
|
||||
pub tab_id: Option<i64>,
|
||||
|
||||
/// Comma-separated modifiers (`alt,ctrl,shift,meta`).
|
||||
#[arg(long, default_value = "")]
|
||||
pub modifiers: String,
|
||||
|
||||
/// Wait after moving the mouse so hover-triggered UI can settle.
|
||||
#[arg(long = "settle", default_value = "200ms", value_parser = parse_timeout_ms)]
|
||||
pub settle: u32,
|
||||
|
||||
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
|
||||
pub timeout: u32,
|
||||
}
|
||||
|
||||
pub fn dispatch_hover(args: HoverArgs, format: Format) -> Result<(), CliError> {
|
||||
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||
let (ref_, selector) = split_target(
|
||||
args.target.clone(),
|
||||
args.ref_.clone(),
|
||||
args.selector.clone(),
|
||||
)?;
|
||||
let modifiers = parse_modifiers(&args.modifiers)
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!("--modifiers: {e}")))?;
|
||||
let params = HoverParams {
|
||||
session_id: args.session.clone(),
|
||||
ref_,
|
||||
selector,
|
||||
tab_id: args.tab_id,
|
||||
modifiers: if modifiers.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(modifiers)
|
||||
},
|
||||
settle_ms: Some(args.settle),
|
||||
timeout_ms: Some(args.timeout),
|
||||
};
|
||||
let reply: HoverResult = call(
|
||||
info.sock_path,
|
||||
Method::ToolHover,
|
||||
params,
|
||||
"hover-1",
|
||||
args.timeout,
|
||||
)?;
|
||||
match format {
|
||||
Format::Json => {
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&reply)
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
|
||||
);
|
||||
}
|
||||
Format::Human => {
|
||||
let target =
|
||||
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref());
|
||||
println!(
|
||||
"hover ok tab={} target={target} at=({}, {})",
|
||||
reply.tab_id, reply.x, reply.y
|
||||
);
|
||||
print_dialog_summaries(&reply.dialogs);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// bsk fill
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -42,7 +42,7 @@ use crate::cli::evaluate::EvaluateArgs;
|
||||
use crate::cli::get_html::GetHtmlArgs;
|
||||
use crate::cli::human_loop::RequestHelpArgs;
|
||||
use crate::cli::install_skill::InstallSkillArgs;
|
||||
use crate::cli::interaction::{ClickArgs, FillArgs, PressArgs, SelectArgs};
|
||||
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs};
|
||||
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
|
||||
use crate::cli::network::NetworkArgs;
|
||||
use crate::cli::observe::ObserveArgs;
|
||||
@@ -164,6 +164,9 @@ pub enum Command {
|
||||
/// Click a snapshot ref or CSS selector.
|
||||
Click(ClickArgs),
|
||||
|
||||
/// Hover a snapshot ref or CSS selector.
|
||||
Hover(HoverArgs),
|
||||
|
||||
/// Fill an input / textarea / contenteditable.
|
||||
Fill(FillArgs),
|
||||
|
||||
|
||||
@@ -242,6 +242,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
|
||||
| Method::ToolNavigateForward
|
||||
| Method::ToolReload
|
||||
| Method::ToolClick
|
||||
| Method::ToolHover
|
||||
| Method::ToolFill
|
||||
| Method::ToolPress
|
||||
| Method::ToolSelect
|
||||
|
||||
@@ -93,6 +93,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
|
||||
Command::NavigateForward(args) => cli::navigate::dispatch_navigate_forward(args, format),
|
||||
Command::Reload(args) => cli::navigate::dispatch_reload(args, format),
|
||||
Command::Click(args) => cli::interaction::dispatch_click(args, format),
|
||||
Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
|
||||
Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
|
||||
Command::Press(args) => cli::interaction::dispatch_press(args, format),
|
||||
Command::Select(args) => cli::interaction::dispatch_select(args, format),
|
||||
|
||||
@@ -187,6 +187,24 @@ fn parses_click_count_alias() {
|
||||
assert_eq!(args.click_count, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_hover_with_settle() {
|
||||
let cli = parse(&[
|
||||
"bsk",
|
||||
"hover",
|
||||
"@e1",
|
||||
"--session",
|
||||
"s1",
|
||||
"--settle",
|
||||
"300ms",
|
||||
]);
|
||||
let Command::Hover(args) = cli.command else {
|
||||
panic!("expected hover command");
|
||||
};
|
||||
assert_eq!(args.target.as_deref(), Some("@e1"));
|
||||
assert_eq!(args.settle, 300);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_zero_click_count() {
|
||||
assert!(
|
||||
|
||||
Reference in New Issue
Block a user