mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(interaction): add explicit focus and blur primitives
This commit is contained in:
@@ -70,6 +70,8 @@ bsk navigate <url> --session <id>
|
||||
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
||||
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
||||
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
||||
bsk focus @e4 --session <id> → explicitly enter focus-driven UI states
|
||||
bsk blur @e4 --session <id> → explicitly leave focus-driven UI states
|
||||
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
||||
bsk observe --session <id> → again after navigation / DOM change
|
||||
```
|
||||
@@ -202,6 +204,8 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
|
||||
|---------|---------|
|
||||
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||
| `bsk focus <ref-or-selector>` | Focus an element and verify it became the deep active element |
|
||||
| `bsk blur <ref-or-selector>` | Remove focus from an element and report whether it was focused |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
//! `bsk click` / `bsk fill` / `bsk press` (M7 interaction tools).
|
||||
//! `bsk click` / `bsk focus` / `bsk blur` / `bsk fill` / `bsk press`
|
||||
//! interaction tools.
|
||||
//!
|
||||
//! The `<target>` positional accepts either a snapshot ref (`@e3`,
|
||||
//! `e3`) or a CSS selector. Use `--ref` / `--selector` explicitly when
|
||||
@@ -10,8 +11,9 @@ use std::time::Duration;
|
||||
use anyhow::Context;
|
||||
use bsk_protocol::Method;
|
||||
use bsk_protocol::tools::{
|
||||
ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier,
|
||||
MouseButton, PressParams, PressResult, SelectParams, SelectResult,
|
||||
BlurParams, BlurResult, ClickParams, ClickResult, FillParams, FillResult, FocusParams,
|
||||
FocusResult, HoverParams, HoverResult, KeyModifier, MouseButton, PressParams, PressResult,
|
||||
SelectParams, SelectResult,
|
||||
};
|
||||
use clap::{Args, ValueEnum};
|
||||
|
||||
@@ -274,6 +276,124 @@ pub fn dispatch_hover(args: HoverArgs, format: Format) -> Result<(), CliError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// bsk focus / bsk blur
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct FocusArgs {
|
||||
/// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used.
|
||||
pub target: Option<String>,
|
||||
|
||||
#[arg(long = "ref")]
|
||||
pub ref_: Option<String>,
|
||||
|
||||
#[arg(long = "selector")]
|
||||
pub selector: Option<String>,
|
||||
|
||||
#[arg(long)]
|
||||
pub session: String,
|
||||
|
||||
#[arg(long = "tab-id")]
|
||||
pub tab_id: Option<i64>,
|
||||
|
||||
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
|
||||
pub timeout: u32,
|
||||
}
|
||||
|
||||
pub fn dispatch_focus(args: FocusArgs, format: Format) -> Result<(), CliError> {
|
||||
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||
let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?;
|
||||
let params = FocusParams {
|
||||
session_id: args.session,
|
||||
ref_,
|
||||
selector,
|
||||
tab_id: args.tab_id,
|
||||
timeout_ms: Some(args.timeout),
|
||||
};
|
||||
let reply: FocusResult = call(
|
||||
info.sock_path,
|
||||
Method::ToolFocus,
|
||||
params,
|
||||
"focus-1",
|
||||
args.timeout,
|
||||
)?;
|
||||
match format {
|
||||
Format::Json => println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&reply)
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
|
||||
),
|
||||
Format::Human => {
|
||||
let target =
|
||||
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref());
|
||||
println!(
|
||||
"focus ok tab={} target={target} focused={}",
|
||||
reply.tab_id, reply.focused
|
||||
);
|
||||
print_dialog_summaries(&reply.dialogs);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct BlurArgs {
|
||||
/// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used.
|
||||
pub target: Option<String>,
|
||||
|
||||
#[arg(long = "ref")]
|
||||
pub ref_: Option<String>,
|
||||
|
||||
#[arg(long = "selector")]
|
||||
pub selector: Option<String>,
|
||||
|
||||
#[arg(long)]
|
||||
pub session: String,
|
||||
|
||||
#[arg(long = "tab-id")]
|
||||
pub tab_id: Option<i64>,
|
||||
|
||||
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
|
||||
pub timeout: u32,
|
||||
}
|
||||
|
||||
pub fn dispatch_blur(args: BlurArgs, format: Format) -> Result<(), CliError> {
|
||||
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||
let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?;
|
||||
let params = BlurParams {
|
||||
session_id: args.session,
|
||||
ref_,
|
||||
selector,
|
||||
tab_id: args.tab_id,
|
||||
timeout_ms: Some(args.timeout),
|
||||
};
|
||||
let reply: BlurResult = call(
|
||||
info.sock_path,
|
||||
Method::ToolBlur,
|
||||
params,
|
||||
"blur-1",
|
||||
args.timeout,
|
||||
)?;
|
||||
match format {
|
||||
Format::Json => println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&reply)
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
|
||||
),
|
||||
Format::Human => {
|
||||
let target =
|
||||
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref());
|
||||
println!(
|
||||
"blur ok tab={} target={target} was_focused={} focused={}",
|
||||
reply.tab_id, reply.was_focused, reply.focused
|
||||
);
|
||||
print_dialog_summaries(&reply.dialogs);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// bsk fill
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -42,7 +42,9 @@ use crate::cli::evaluate::EvaluateArgs;
|
||||
use crate::cli::get_html::GetHtmlArgs;
|
||||
use crate::cli::human_loop::RequestHelpArgs;
|
||||
use crate::cli::install_skill::InstallSkillArgs;
|
||||
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs};
|
||||
use crate::cli::interaction::{
|
||||
BlurArgs, ClickArgs, FillArgs, FocusArgs, HoverArgs, PressArgs, SelectArgs,
|
||||
};
|
||||
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
|
||||
use crate::cli::network::NetworkArgs;
|
||||
use crate::cli::observe::ObserveArgs;
|
||||
@@ -167,6 +169,12 @@ pub enum Command {
|
||||
/// Hover a snapshot ref or CSS selector.
|
||||
Hover(HoverArgs),
|
||||
|
||||
/// Focus a snapshot ref or CSS selector.
|
||||
Focus(FocusArgs),
|
||||
|
||||
/// Remove focus from a snapshot ref or CSS selector.
|
||||
Blur(BlurArgs),
|
||||
|
||||
/// Fill an input / textarea / contenteditable.
|
||||
Fill(FillArgs),
|
||||
|
||||
|
||||
@@ -243,6 +243,8 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
|
||||
| Method::ToolReload
|
||||
| Method::ToolClick
|
||||
| Method::ToolHover
|
||||
| Method::ToolFocus
|
||||
| Method::ToolBlur
|
||||
| Method::ToolFill
|
||||
| Method::ToolPress
|
||||
| Method::ToolSelect
|
||||
|
||||
@@ -94,6 +94,8 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
|
||||
Command::Reload(args) => cli::navigate::dispatch_reload(args, format),
|
||||
Command::Click(args) => cli::interaction::dispatch_click(args, format),
|
||||
Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
|
||||
Command::Focus(args) => cli::interaction::dispatch_focus(args, format),
|
||||
Command::Blur(args) => cli::interaction::dispatch_blur(args, format),
|
||||
Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
|
||||
Command::Press(args) => cli::interaction::dispatch_press(args, format),
|
||||
Command::Select(args) => cli::interaction::dispatch_select(args, format),
|
||||
|
||||
@@ -206,6 +206,24 @@ fn parses_hover_with_settle() {
|
||||
assert_eq!(args.settle, 300);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_focus_target() {
|
||||
let cli = parse(&["bsk", "focus", "@e2", "--session", "s1"]);
|
||||
let Command::Focus(args) = cli.command else {
|
||||
panic!("expected focus command");
|
||||
};
|
||||
assert_eq!(args.target.as_deref(), Some("@e2"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_blur_selector() {
|
||||
let cli = parse(&["bsk", "blur", "--selector", "#search", "--session", "s1"]);
|
||||
let Command::Blur(args) = cli.command else {
|
||||
panic!("expected blur command");
|
||||
};
|
||||
assert_eq!(args.selector.as_deref(), Some("#search"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_zero_click_count() {
|
||||
assert!(
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
//! M7 integration: drive the 7 new tool RPCs (navigate /
|
||||
//! navigate_back / navigate_forward / reload / click / fill / press)
|
||||
//! M7 integration: drive interaction and navigation tool RPCs
|
||||
//! through the IPC + per-session queue + fake extension and assert
|
||||
//! the wire shapes line up. The extension stub mirrors each method's
|
||||
//! `*Result` so we exercise the daemon's serialise → forward →
|
||||
@@ -13,10 +12,11 @@ use bsk::daemon::{self, DaemonConfig};
|
||||
use bsk::ipc_client::IpcClient;
|
||||
use bsk_protocol::system::{HandshakeParams, HandshakeResult};
|
||||
use bsk_protocol::tools::{
|
||||
ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, NavigateBackParams,
|
||||
NavigateBackResult, NavigateForwardParams, NavigateForwardResult, NavigateParams,
|
||||
NavigateResult, PressParams, PressResult, ReloadParams, ReloadResult, SelectParams,
|
||||
SelectResult, SessionStartParams, SessionStartResult, WaitUntil,
|
||||
BlurParams, BlurResult, ClickParams, ClickResult, FillParams, FillResult, FocusParams,
|
||||
FocusResult, KeyModifier, MouseButton, NavigateBackParams, NavigateBackResult,
|
||||
NavigateForwardParams, NavigateForwardResult, NavigateParams, NavigateResult, PressParams,
|
||||
PressResult, ReloadParams, ReloadResult, SelectParams, SelectResult, SessionStartParams,
|
||||
SessionStartResult, WaitUntil,
|
||||
};
|
||||
use bsk_protocol::{
|
||||
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
|
||||
@@ -382,6 +382,92 @@ async fn click_round_trips_ref_and_modifiers() {
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn focus_round_trips_verified_state() {
|
||||
let (handle, sock) = spawn_daemon().await;
|
||||
let mut ws = connect_ext(handle.ws_addr()).await;
|
||||
let _ = do_handshake(&mut ws).await;
|
||||
|
||||
run_extension(ws, |req| {
|
||||
assert_eq!(req.method, Method::ToolFocus);
|
||||
let params = req.params.clone().unwrap();
|
||||
let p: FocusParams = serde_json::from_value(params).unwrap();
|
||||
assert_eq!(p.ref_.as_deref(), Some("@e3"));
|
||||
ResponseBody::Ok(
|
||||
serde_json::to_value(FocusResult {
|
||||
tab_id: 9,
|
||||
used_ref: Some("e3".into()),
|
||||
used_selector: None,
|
||||
focused: true,
|
||||
dialogs: vec![],
|
||||
})
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
let session_id = ipc_session_start(&sock).await;
|
||||
let result: FocusResult = ipc_tool_call(
|
||||
&sock,
|
||||
Method::ToolFocus,
|
||||
FocusParams {
|
||||
session_id,
|
||||
ref_: Some("@e3".into()),
|
||||
selector: None,
|
||||
tab_id: None,
|
||||
timeout_ms: Some(5_000),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("focus ok");
|
||||
assert!(result.focused);
|
||||
assert_eq!(result.used_ref.as_deref(), Some("e3"));
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn blur_round_trips_previous_focus_state() {
|
||||
let (handle, sock) = spawn_daemon().await;
|
||||
let mut ws = connect_ext(handle.ws_addr()).await;
|
||||
let _ = do_handshake(&mut ws).await;
|
||||
|
||||
run_extension(ws, |req| {
|
||||
assert_eq!(req.method, Method::ToolBlur);
|
||||
let params = req.params.clone().unwrap();
|
||||
let p: BlurParams = serde_json::from_value(params).unwrap();
|
||||
assert_eq!(p.selector.as_deref(), Some("#search"));
|
||||
ResponseBody::Ok(
|
||||
serde_json::to_value(BlurResult {
|
||||
tab_id: 9,
|
||||
used_ref: None,
|
||||
used_selector: Some("#search".into()),
|
||||
was_focused: true,
|
||||
focused: false,
|
||||
dialogs: vec![],
|
||||
})
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
let session_id = ipc_session_start(&sock).await;
|
||||
let result: BlurResult = ipc_tool_call(
|
||||
&sock,
|
||||
Method::ToolBlur,
|
||||
BlurParams {
|
||||
session_id,
|
||||
ref_: None,
|
||||
selector: Some("#search".into()),
|
||||
tab_id: None,
|
||||
timeout_ms: Some(5_000),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("blur ok");
|
||||
assert!(result.was_focused);
|
||||
assert!(!result.focused);
|
||||
assert_eq!(result.used_selector.as_deref(), Some("#search"));
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn fill_round_trips_clear_before_default() {
|
||||
let (handle, sock) = spawn_daemon().await;
|
||||
|
||||
Reference in New Issue
Block a user