mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(vom): add type primitive
This commit is contained in:
@@ -70,10 +70,31 @@ bsk navigate <url> --session <id>
|
||||
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
||||
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
||||
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
||||
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
||||
bsk click @e4 --session <id> → or bsk fill, bsk type, bsk select, bsk press
|
||||
bsk observe --session <id> → again after navigation / DOM change
|
||||
```
|
||||
|
||||
### Choosing `fill`, `type`, and `press`
|
||||
|
||||
Choose by the intended postcondition, not by whether observation happens to
|
||||
render `textbox [empty]`:
|
||||
|
||||
- `fill` sets a field's retained value. Use it when success means the target
|
||||
value must equal the requested text; it replaces existing content by default.
|
||||
- `type` inserts text at the active caret/selection in a live editing session.
|
||||
Use it when the application must consume keyboard/IME text; it never clears
|
||||
first and reports dispatch only.
|
||||
- `press` sends a discrete key or shortcut such as `Enter`, `Tab`, or `Ctrl+A`.
|
||||
Do not encode special keys inside `type --text`.
|
||||
|
||||
`type` requires an explicit targeting mode: pass a ref/selector when one is
|
||||
available, or pass `--focused` only after an immediately preceding action in
|
||||
the same tab deliberately established a text-entry focus. Never use `fill`
|
||||
as a probe and never automatically retry a failed `fill` with `type`: an
|
||||
application may already have consumed the text before clearing its proxy DOM
|
||||
value. Re-observe first. After `type`, verify the application output with one
|
||||
observation or screenshot; `verification=dispatched` is not business success.
|
||||
|
||||
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
||||
|
||||
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
||||
@@ -88,7 +109,7 @@ If you can actually inspect image output, use `bsk screenshot --ref @eN --sessio
|
||||
bsk click @eN --capture <capture-id> --image-x <px> --image-y <px> --session <id>
|
||||
```
|
||||
|
||||
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/press workflow. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/hover/select reject them.
|
||||
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/type/press workflow. Re-observe first: if the application exposes a real DOM editable, use its new ref; use `type --focused` only when the observation and immediately preceding click establish a focused text-entry session without a usable ref. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/type/hover/select reject them.
|
||||
|
||||
## Observation priority
|
||||
|
||||
@@ -212,7 +233,9 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
|
||||
|---------|---------|
|
||||
| `bsk click <ref-or-selector>` | Click a DOM element (`--button`, `--click-count`, `--modifiers`), or perform one screenshot-bound Surface point click with `--capture`, `--image-x`, and `--image-y` |
|
||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Replace and verify a field's retained value |
|
||||
| `bsk type <ref-or-selector> --text <text>` | Insert keyboard/IME text into an explicitly targeted editing session; reports dispatch only |
|
||||
| `bsk type --focused --text <text>` | Insert into the unique focused text-entry session after an intentional focus-producing action |
|
||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ use anyhow::Context;
|
||||
use bsk_protocol::Method;
|
||||
use bsk_protocol::tools::{
|
||||
ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier,
|
||||
MouseButton, PressParams, PressResult, SelectParams, SelectResult,
|
||||
MouseButton, PressParams, PressResult, SelectParams, SelectResult, TypeParams, TypeResult,
|
||||
};
|
||||
use clap::{Args, ValueEnum};
|
||||
|
||||
@@ -95,6 +95,27 @@ fn split_target(
|
||||
}
|
||||
}
|
||||
|
||||
fn split_type_target(
|
||||
positional: Option<String>,
|
||||
explicit_ref: Option<String>,
|
||||
explicit_selector: Option<String>,
|
||||
focused: bool,
|
||||
) -> Result<(Option<String>, Option<String>), CliError> {
|
||||
if focused {
|
||||
if positional.is_some() || explicit_ref.is_some() || explicit_selector.is_some() {
|
||||
return Err(CliError::Local(anyhow::anyhow!(
|
||||
"pass a target or --focused, not both"
|
||||
)));
|
||||
}
|
||||
return Ok((None, None));
|
||||
}
|
||||
split_target(positional, explicit_ref, explicit_selector).map_err(|_| {
|
||||
CliError::Local(anyhow::anyhow!(
|
||||
"type requires exactly one of: <target>, --ref, --selector, or --focused"
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// bsk click
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -383,6 +404,95 @@ pub fn dispatch_fill(args: FillArgs, format: Format) -> Result<(), CliError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// bsk type
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
#[group(id = "type_destination", required = true, multiple = false)]
|
||||
pub struct TypeDestinationArgs {
|
||||
/// Snapshot ref (`@e3`, `e3`) or CSS selector to focus before insertion.
|
||||
#[arg(value_name = "TARGET")]
|
||||
pub target: Option<String>,
|
||||
|
||||
#[arg(long = "ref")]
|
||||
pub ref_: Option<String>,
|
||||
|
||||
#[arg(long = "selector")]
|
||||
pub selector: Option<String>,
|
||||
|
||||
/// Insert into the unique focused text-entry target instead of focusing a target.
|
||||
#[arg(long)]
|
||||
pub focused: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct TypeArgs {
|
||||
#[command(flatten)]
|
||||
pub destination: TypeDestinationArgs,
|
||||
|
||||
/// Text to dispatch through the live keyboard/IME editing session.
|
||||
#[arg(long)]
|
||||
pub text: String,
|
||||
|
||||
#[arg(long)]
|
||||
pub session: String,
|
||||
|
||||
#[arg(long = "tab-id")]
|
||||
pub tab_id: Option<i64>,
|
||||
|
||||
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
|
||||
pub timeout: u32,
|
||||
}
|
||||
|
||||
pub fn dispatch_type(args: TypeArgs, format: Format) -> Result<(), CliError> {
|
||||
let (ref_, selector) = split_type_target(
|
||||
args.destination.target.clone(),
|
||||
args.destination.ref_.clone(),
|
||||
args.destination.selector.clone(),
|
||||
args.destination.focused,
|
||||
)?;
|
||||
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||
let params = TypeParams {
|
||||
session_id: args.session.clone(),
|
||||
text: args.text.clone(),
|
||||
ref_,
|
||||
selector,
|
||||
focused: args.destination.focused,
|
||||
tab_id: args.tab_id,
|
||||
timeout_ms: Some(args.timeout),
|
||||
};
|
||||
let reply: TypeResult = call(
|
||||
info.sock_path,
|
||||
Method::ToolType,
|
||||
params,
|
||||
"type-1",
|
||||
args.timeout,
|
||||
)?;
|
||||
match format {
|
||||
Format::Json => {
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&reply)
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
|
||||
);
|
||||
}
|
||||
Format::Human => {
|
||||
let target = if args.destination.focused {
|
||||
"focused".into()
|
||||
} else {
|
||||
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref())
|
||||
};
|
||||
println!(
|
||||
"type dispatched tab={} target={target} length={} verification=dispatched",
|
||||
reply.tab_id, reply.text_length
|
||||
);
|
||||
print_dialog_summaries(&reply.dialogs);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// bsk press
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -659,6 +769,52 @@ mod tests {
|
||||
assert_eq!(parsed.args.session, "ohli");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn type_clap_accepts_explicit_target_and_text_flag() {
|
||||
use clap::Parser;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "type")]
|
||||
struct Wrapper {
|
||||
#[command(flatten)]
|
||||
args: TypeArgs,
|
||||
}
|
||||
|
||||
let parsed =
|
||||
Wrapper::try_parse_from(["type", "@e138", "--text", "测试", "--session", "ohli"])
|
||||
.expect("type args should parse");
|
||||
assert_eq!(parsed.args.destination.target.as_deref(), Some("@e138"));
|
||||
assert_eq!(parsed.args.text, "测试");
|
||||
assert_eq!(parsed.args.session, "ohli");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn type_targeting_requires_exactly_one_explicit_mode() {
|
||||
use clap::Parser;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "type")]
|
||||
struct Wrapper {
|
||||
#[command(flatten)]
|
||||
args: TypeArgs,
|
||||
}
|
||||
|
||||
let parsed =
|
||||
Wrapper::try_parse_from(["type", "--focused", "--text", "测试", "--session", "ohli"])
|
||||
.expect("focused type args should parse");
|
||||
assert!(parsed.args.destination.focused);
|
||||
assert!(parsed.args.destination.target.is_none());
|
||||
|
||||
let focused = split_type_target(None, None, None, true).unwrap();
|
||||
assert_eq!(focused, (None, None));
|
||||
|
||||
let missing = split_type_target(None, None, None, false).unwrap_err();
|
||||
assert!(missing.to_string().contains("--focused"));
|
||||
|
||||
let mixed = split_type_target(Some("@e1".into()), None, None, true).unwrap_err();
|
||||
assert!(mixed.to_string().contains("not both"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_clap_accepts_target_positional_with_repeated_value_flags() {
|
||||
use clap::Parser;
|
||||
|
||||
@@ -43,7 +43,7 @@ use crate::cli::evaluate::EvaluateArgs;
|
||||
use crate::cli::get_html::GetHtmlArgs;
|
||||
use crate::cli::human_loop::RequestHelpArgs;
|
||||
use crate::cli::install_skill::InstallSkillArgs;
|
||||
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs};
|
||||
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs, TypeArgs};
|
||||
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
|
||||
use crate::cli::network::NetworkArgs;
|
||||
use crate::cli::observe::ObserveArgs;
|
||||
@@ -171,6 +171,9 @@ pub enum Command {
|
||||
/// Fill an input / textarea / contenteditable.
|
||||
Fill(FillArgs),
|
||||
|
||||
/// Insert text into an explicit target or focused keyboard/IME editing session.
|
||||
Type(TypeArgs),
|
||||
|
||||
/// Dispatch a keyboard key combo.
|
||||
Press(PressArgs),
|
||||
|
||||
|
||||
@@ -244,6 +244,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
|
||||
| Method::ToolClick
|
||||
| Method::ToolHover
|
||||
| Method::ToolFill
|
||||
| Method::ToolType
|
||||
| Method::ToolPress
|
||||
| Method::ToolSelect
|
||||
| Method::ToolEvaluate
|
||||
|
||||
@@ -95,6 +95,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
|
||||
Command::Click(args) => cli::interaction::dispatch_click(args, format),
|
||||
Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
|
||||
Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
|
||||
Command::Type(args) => cli::interaction::dispatch_type(args, format),
|
||||
Command::Press(args) => cli::interaction::dispatch_press(args, format),
|
||||
Command::Select(args) => cli::interaction::dispatch_select(args, format),
|
||||
Command::Evaluate(args) => cli::evaluate::dispatch(args, format),
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
//! M7 integration: drive the 7 new tool RPCs (navigate /
|
||||
//! navigate_back / navigate_forward / reload / click / fill / press)
|
||||
//! M7 integration: drive the navigation and interaction tool RPCs
|
||||
//! through the IPC + per-session queue + fake extension and assert
|
||||
//! the wire shapes line up. The extension stub mirrors each method's
|
||||
//! `*Result` so we exercise the daemon's serialise → forward →
|
||||
@@ -16,7 +15,8 @@ use bsk_protocol::tools::{
|
||||
ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, NavigateBackParams,
|
||||
NavigateBackResult, NavigateForwardParams, NavigateForwardResult, NavigateParams,
|
||||
NavigateResult, PressParams, PressResult, ReloadParams, ReloadResult, SelectParams,
|
||||
SelectResult, SessionStartParams, SessionStartResult, WaitUntil,
|
||||
SelectResult, SessionStartParams, SessionStartResult, TypeParams, TypeResult, TypeVerification,
|
||||
WaitUntil,
|
||||
};
|
||||
use bsk_protocol::{
|
||||
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
|
||||
@@ -433,6 +433,51 @@ async fn fill_round_trips_clear_before_default() {
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn type_round_trips_dispatched_verification() {
|
||||
let (handle, sock) = spawn_daemon().await;
|
||||
let mut ws = connect_ext(handle.ws_addr()).await;
|
||||
let _ = do_handshake(&mut ws).await;
|
||||
|
||||
run_extension(ws, |req| {
|
||||
assert_eq!(req.method, Method::ToolType);
|
||||
let p: TypeParams = serde_json::from_value(req.params.clone().unwrap()).unwrap();
|
||||
assert_eq!(p.text, "测试");
|
||||
assert!(!p.focused);
|
||||
ResponseBody::Ok(
|
||||
serde_json::to_value(TypeResult {
|
||||
tab_id: 12,
|
||||
used_ref: Some("e3".into()),
|
||||
used_selector: None,
|
||||
text_length: 2,
|
||||
verification: TypeVerification::Dispatched,
|
||||
dialogs: vec![],
|
||||
})
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
let session_id = ipc_session_start(&sock).await;
|
||||
let result: TypeResult = ipc_tool_call(
|
||||
&sock,
|
||||
Method::ToolType,
|
||||
TypeParams {
|
||||
session_id,
|
||||
text: "测试".into(),
|
||||
ref_: Some("@e3".into()),
|
||||
selector: None,
|
||||
focused: false,
|
||||
tab_id: None,
|
||||
timeout_ms: Some(5_000),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("type ok");
|
||||
assert_eq!(result.text_length, 2);
|
||||
assert_eq!(result.verification, TypeVerification::Dispatched);
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn press_round_trips_compound_key() {
|
||||
let (handle, sock) = spawn_daemon().await;
|
||||
@@ -535,6 +580,7 @@ async fn m7_tools_propagate_extension_errors() {
|
||||
| Method::ToolReload
|
||||
| Method::ToolClick
|
||||
| Method::ToolFill
|
||||
| Method::ToolType
|
||||
| Method::ToolPress
|
||||
| Method::ToolSelect => ResponseBody::Err(RpcError {
|
||||
code: ErrorCode::CdpFailed,
|
||||
@@ -642,6 +688,23 @@ async fn m7_tools_propagate_extension_errors() {
|
||||
.unwrap_err();
|
||||
assert_eq!(fill.code, ErrorCode::CdpFailed);
|
||||
|
||||
let type_result = ipc_tool_call::<_, Value>(
|
||||
&sock,
|
||||
Method::ToolType,
|
||||
TypeParams {
|
||||
session_id: session_id.clone(),
|
||||
text: "hello".into(),
|
||||
ref_: Some("@e1".into()),
|
||||
selector: None,
|
||||
focused: false,
|
||||
tab_id: None,
|
||||
timeout_ms: Some(5_000),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(type_result.code, ErrorCode::CdpFailed);
|
||||
|
||||
let press = ipc_tool_call::<_, Value>(
|
||||
&sock,
|
||||
Method::ToolPress,
|
||||
|
||||
Reference in New Issue
Block a user