feat(vom): add type primitive

This commit is contained in:
Ljy-0827
2026-08-31 17:48:45 +08:00
parent e069abaf2e
commit a1904f8141
23 changed files with 1776 additions and 156 deletions
+26 -3
View File
@@ -70,10 +70,31 @@ bsk navigate <url> --session <id>
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
bsk click @e4 --session <id> → or bsk fill, bsk type, bsk select, bsk press
bsk observe --session <id> → again after navigation / DOM change
```
### Choosing `fill`, `type`, and `press`
Choose by the intended postcondition, not by whether observation happens to
render `textbox [empty]`:
- `fill` sets a field's retained value. Use it when success means the target
value must equal the requested text; it replaces existing content by default.
- `type` inserts text at the active caret/selection in a live editing session.
Use it when the application must consume keyboard/IME text; it never clears
first and reports dispatch only.
- `press` sends a discrete key or shortcut such as `Enter`, `Tab`, or `Ctrl+A`.
Do not encode special keys inside `type --text`.
`type` requires an explicit targeting mode: pass a ref/selector when one is
available, or pass `--focused` only after an immediately preceding action in
the same tab deliberately established a text-entry focus. Never use `fill`
as a probe and never automatically retry a failed `fill` with `type`: an
application may already have consumed the text before clearing its proxy DOM
value. Re-observe first. After `type`, verify the application output with one
observation or screenshot; `verification=dispatched` is not business success.
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
@@ -88,7 +109,7 @@ If you can actually inspect image output, use `bsk screenshot --ref @eN --sessio
bsk click @eN --capture <capture-id> --image-x <px> --image-y <px> --session <id>
```
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/press workflow. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/hover/select reject them.
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/type/press workflow. Re-observe first: if the application exposes a real DOM editable, use its new ref; use `type --focused` only when the observation and immediately preceding click establish a focused text-entry session without a usable ref. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/type/hover/select reject them.
## Observation priority
@@ -212,7 +233,9 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
|---------|---------|
| `bsk click <ref-or-selector>` | Click a DOM element (`--button`, `--click-count`, `--modifiers`), or perform one screenshot-bound Surface point click with `--capture`, `--image-x`, and `--image-y` |
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
| `bsk fill <ref-or-selector> --value <text>` | Replace and verify a field's retained value |
| `bsk type <ref-or-selector> --text <text>` | Insert keyboard/IME text into an explicitly targeted editing session; reports dispatch only |
| `bsk type --focused --text <text>` | Insert into the unique focused text-entry session after an intentional focus-producing action |
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
+157 -1
View File
@@ -11,7 +11,7 @@ use anyhow::Context;
use bsk_protocol::Method;
use bsk_protocol::tools::{
ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier,
MouseButton, PressParams, PressResult, SelectParams, SelectResult,
MouseButton, PressParams, PressResult, SelectParams, SelectResult, TypeParams, TypeResult,
};
use clap::{Args, ValueEnum};
@@ -95,6 +95,27 @@ fn split_target(
}
}
fn split_type_target(
positional: Option<String>,
explicit_ref: Option<String>,
explicit_selector: Option<String>,
focused: bool,
) -> Result<(Option<String>, Option<String>), CliError> {
if focused {
if positional.is_some() || explicit_ref.is_some() || explicit_selector.is_some() {
return Err(CliError::Local(anyhow::anyhow!(
"pass a target or --focused, not both"
)));
}
return Ok((None, None));
}
split_target(positional, explicit_ref, explicit_selector).map_err(|_| {
CliError::Local(anyhow::anyhow!(
"type requires exactly one of: <target>, --ref, --selector, or --focused"
))
})
}
// ---------------------------------------------------------------------------
// bsk click
// ---------------------------------------------------------------------------
@@ -383,6 +404,95 @@ pub fn dispatch_fill(args: FillArgs, format: Format) -> Result<(), CliError> {
Ok(())
}
// ---------------------------------------------------------------------------
// bsk type
// ---------------------------------------------------------------------------
#[derive(Debug, Clone, Args)]
#[group(id = "type_destination", required = true, multiple = false)]
pub struct TypeDestinationArgs {
/// Snapshot ref (`@e3`, `e3`) or CSS selector to focus before insertion.
#[arg(value_name = "TARGET")]
pub target: Option<String>,
#[arg(long = "ref")]
pub ref_: Option<String>,
#[arg(long = "selector")]
pub selector: Option<String>,
/// Insert into the unique focused text-entry target instead of focusing a target.
#[arg(long)]
pub focused: bool,
}
#[derive(Debug, Clone, Args)]
pub struct TypeArgs {
#[command(flatten)]
pub destination: TypeDestinationArgs,
/// Text to dispatch through the live keyboard/IME editing session.
#[arg(long)]
pub text: String,
#[arg(long)]
pub session: String,
#[arg(long = "tab-id")]
pub tab_id: Option<i64>,
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
pub timeout: u32,
}
pub fn dispatch_type(args: TypeArgs, format: Format) -> Result<(), CliError> {
let (ref_, selector) = split_type_target(
args.destination.target.clone(),
args.destination.ref_.clone(),
args.destination.selector.clone(),
args.destination.focused,
)?;
let info = ensure_daemon().context("ensure daemon is running")?;
let params = TypeParams {
session_id: args.session.clone(),
text: args.text.clone(),
ref_,
selector,
focused: args.destination.focused,
tab_id: args.tab_id,
timeout_ms: Some(args.timeout),
};
let reply: TypeResult = call(
info.sock_path,
Method::ToolType,
params,
"type-1",
args.timeout,
)?;
match format {
Format::Json => {
println!(
"{}",
serde_json::to_string_pretty(&reply)
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
);
}
Format::Human => {
let target = if args.destination.focused {
"focused".into()
} else {
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref())
};
println!(
"type dispatched tab={} target={target} length={} verification=dispatched",
reply.tab_id, reply.text_length
);
print_dialog_summaries(&reply.dialogs);
}
}
Ok(())
}
// ---------------------------------------------------------------------------
// bsk press
// ---------------------------------------------------------------------------
@@ -659,6 +769,52 @@ mod tests {
assert_eq!(parsed.args.session, "ohli");
}
#[test]
fn type_clap_accepts_explicit_target_and_text_flag() {
use clap::Parser;
#[derive(Parser)]
#[command(name = "type")]
struct Wrapper {
#[command(flatten)]
args: TypeArgs,
}
let parsed =
Wrapper::try_parse_from(["type", "@e138", "--text", "测试", "--session", "ohli"])
.expect("type args should parse");
assert_eq!(parsed.args.destination.target.as_deref(), Some("@e138"));
assert_eq!(parsed.args.text, "测试");
assert_eq!(parsed.args.session, "ohli");
}
#[test]
fn type_targeting_requires_exactly_one_explicit_mode() {
use clap::Parser;
#[derive(Parser)]
#[command(name = "type")]
struct Wrapper {
#[command(flatten)]
args: TypeArgs,
}
let parsed =
Wrapper::try_parse_from(["type", "--focused", "--text", "测试", "--session", "ohli"])
.expect("focused type args should parse");
assert!(parsed.args.destination.focused);
assert!(parsed.args.destination.target.is_none());
let focused = split_type_target(None, None, None, true).unwrap();
assert_eq!(focused, (None, None));
let missing = split_type_target(None, None, None, false).unwrap_err();
assert!(missing.to_string().contains("--focused"));
let mixed = split_type_target(Some("@e1".into()), None, None, true).unwrap_err();
assert!(mixed.to_string().contains("not both"));
}
#[test]
fn select_clap_accepts_target_positional_with_repeated_value_flags() {
use clap::Parser;
+4 -1
View File
@@ -43,7 +43,7 @@ use crate::cli::evaluate::EvaluateArgs;
use crate::cli::get_html::GetHtmlArgs;
use crate::cli::human_loop::RequestHelpArgs;
use crate::cli::install_skill::InstallSkillArgs;
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs};
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs, TypeArgs};
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
use crate::cli::network::NetworkArgs;
use crate::cli::observe::ObserveArgs;
@@ -171,6 +171,9 @@ pub enum Command {
/// Fill an input / textarea / contenteditable.
Fill(FillArgs),
/// Insert text into an explicit target or focused keyboard/IME editing session.
Type(TypeArgs),
/// Dispatch a keyboard key combo.
Press(PressArgs),
+1
View File
@@ -244,6 +244,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
| Method::ToolClick
| Method::ToolHover
| Method::ToolFill
| Method::ToolType
| Method::ToolPress
| Method::ToolSelect
| Method::ToolEvaluate
+1
View File
@@ -95,6 +95,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
Command::Click(args) => cli::interaction::dispatch_click(args, format),
Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
Command::Type(args) => cli::interaction::dispatch_type(args, format),
Command::Press(args) => cli::interaction::dispatch_press(args, format),
Command::Select(args) => cli::interaction::dispatch_select(args, format),
Command::Evaluate(args) => cli::evaluate::dispatch(args, format),
+66 -3
View File
@@ -1,5 +1,4 @@
//! M7 integration: drive the 7 new tool RPCs (navigate /
//! navigate_back / navigate_forward / reload / click / fill / press)
//! M7 integration: drive the navigation and interaction tool RPCs
//! through the IPC + per-session queue + fake extension and assert
//! the wire shapes line up. The extension stub mirrors each method's
//! `*Result` so we exercise the daemon's serialise → forward →
@@ -16,7 +15,8 @@ use bsk_protocol::tools::{
ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, NavigateBackParams,
NavigateBackResult, NavigateForwardParams, NavigateForwardResult, NavigateParams,
NavigateResult, PressParams, PressResult, ReloadParams, ReloadResult, SelectParams,
SelectResult, SessionStartParams, SessionStartResult, WaitUntil,
SelectResult, SessionStartParams, SessionStartResult, TypeParams, TypeResult, TypeVerification,
WaitUntil,
};
use bsk_protocol::{
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
@@ -433,6 +433,51 @@ async fn fill_round_trips_clear_before_default() {
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn type_round_trips_dispatched_verification() {
let (handle, sock) = spawn_daemon().await;
let mut ws = connect_ext(handle.ws_addr()).await;
let _ = do_handshake(&mut ws).await;
run_extension(ws, |req| {
assert_eq!(req.method, Method::ToolType);
let p: TypeParams = serde_json::from_value(req.params.clone().unwrap()).unwrap();
assert_eq!(p.text, "测试");
assert!(!p.focused);
ResponseBody::Ok(
serde_json::to_value(TypeResult {
tab_id: 12,
used_ref: Some("e3".into()),
used_selector: None,
text_length: 2,
verification: TypeVerification::Dispatched,
dialogs: vec![],
})
.unwrap(),
)
});
let session_id = ipc_session_start(&sock).await;
let result: TypeResult = ipc_tool_call(
&sock,
Method::ToolType,
TypeParams {
session_id,
text: "测试".into(),
ref_: Some("@e3".into()),
selector: None,
focused: false,
tab_id: None,
timeout_ms: Some(5_000),
},
)
.await
.expect("type ok");
assert_eq!(result.text_length, 2);
assert_eq!(result.verification, TypeVerification::Dispatched);
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn press_round_trips_compound_key() {
let (handle, sock) = spawn_daemon().await;
@@ -535,6 +580,7 @@ async fn m7_tools_propagate_extension_errors() {
| Method::ToolReload
| Method::ToolClick
| Method::ToolFill
| Method::ToolType
| Method::ToolPress
| Method::ToolSelect => ResponseBody::Err(RpcError {
code: ErrorCode::CdpFailed,
@@ -642,6 +688,23 @@ async fn m7_tools_propagate_extension_errors() {
.unwrap_err();
assert_eq!(fill.code, ErrorCode::CdpFailed);
let type_result = ipc_tool_call::<_, Value>(
&sock,
Method::ToolType,
TypeParams {
session_id: session_id.clone(),
text: "hello".into(),
ref_: Some("@e1".into()),
selector: None,
focused: false,
tab_id: None,
timeout_ms: Some(5_000),
},
)
.await
.unwrap_err();
assert_eq!(type_result.code, ErrorCode::CdpFailed);
let press = ipc_tool_call::<_, Value>(
&sock,
Method::ToolPress,