mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(vom): add type primitive
This commit is contained in:
@@ -70,10 +70,31 @@ bsk navigate <url> --session <id>
|
||||
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
||||
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
||||
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
||||
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
||||
bsk click @e4 --session <id> → or bsk fill, bsk type, bsk select, bsk press
|
||||
bsk observe --session <id> → again after navigation / DOM change
|
||||
```
|
||||
|
||||
### Choosing `fill`, `type`, and `press`
|
||||
|
||||
Choose by the intended postcondition, not by whether observation happens to
|
||||
render `textbox [empty]`:
|
||||
|
||||
- `fill` sets a field's retained value. Use it when success means the target
|
||||
value must equal the requested text; it replaces existing content by default.
|
||||
- `type` inserts text at the active caret/selection in a live editing session.
|
||||
Use it when the application must consume keyboard/IME text; it never clears
|
||||
first and reports dispatch only.
|
||||
- `press` sends a discrete key or shortcut such as `Enter`, `Tab`, or `Ctrl+A`.
|
||||
Do not encode special keys inside `type --text`.
|
||||
|
||||
`type` requires an explicit targeting mode: pass a ref/selector when one is
|
||||
available, or pass `--focused` only after an immediately preceding action in
|
||||
the same tab deliberately established a text-entry focus. Never use `fill`
|
||||
as a probe and never automatically retry a failed `fill` with `type`: an
|
||||
application may already have consumed the text before clearing its proxy DOM
|
||||
value. Re-observe first. After `type`, verify the application output with one
|
||||
observation or screenshot; `verification=dispatched` is not business success.
|
||||
|
||||
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
||||
|
||||
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
||||
@@ -88,7 +109,7 @@ If you can actually inspect image output, use `bsk screenshot --ref @eN --sessio
|
||||
bsk click @eN --capture <capture-id> --image-x <px> --image-y <px> --session <id>
|
||||
```
|
||||
|
||||
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/press workflow. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/hover/select reject them.
|
||||
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing, navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates. A point click does not reveal Canvas semantics and must not be followed by an assumed fill/type/press workflow. Re-observe first: if the application exposes a real DOM editable, use its new ref; use `type --focused` only when the observation and immediately preceding click establish a focused text-entry session without a usable ref. Without all three capture arguments, visual Surface refs remain screenshot-only and click/fill/type/hover/select reject them.
|
||||
|
||||
## Observation priority
|
||||
|
||||
@@ -212,7 +233,9 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
|
||||
|---------|---------|
|
||||
| `bsk click <ref-or-selector>` | Click a DOM element (`--button`, `--click-count`, `--modifiers`), or perform one screenshot-bound Surface point click with `--capture`, `--image-x`, and `--image-y` |
|
||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Replace and verify a field's retained value |
|
||||
| `bsk type <ref-or-selector> --text <text>` | Insert keyboard/IME text into an explicitly targeted editing session; reports dispatch only |
|
||||
| `bsk type --focused --text <text>` | Insert into the unique focused text-entry session after an intentional focus-producing action |
|
||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
||||
|
||||
|
||||
@@ -11,7 +11,7 @@ use anyhow::Context;
|
||||
use bsk_protocol::Method;
|
||||
use bsk_protocol::tools::{
|
||||
ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier,
|
||||
MouseButton, PressParams, PressResult, SelectParams, SelectResult,
|
||||
MouseButton, PressParams, PressResult, SelectParams, SelectResult, TypeParams, TypeResult,
|
||||
};
|
||||
use clap::{Args, ValueEnum};
|
||||
|
||||
@@ -95,6 +95,27 @@ fn split_target(
|
||||
}
|
||||
}
|
||||
|
||||
fn split_type_target(
|
||||
positional: Option<String>,
|
||||
explicit_ref: Option<String>,
|
||||
explicit_selector: Option<String>,
|
||||
focused: bool,
|
||||
) -> Result<(Option<String>, Option<String>), CliError> {
|
||||
if focused {
|
||||
if positional.is_some() || explicit_ref.is_some() || explicit_selector.is_some() {
|
||||
return Err(CliError::Local(anyhow::anyhow!(
|
||||
"pass a target or --focused, not both"
|
||||
)));
|
||||
}
|
||||
return Ok((None, None));
|
||||
}
|
||||
split_target(positional, explicit_ref, explicit_selector).map_err(|_| {
|
||||
CliError::Local(anyhow::anyhow!(
|
||||
"type requires exactly one of: <target>, --ref, --selector, or --focused"
|
||||
))
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// bsk click
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -383,6 +404,95 @@ pub fn dispatch_fill(args: FillArgs, format: Format) -> Result<(), CliError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// bsk type
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
#[group(id = "type_destination", required = true, multiple = false)]
|
||||
pub struct TypeDestinationArgs {
|
||||
/// Snapshot ref (`@e3`, `e3`) or CSS selector to focus before insertion.
|
||||
#[arg(value_name = "TARGET")]
|
||||
pub target: Option<String>,
|
||||
|
||||
#[arg(long = "ref")]
|
||||
pub ref_: Option<String>,
|
||||
|
||||
#[arg(long = "selector")]
|
||||
pub selector: Option<String>,
|
||||
|
||||
/// Insert into the unique focused text-entry target instead of focusing a target.
|
||||
#[arg(long)]
|
||||
pub focused: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct TypeArgs {
|
||||
#[command(flatten)]
|
||||
pub destination: TypeDestinationArgs,
|
||||
|
||||
/// Text to dispatch through the live keyboard/IME editing session.
|
||||
#[arg(long)]
|
||||
pub text: String,
|
||||
|
||||
#[arg(long)]
|
||||
pub session: String,
|
||||
|
||||
#[arg(long = "tab-id")]
|
||||
pub tab_id: Option<i64>,
|
||||
|
||||
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
|
||||
pub timeout: u32,
|
||||
}
|
||||
|
||||
pub fn dispatch_type(args: TypeArgs, format: Format) -> Result<(), CliError> {
|
||||
let (ref_, selector) = split_type_target(
|
||||
args.destination.target.clone(),
|
||||
args.destination.ref_.clone(),
|
||||
args.destination.selector.clone(),
|
||||
args.destination.focused,
|
||||
)?;
|
||||
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||
let params = TypeParams {
|
||||
session_id: args.session.clone(),
|
||||
text: args.text.clone(),
|
||||
ref_,
|
||||
selector,
|
||||
focused: args.destination.focused,
|
||||
tab_id: args.tab_id,
|
||||
timeout_ms: Some(args.timeout),
|
||||
};
|
||||
let reply: TypeResult = call(
|
||||
info.sock_path,
|
||||
Method::ToolType,
|
||||
params,
|
||||
"type-1",
|
||||
args.timeout,
|
||||
)?;
|
||||
match format {
|
||||
Format::Json => {
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&reply)
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
|
||||
);
|
||||
}
|
||||
Format::Human => {
|
||||
let target = if args.destination.focused {
|
||||
"focused".into()
|
||||
} else {
|
||||
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref())
|
||||
};
|
||||
println!(
|
||||
"type dispatched tab={} target={target} length={} verification=dispatched",
|
||||
reply.tab_id, reply.text_length
|
||||
);
|
||||
print_dialog_summaries(&reply.dialogs);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// bsk press
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -659,6 +769,52 @@ mod tests {
|
||||
assert_eq!(parsed.args.session, "ohli");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn type_clap_accepts_explicit_target_and_text_flag() {
|
||||
use clap::Parser;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "type")]
|
||||
struct Wrapper {
|
||||
#[command(flatten)]
|
||||
args: TypeArgs,
|
||||
}
|
||||
|
||||
let parsed =
|
||||
Wrapper::try_parse_from(["type", "@e138", "--text", "测试", "--session", "ohli"])
|
||||
.expect("type args should parse");
|
||||
assert_eq!(parsed.args.destination.target.as_deref(), Some("@e138"));
|
||||
assert_eq!(parsed.args.text, "测试");
|
||||
assert_eq!(parsed.args.session, "ohli");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn type_targeting_requires_exactly_one_explicit_mode() {
|
||||
use clap::Parser;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(name = "type")]
|
||||
struct Wrapper {
|
||||
#[command(flatten)]
|
||||
args: TypeArgs,
|
||||
}
|
||||
|
||||
let parsed =
|
||||
Wrapper::try_parse_from(["type", "--focused", "--text", "测试", "--session", "ohli"])
|
||||
.expect("focused type args should parse");
|
||||
assert!(parsed.args.destination.focused);
|
||||
assert!(parsed.args.destination.target.is_none());
|
||||
|
||||
let focused = split_type_target(None, None, None, true).unwrap();
|
||||
assert_eq!(focused, (None, None));
|
||||
|
||||
let missing = split_type_target(None, None, None, false).unwrap_err();
|
||||
assert!(missing.to_string().contains("--focused"));
|
||||
|
||||
let mixed = split_type_target(Some("@e1".into()), None, None, true).unwrap_err();
|
||||
assert!(mixed.to_string().contains("not both"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn select_clap_accepts_target_positional_with_repeated_value_flags() {
|
||||
use clap::Parser;
|
||||
|
||||
@@ -43,7 +43,7 @@ use crate::cli::evaluate::EvaluateArgs;
|
||||
use crate::cli::get_html::GetHtmlArgs;
|
||||
use crate::cli::human_loop::RequestHelpArgs;
|
||||
use crate::cli::install_skill::InstallSkillArgs;
|
||||
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs};
|
||||
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs, TypeArgs};
|
||||
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
|
||||
use crate::cli::network::NetworkArgs;
|
||||
use crate::cli::observe::ObserveArgs;
|
||||
@@ -171,6 +171,9 @@ pub enum Command {
|
||||
/// Fill an input / textarea / contenteditable.
|
||||
Fill(FillArgs),
|
||||
|
||||
/// Insert text into an explicit target or focused keyboard/IME editing session.
|
||||
Type(TypeArgs),
|
||||
|
||||
/// Dispatch a keyboard key combo.
|
||||
Press(PressArgs),
|
||||
|
||||
|
||||
@@ -244,6 +244,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
|
||||
| Method::ToolClick
|
||||
| Method::ToolHover
|
||||
| Method::ToolFill
|
||||
| Method::ToolType
|
||||
| Method::ToolPress
|
||||
| Method::ToolSelect
|
||||
| Method::ToolEvaluate
|
||||
|
||||
@@ -95,6 +95,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
|
||||
Command::Click(args) => cli::interaction::dispatch_click(args, format),
|
||||
Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
|
||||
Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
|
||||
Command::Type(args) => cli::interaction::dispatch_type(args, format),
|
||||
Command::Press(args) => cli::interaction::dispatch_press(args, format),
|
||||
Command::Select(args) => cli::interaction::dispatch_select(args, format),
|
||||
Command::Evaluate(args) => cli::evaluate::dispatch(args, format),
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
//! M7 integration: drive the 7 new tool RPCs (navigate /
|
||||
//! navigate_back / navigate_forward / reload / click / fill / press)
|
||||
//! M7 integration: drive the navigation and interaction tool RPCs
|
||||
//! through the IPC + per-session queue + fake extension and assert
|
||||
//! the wire shapes line up. The extension stub mirrors each method's
|
||||
//! `*Result` so we exercise the daemon's serialise → forward →
|
||||
@@ -16,7 +15,8 @@ use bsk_protocol::tools::{
|
||||
ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, NavigateBackParams,
|
||||
NavigateBackResult, NavigateForwardParams, NavigateForwardResult, NavigateParams,
|
||||
NavigateResult, PressParams, PressResult, ReloadParams, ReloadResult, SelectParams,
|
||||
SelectResult, SessionStartParams, SessionStartResult, WaitUntil,
|
||||
SelectResult, SessionStartParams, SessionStartResult, TypeParams, TypeResult, TypeVerification,
|
||||
WaitUntil,
|
||||
};
|
||||
use bsk_protocol::{
|
||||
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
|
||||
@@ -433,6 +433,51 @@ async fn fill_round_trips_clear_before_default() {
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn type_round_trips_dispatched_verification() {
|
||||
let (handle, sock) = spawn_daemon().await;
|
||||
let mut ws = connect_ext(handle.ws_addr()).await;
|
||||
let _ = do_handshake(&mut ws).await;
|
||||
|
||||
run_extension(ws, |req| {
|
||||
assert_eq!(req.method, Method::ToolType);
|
||||
let p: TypeParams = serde_json::from_value(req.params.clone().unwrap()).unwrap();
|
||||
assert_eq!(p.text, "测试");
|
||||
assert!(!p.focused);
|
||||
ResponseBody::Ok(
|
||||
serde_json::to_value(TypeResult {
|
||||
tab_id: 12,
|
||||
used_ref: Some("e3".into()),
|
||||
used_selector: None,
|
||||
text_length: 2,
|
||||
verification: TypeVerification::Dispatched,
|
||||
dialogs: vec![],
|
||||
})
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
let session_id = ipc_session_start(&sock).await;
|
||||
let result: TypeResult = ipc_tool_call(
|
||||
&sock,
|
||||
Method::ToolType,
|
||||
TypeParams {
|
||||
session_id,
|
||||
text: "测试".into(),
|
||||
ref_: Some("@e3".into()),
|
||||
selector: None,
|
||||
focused: false,
|
||||
tab_id: None,
|
||||
timeout_ms: Some(5_000),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("type ok");
|
||||
assert_eq!(result.text_length, 2);
|
||||
assert_eq!(result.verification, TypeVerification::Dispatched);
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn press_round_trips_compound_key() {
|
||||
let (handle, sock) = spawn_daemon().await;
|
||||
@@ -535,6 +580,7 @@ async fn m7_tools_propagate_extension_errors() {
|
||||
| Method::ToolReload
|
||||
| Method::ToolClick
|
||||
| Method::ToolFill
|
||||
| Method::ToolType
|
||||
| Method::ToolPress
|
||||
| Method::ToolSelect => ResponseBody::Err(RpcError {
|
||||
code: ErrorCode::CdpFailed,
|
||||
@@ -642,6 +688,23 @@ async fn m7_tools_propagate_extension_errors() {
|
||||
.unwrap_err();
|
||||
assert_eq!(fill.code, ErrorCode::CdpFailed);
|
||||
|
||||
let type_result = ipc_tool_call::<_, Value>(
|
||||
&sock,
|
||||
Method::ToolType,
|
||||
TypeParams {
|
||||
session_id: session_id.clone(),
|
||||
text: "hello".into(),
|
||||
ref_: Some("@e1".into()),
|
||||
selector: None,
|
||||
focused: false,
|
||||
tab_id: None,
|
||||
timeout_ms: Some(5_000),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(type_result.code, ErrorCode::CdpFailed);
|
||||
|
||||
let press = ipc_tool_call::<_, Value>(
|
||||
&sock,
|
||||
Method::ToolPress,
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
"format": "int64"
|
||||
},
|
||||
"text": {
|
||||
"description": "Indented aria-snapshot text. Refs are rendered as `@e<N>` so the agent can copy them into subsequent `tool.click` / `tool.fill` selectors.",
|
||||
"description": "Indented aria-snapshot text. Refs are rendered as `@e<N>` so the agent can copy them into subsequent `tool.click` / `tool.fill` / `tool.type` targets.",
|
||||
"type": "string"
|
||||
},
|
||||
"truncated": {
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "TypeParams",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"session_id",
|
||||
"text"
|
||||
],
|
||||
"properties": {
|
||||
"focused": {
|
||||
"description": "Explicitly dispatch to the unique focused text-entry target instead of focusing a ref or selector.",
|
||||
"type": "boolean"
|
||||
},
|
||||
"ref": {
|
||||
"description": "Target to focus before dispatching text. Exactly one targeting mode must be selected: `ref` / `selector`, or `focused=true`.",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"selector": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"session_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"tab_id": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "int64"
|
||||
},
|
||||
"text": {
|
||||
"description": "Text to dispatch to the live editing session. Unlike `fill`, `type` does not require the focused DOM editable to retain this text.",
|
||||
"type": "string"
|
||||
},
|
||||
"timeout_ms": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "uint32",
|
||||
"minimum": 1.0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "TypeResult",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"tab_id",
|
||||
"text_length",
|
||||
"verification"
|
||||
],
|
||||
"properties": {
|
||||
"dialogs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/JavaScriptDialogInfo"
|
||||
}
|
||||
},
|
||||
"tab_id": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
},
|
||||
"text_length": {
|
||||
"description": "UTF-16 code-unit length of text accepted for dispatch.",
|
||||
"type": "integer",
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"used_ref": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"used_selector": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"verification": {
|
||||
"$ref": "#/definitions/TypeVerification"
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"JavaScriptDialogHandledAction": {
|
||||
"description": "How the extension resolved the dialog so CDP could continue.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"accepted",
|
||||
"dismissed"
|
||||
]
|
||||
},
|
||||
"JavaScriptDialogInfo": {
|
||||
"description": "One observed + handled JavaScript dialog during a tool call.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"handled",
|
||||
"message",
|
||||
"sequence",
|
||||
"tab_id",
|
||||
"type"
|
||||
],
|
||||
"properties": {
|
||||
"default_prompt": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"handled": {
|
||||
"$ref": "#/definitions/JavaScriptDialogHandledAction"
|
||||
},
|
||||
"has_browser_handler": {
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"message": {
|
||||
"type": "string"
|
||||
},
|
||||
"sequence": {
|
||||
"description": "Monotonic per-tab sequence for ordering within a session.",
|
||||
"type": "integer",
|
||||
"format": "uint64",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"tab_id": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
},
|
||||
"type": {
|
||||
"$ref": "#/definitions/JavaScriptDialogType"
|
||||
},
|
||||
"url": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"JavaScriptDialogType": {
|
||||
"description": "Native JS dialog kind reported by CDP.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"alert",
|
||||
"confirm",
|
||||
"prompt",
|
||||
"beforeunload"
|
||||
]
|
||||
},
|
||||
"TypeVerification": {
|
||||
"oneOf": [
|
||||
{
|
||||
"description": "Chromium accepted the complete keyboard/IME dispatch transaction. The caller must observe the application output to verify business state.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"dispatched"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -81,6 +81,8 @@ fn main() {
|
||||
dump!(HoverResult, "tool_hover_result");
|
||||
dump!(FillParams, "tool_fill_params");
|
||||
dump!(FillResult, "tool_fill_result");
|
||||
dump!(TypeParams, "tool_type_params");
|
||||
dump!(TypeResult, "tool_type_result");
|
||||
dump!(PressParams, "tool_press_params");
|
||||
dump!(PressResult, "tool_press_result");
|
||||
dump!(SelectParams, "tool_select_params");
|
||||
|
||||
@@ -74,6 +74,8 @@ pub enum Method {
|
||||
ToolHover,
|
||||
#[serde(rename = "tool.fill")]
|
||||
ToolFill,
|
||||
#[serde(rename = "tool.type")]
|
||||
ToolType,
|
||||
#[serde(rename = "tool.press")]
|
||||
ToolPress,
|
||||
#[serde(rename = "tool.select")]
|
||||
@@ -154,6 +156,7 @@ impl Method {
|
||||
| Method::ToolReload
|
||||
| Method::ToolClick
|
||||
| Method::ToolFill
|
||||
| Method::ToolType
|
||||
| Method::ToolPress
|
||||
| Method::ToolSelect
|
||||
| Method::ToolEvaluate
|
||||
@@ -246,6 +249,13 @@ mod tests {
|
||||
assert_eq!(serde_json::to_value(method).unwrap(), json!("tool.emulate"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn type_method_round_trips() {
|
||||
let method: Method = serde_json::from_value(json!("tool.type")).unwrap();
|
||||
assert_eq!(method, Method::ToolType);
|
||||
assert_eq!(serde_json::to_value(method).unwrap(), json!("tool.type"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cancel_params_and_result_round_trip() {
|
||||
let params: CancelParams = serde_json::from_value(json!({ "rpc_id": "wait-1" })).unwrap();
|
||||
@@ -284,6 +294,7 @@ mod tests {
|
||||
assert!(Method::ToolReload.is_mutating());
|
||||
assert!(Method::ToolClick.is_mutating());
|
||||
assert!(Method::ToolFill.is_mutating());
|
||||
assert!(Method::ToolType.is_mutating());
|
||||
assert!(Method::ToolPress.is_mutating());
|
||||
assert!(Method::ToolSelect.is_mutating());
|
||||
assert!(Method::ToolEvaluate.is_mutating());
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
//! DOM interaction tools (`tool.click`, `tool.fill`, `tool.press`,
|
||||
//! `tool.select`).
|
||||
//! DOM interaction tools (`tool.click`, `tool.fill`, `tool.type`,
|
||||
//! `tool.press`, `tool.select`).
|
||||
//!
|
||||
//! Element-targeted tools accept either a snapshot `ref` (`@e<N>` form,
|
||||
//! normalised against the session's RefStore) **or** a CSS selector
|
||||
@@ -188,6 +188,60 @@ pub struct FillResult {
|
||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// type
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct TypeParams {
|
||||
pub session_id: String,
|
||||
/// Text to dispatch to the live editing session. Unlike `fill`, `type`
|
||||
/// does not require the focused DOM editable to retain this text.
|
||||
pub text: String,
|
||||
/// Target to focus before dispatching text. Exactly one targeting mode
|
||||
/// must be selected: `ref` / `selector`, or `focused=true`.
|
||||
#[serde(
|
||||
rename = "ref",
|
||||
alias = "ref_",
|
||||
default,
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub ref_: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub selector: Option<String>,
|
||||
/// Explicitly dispatch to the unique focused text-entry target instead
|
||||
/// of focusing a ref or selector.
|
||||
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
|
||||
pub focused: bool,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub tab_id: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[schemars(range(min = 1))]
|
||||
pub timeout_ms: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum TypeVerification {
|
||||
/// Chromium accepted the complete keyboard/IME dispatch transaction.
|
||||
/// The caller must observe the application output to verify business state.
|
||||
Dispatched,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct TypeResult {
|
||||
pub tab_id: i64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub used_ref: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub used_selector: Option<String>,
|
||||
/// UTF-16 code-unit length of text accepted for dispatch.
|
||||
pub text_length: u32,
|
||||
pub verification: TypeVerification,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// press
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -351,6 +405,50 @@ mod tests {
|
||||
assert_eq!(v, json!("meta"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn type_payloads_distinguish_dispatch_from_value_verification() {
|
||||
let params = TypeParams {
|
||||
session_id: "abcd".into(),
|
||||
text: "测试".into(),
|
||||
ref_: Some("@e3".into()),
|
||||
selector: None,
|
||||
focused: false,
|
||||
tab_id: Some(4),
|
||||
timeout_ms: Some(5_000),
|
||||
};
|
||||
let encoded = serde_json::to_value(¶ms).unwrap();
|
||||
assert_eq!(encoded["ref"], json!("@e3"));
|
||||
assert_eq!(encoded["text"], json!("测试"));
|
||||
assert!(encoded.get("focused").is_none());
|
||||
|
||||
let focused = TypeParams {
|
||||
session_id: "abcd".into(),
|
||||
text: "hello".into(),
|
||||
ref_: None,
|
||||
selector: None,
|
||||
focused: true,
|
||||
tab_id: None,
|
||||
timeout_ms: None,
|
||||
};
|
||||
assert_eq!(
|
||||
serde_json::to_value(focused).unwrap()["focused"],
|
||||
json!(true)
|
||||
);
|
||||
|
||||
let result = TypeResult {
|
||||
tab_id: 4,
|
||||
used_ref: Some("e3".into()),
|
||||
used_selector: None,
|
||||
text_length: 2,
|
||||
verification: TypeVerification::Dispatched,
|
||||
dialogs: vec![],
|
||||
};
|
||||
assert_eq!(
|
||||
serde_json::to_value(result).unwrap()["verification"],
|
||||
json!("dispatched")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn press_result_round_trips() {
|
||||
let r = PressResult {
|
||||
|
||||
@@ -33,8 +33,8 @@ pub struct SnapshotParams {
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct SnapshotResult {
|
||||
/// Indented aria-snapshot text. Refs are rendered as `@e<N>` so the
|
||||
/// agent can copy them into subsequent `tool.click` / `tool.fill`
|
||||
/// selectors.
|
||||
/// agent can copy them into subsequent `tool.click` / `tool.fill` /
|
||||
/// `tool.type` targets.
|
||||
pub text: String,
|
||||
/// Number of `@e<N>` refs registered for this session by this
|
||||
/// snapshot. Equivalent to the size of the new ref-store map.
|
||||
|
||||
Reference in New Issue
Block a user