figmar/BrowserSkill
1
0
Fork 0
mirror of https://github.com/Tencent/BrowserSkill.git synced 2026-09-10 13:01:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(interaction): add scroll-to-element primitive

Browse Source
This commit is contained in:
NianJiuZst
2026-08-21 20:56:24 +08:00
parent dd9fc9edba
commit 7a10be21cb
19 changed files with 672 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/bsk-cli/skill/SKILL.md
+1
View File
@@ -202,6 +202,7 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
|---------|---------|
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
| `bsk scroll-to <ref-or-selector>` | Scroll an element and its frame owners into the visible viewport |
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
crates/bsk-cli/src/cli/mod.rs
+6
View File
@@ -25,6 +25,7 @@ pub mod record;
pub mod record_state;
pub mod render_error;
pub mod screenshot;
pub mod scroll;
pub mod session;
pub mod snapshot;
pub mod status;
@@ -48,6 +49,7 @@ use crate::cli::network::NetworkArgs;
use crate::cli::observe::ObserveArgs;
use crate::cli::record::RecordCmd;
use crate::cli::screenshot::ScreenshotArgs;
use crate::cli::scroll::ScrollToArgs;
use crate::cli::session::SessionCmd;
use crate::cli::snapshot::SnapshotArgs;
use crate::cli::tab::TabCmd;
@@ -167,6 +169,10 @@ pub enum Command {
/// Hover a snapshot ref or CSS selector.
Hover(HoverArgs),
/// Scroll a snapshot ref or CSS selector into the visible viewport.
#[command(name = "scroll-to")]
ScrollTo(ScrollToArgs),
/// Fill an input / textarea / contenteditable.
Fill(FillArgs),
crates/bsk-cli/src/cli/scroll.rs
+117
View File
@@ -0,0 +1,117 @@
//! `bsk scroll-to` — bring a target element into the visible viewport.
use std::time::Duration;
use anyhow::Context;
use bsk_protocol::Method;
use bsk_protocol::tools::{ScrollToParams, ScrollToResult};
use clap::Args;
use crate::cli::ensure_daemon::ensure_daemon;
use crate::cli::error::{CliError, Format};
use crate::cli::interaction::looks_like_ref;
use crate::cli::navigate::parse_timeout_ms;
#[derive(Debug, Clone, Args)]
pub struct ScrollToArgs {
/// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used.
pub target: Option<String>,
#[arg(long = "ref")]
pub ref_: Option<String>,
#[arg(long = "selector")]
pub selector: Option<String>,
#[arg(long)]
pub session: String,
#[arg(long = "tab-id")]
pub tab_id: Option<i64>,
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
pub timeout: u32,
}
pub fn dispatch(args: ScrollToArgs, format: Format) -> Result<(), CliError> {
let info = ensure_daemon().context("ensure daemon is running")?;
let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?;
let params = ScrollToParams {
session_id: args.session,
ref_,
selector,
tab_id: args.tab_id,
timeout_ms: Some(args.timeout),
};
let reply: ScrollToResult = crate::cli::business_rpc::call(
info.sock_path,
"scroll-to-1",
Method::ToolScrollTo,
Some(params),
ipc_timeout(args.timeout),
)?;
match format {
Format::Json => println!(
"{}",
serde_json::to_string_pretty(&reply)
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
),
Format::Human => {
let target =
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref());
println!(
"scroll-to ok tab={} target={target} bounds=({}, {}, {}, {})",
reply.tab_id, reply.x, reply.y, reply.width, reply.height
);
}
}
Ok(())
}
fn split_target(
positional: Option<String>,
explicit_ref: Option<String>,
explicit_selector: Option<String>,
) -> Result<(Option<String>, Option<String>), CliError> {
match (positional, explicit_ref, explicit_selector) {
(None, None, None) => Err(CliError::Local(anyhow::anyhow!(
"missing target: pass <ref-or-selector>, --ref @eN, or --selector <css>"
))),
(None, Some(r), None) => Ok((Some(r), None)),
(None, None, Some(s)) => Ok((None, Some(s))),
(Some(target), None, None) if looks_like_ref(&target) => Ok((Some(target), None)),
(Some(target), None, None) => Ok((None, Some(target))),
_ => Err(CliError::Local(anyhow::anyhow!(
"pass exactly one of: <target>, --ref, or --selector"
))),
}
}
fn format_used_target(used_ref: Option<&str>, used_selector: Option<&str>) -> String {
used_ref
.map(|r| format!("@{r}"))
.or_else(|| used_selector.map(str::to_string))
.unwrap_or_else(|| "?".into())
}
fn ipc_timeout(timeout_ms: u32) -> Duration {
Duration::from_millis(u64::from(timeout_ms))
.checked_add(Duration::from_secs(15))
.unwrap_or(Duration::from_secs(u64::from(timeout_ms / 1_000) + 15))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn split_target_detects_refs_and_selectors() {
let (ref_, selector) = split_target(Some("@e3".into()), None, None).unwrap();
assert_eq!(ref_.as_deref(), Some("@e3"));
assert!(selector.is_none());
let (ref_, selector) = split_target(Some("#target".into()), None, None).unwrap();
assert!(ref_.is_none());
assert_eq!(selector.as_deref(), Some("#target"));
}
}
crates/bsk-cli/src/daemon/ipc.rs
+1
View File
@@ -243,6 +243,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
| Method::ToolReload
| Method::ToolClick
| Method::ToolHover
| Method::ToolScrollTo
| Method::ToolFill
| Method::ToolPress
| Method::ToolSelect
crates/bsk-cli/src/main.rs
+1
View File
@@ -94,6 +94,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
Command::Reload(args) => cli::navigate::dispatch_reload(args, format),
Command::Click(args) => cli::interaction::dispatch_click(args, format),
Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
Command::ScrollTo(args) => cli::scroll::dispatch(args, format),
Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
Command::Press(args) => cli::interaction::dispatch_press(args, format),
Command::Select(args) => cli::interaction::dispatch_select(args, format),
crates/bsk-cli/tests/cli_parse.rs
+9
View File
@@ -206,6 +206,15 @@ fn parses_hover_with_settle() {
assert_eq!(args.settle, 300);
}
#[test]
fn parses_scroll_to_target() {
let cli = parse(&["bsk", "scroll-to", "@e2", "--session", "s1"]);
let Command::ScrollTo(args) = cli.command else {
panic!("expected scroll-to command");
};
assert_eq!(args.target.as_deref(), Some("@e2"));
}
#[test]
fn rejects_zero_click_count() {
assert!(
crates/bsk-cli/tests/tools_m7_ipc.rs
+48 -4
View File
@@ -1,5 +1,4 @@
//! M7 integration: drive the 7 new tool RPCs (navigate /
//! navigate_back / navigate_forward / reload / click / fill / press)
//! M7 integration: drive interaction and navigation tool RPCs
//! through the IPC + per-session queue + fake extension and assert
//! the wire shapes line up. The extension stub mirrors each method's
//! `*Result` so we exercise the daemon's serialise → forward →
@@ -15,8 +14,8 @@ use bsk_protocol::system::{HandshakeParams, HandshakeResult};
use bsk_protocol::tools::{
ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, NavigateBackParams,
NavigateBackResult, NavigateForwardParams, NavigateForwardResult, NavigateParams,
NavigateResult, PressParams, PressResult, ReloadParams, ReloadResult, SelectParams,
SelectResult, SessionStartParams, SessionStartResult, WaitUntil,
NavigateResult, PressParams, PressResult, ReloadParams, ReloadResult, ScrollToParams,
ScrollToResult, SelectParams, SelectResult, SessionStartParams, SessionStartResult, WaitUntil,
};
use bsk_protocol::{
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
@@ -382,6 +381,51 @@ async fn click_round_trips_ref_and_modifiers() {
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn scroll_to_round_trips_visible_bounds() {
let (handle, sock) = spawn_daemon().await;
let mut ws = connect_ext(handle.ws_addr()).await;
let _ = do_handshake(&mut ws).await;
run_extension(ws, |req| {
assert_eq!(req.method, Method::ToolScrollTo);
let params = req.params.clone().unwrap();
let parsed: ScrollToParams = serde_json::from_value(params).unwrap();
assert_eq!(parsed.ref_.as_deref(), Some("@e3"));
ResponseBody::Ok(
serde_json::to_value(ScrollToResult {
tab_id: 9,
used_ref: Some("e3".into()),
used_selector: None,
x: 10.0,
y: 20.0,
width: 100.0,
height: 40.0,
dialogs: vec![],
})
.unwrap(),
)
});
let session_id = ipc_session_start(&sock).await;
let result: ScrollToResult = ipc_tool_call(
&sock,
Method::ToolScrollTo,
ScrollToParams {
session_id,
ref_: Some("@e3".into()),
selector: None,
tab_id: None,
timeout_ms: Some(5_000),
},
)
.await
.expect("scroll-to ok");
assert_eq!(result.used_ref.as_deref(), Some("e3"));
assert_eq!(result.width, 100.0);
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn fill_round_trips_clear_before_default() {
let (handle, sock) = spawn_daemon().await;
crates/bsk-protocol/schema/tool_scroll_to_params.json
+42
View File
@@ -0,0 +1,42 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "ScrollToParams",
"type": "object",
"required": [
"session_id"
],
"properties": {
"ref": {
"description": "Optional `@e<N>` ref allocated by the last observation. Mutually exclusive with `selector`.",
"type": [
"string",
"null"
]
},
"selector": {
"type": [
"string",
"null"
]
},
"session_id": {
"type": "string"
},
"tab_id": {
"description": "Target tab. Defaults to the Agent Window's active tab.",
"type": [
"integer",
"null"
],
"format": "int64"
},
"timeout_ms": {
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 1.0
}
}
}
crates/bsk-protocol/schema/tool_scroll_to_result.json
+123
View File
@@ -0,0 +1,123 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "ScrollToResult",
"type": "object",
"required": [
"height",
"tab_id",
"width",
"x",
"y"
],
"properties": {
"dialogs": {
"type": "array",
"items": {
"$ref": "#/definitions/JavaScriptDialogInfo"
}
},
"height": {
"type": "number",
"format": "double"
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"used_ref": {
"type": [
"string",
"null"
]
},
"used_selector": {
"type": [
"string",
"null"
]
},
"width": {
"type": "number",
"format": "double"
},
"x": {
"description": "Visible target bounds in top-level viewport CSS pixels.",
"type": "number",
"format": "double"
},
"y": {
"type": "number",
"format": "double"
}
},
"definitions": {
"JavaScriptDialogHandledAction": {
"description": "How the extension resolved the dialog so CDP could continue.",
"type": "string",
"enum": [
"accepted",
"dismissed"
]
},
"JavaScriptDialogInfo": {
"description": "One observed + handled JavaScript dialog during a tool call.",
"type": "object",
"required": [
"handled",
"message",
"sequence",
"tab_id",
"type"
],
"properties": {
"default_prompt": {
"type": [
"string",
"null"
]
},
"handled": {
"$ref": "#/definitions/JavaScriptDialogHandledAction"
},
"has_browser_handler": {
"type": [
"boolean",
"null"
]
},
"message": {
"type": "string"
},
"sequence": {
"description": "Monotonic per-tab sequence for ordering within a session.",
"type": "integer",
"format": "uint64",
"minimum": 0.0
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"type": {
"$ref": "#/definitions/JavaScriptDialogType"
},
"url": {
"type": [
"string",
"null"
]
}
}
},
"JavaScriptDialogType": {
"description": "Native JS dialog kind reported by CDP.",
"type": "string",
"enum": [
"alert",
"confirm",
"prompt",
"beforeunload"
]
}
}
}
crates/bsk-protocol/src/bin/dump-schema.rs
+2
View File
@@ -79,6 +79,8 @@ fn main() {
dump!(ClickResult, "tool_click_result");
dump!(HoverParams, "tool_hover_params");
dump!(HoverResult, "tool_hover_result");
dump!(ScrollToParams, "tool_scroll_to_params");
dump!(ScrollToResult, "tool_scroll_to_result");
dump!(FillParams, "tool_fill_params");
dump!(FillResult, "tool_fill_result");
dump!(PressParams, "tool_press_params");
crates/bsk-protocol/src/method.rs
+6
View File
@@ -72,6 +72,8 @@ pub enum Method {
ToolClick,
#[serde(rename = "tool.hover")]
ToolHover,
#[serde(rename = "tool.scroll_to")]
ToolScrollTo,
#[serde(rename = "tool.fill")]
ToolFill,
#[serde(rename = "tool.press")]
@@ -153,6 +155,7 @@ impl Method {
| Method::ToolNavigateForward
| Method::ToolReload
| Method::ToolClick
| Method::ToolScrollTo
| Method::ToolFill
| Method::ToolPress
| Method::ToolSelect
@@ -283,6 +286,7 @@ mod tests {
assert!(Method::ToolNavigateForward.is_mutating());
assert!(Method::ToolReload.is_mutating());
assert!(Method::ToolClick.is_mutating());
assert!(Method::ToolScrollTo.is_mutating());
assert!(Method::ToolFill.is_mutating());
assert!(Method::ToolPress.is_mutating());
assert!(Method::ToolSelect.is_mutating());
@@ -327,6 +331,7 @@ mod tests {
assert_eq!(Method::ToolHover.effect(), MethodEffect::TransientInput);
assert_eq!(Method::ToolObserve.effect(), MethodEffect::TransientInput);
assert_eq!(Method::ToolClick.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::ToolScrollTo.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::Cancel.effect(), MethodEffect::ControlPlane);
}
@@ -336,6 +341,7 @@ mod tests {
assert!(Method::ToolHover.requires_interrupt_gate());
assert!(Method::ToolObserve.requires_interrupt_gate());
assert!(Method::ToolClick.requires_interrupt_gate());
assert!(Method::ToolScrollTo.requires_interrupt_gate());
assert!(!Method::Cancel.requires_interrupt_gate());
}
}
crates/bsk-protocol/src/tools/mod.rs
+2
View File
@@ -10,6 +10,7 @@ pub mod network;
pub mod observation;
pub mod record;
pub mod script;
pub mod scroll;
pub mod session;
pub mod tabs;
pub mod waits;
@@ -25,6 +26,7 @@ pub use network::*;
pub use observation::*;
pub use record::*;
pub use script::*;
pub use scroll::*;
pub use session::*;
pub use tabs::*;
pub use waits::*;
crates/bsk-protocol/src/tools/scroll.rs
+65
View File
@@ -0,0 +1,65 @@
//! Element scrolling primitive (`tool.scroll_to`).
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use super::JavaScriptDialogInfo;
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct ScrollToParams {
pub session_id: String,
/// Optional `@e<N>` ref allocated by the last observation.
/// Mutually exclusive with `selector`.
#[serde(
rename = "ref",
alias = "ref_",
default,
skip_serializing_if = "Option::is_none"
)]
pub ref_: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selector: Option<String>,
/// Target tab. Defaults to the Agent Window's active tab.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tab_id: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 1))]
pub timeout_ms: Option<u32>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct ScrollToResult {
pub tab_id: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_ref: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_selector: Option<String>,
/// Visible target bounds in top-level viewport CSS pixels.
pub x: f64,
pub y: f64,
pub width: f64,
pub height: f64,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dialogs: Vec<JavaScriptDialogInfo>,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn params_serialise_ref_field_name() {
let params = ScrollToParams {
session_id: "abcd".into(),
ref_: Some("@e3".into()),
selector: None,
tab_id: Some(42),
timeout_ms: Some(5_000),
};
let value = serde_json::to_value(&params).unwrap();
assert_eq!(value.get("ref").and_then(|v| v.as_str()), Some("@e3"));
assert!(value.get("ref_").is_none());
let round: ScrollToParams = serde_json::from_value(value).unwrap();
assert_eq!(round, params);
}
}