figmar/BrowserSkill
1
0
Fork 0
mirror of https://github.com/Tencent/BrowserSkill.git synced 2026-09-10 13:01:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(extension): add bsk hover and adjust vom to support

Browse Source
This commit is contained in:
Ljy-0827
2026-08-06 12:19:45 +08:00
parent 0f264c5df1
commit 91225e5fdb
31 changed files with 1051 additions and 47 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/bsk-cli/skill/SKILL.md
+11 -7
View File
@@ -67,22 +67,25 @@ Write operations only affect tabs in the **Agent Window** (or tabs you **borrowe
```
bsk navigate <url> --session <id>
bsk snapshot --session <id> → aria tree with @e1, @e2, … refs
bsk observe --session <id> → semantic VOM view; may reveal hover/focus surfaces
bsk click @e3 --session <id> → or bsk fill, bsk select, bsk press
bsk snapshot --session <id> → again after navigation / DOM change
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
bsk observe --session <id> → again after navigation / DOM change
```
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
## Observation priority
Start with `bsk snapshot` to understand page structure, text, controls, and element refs. Use `bsk observe` when semantic VOM output or conditional hover/focus surfaces would materially help. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
1. `bsk snapshot` — strict static page understanding and interaction planning
2. `bsk observe` — semantic VOM observation; may run bounded perception probes such as hover-surface discovery
1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
2. `bsk snapshot` — strict static accessibility tree fallback
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
@@ -186,6 +189,7 @@ bsk emulate --session <id> --off
| Command | Summary |
|---------|---------|
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
crates/bsk-cli/src/cli/interaction.rs
+87 -2
View File
@@ -10,8 +10,8 @@ use std::time::Duration;
use anyhow::Context;
use bsk_protocol::Method;
use bsk_protocol::tools::{
ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, PressParams,
PressResult, SelectParams, SelectResult,
ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier,
MouseButton, PressParams, PressResult, SelectParams, SelectResult,
};
use clap::{Args, ValueEnum};
@@ -189,6 +189,91 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
Ok(())
}
// ---------------------------------------------------------------------------
// bsk hover
// ---------------------------------------------------------------------------
#[derive(Debug, Clone, Args)]
pub struct HoverArgs {
/// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used.
pub target: Option<String>,
/// Force-treat the value as a snapshot ref (overrides positional detection).
#[arg(long = "ref")]
pub ref_: Option<String>,
/// Force-treat the value as a CSS selector.
#[arg(long = "selector")]
pub selector: Option<String>,
#[arg(long)]
pub session: String,
#[arg(long = "tab-id")]
pub tab_id: Option<i64>,
/// Comma-separated modifiers (`alt,ctrl,shift,meta`).
#[arg(long, default_value = "")]
pub modifiers: String,
/// Wait after moving the mouse so hover-triggered UI can settle.
#[arg(long = "settle", default_value = "200ms", value_parser = parse_timeout_ms)]
pub settle: u32,
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
pub timeout: u32,
}
pub fn dispatch_hover(args: HoverArgs, format: Format) -> Result<(), CliError> {
let info = ensure_daemon().context("ensure daemon is running")?;
let (ref_, selector) = split_target(
args.target.clone(),
args.ref_.clone(),
args.selector.clone(),
)?;
let modifiers = parse_modifiers(&args.modifiers)
.map_err(|e| CliError::Local(anyhow::anyhow!("--modifiers: {e}")))?;
let params = HoverParams {
session_id: args.session.clone(),
ref_,
selector,
tab_id: args.tab_id,
modifiers: if modifiers.is_empty() {
None
} else {
Some(modifiers)
},
settle_ms: Some(args.settle),
timeout_ms: Some(args.timeout),
};
let reply: HoverResult = call(
info.sock_path,
Method::ToolHover,
params,
"hover-1",
args.timeout,
)?;
match format {
Format::Json => {
println!(
"{}",
serde_json::to_string_pretty(&reply)
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
);
}
Format::Human => {
let target =
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref());
println!(
"hover ok tab={} target={target} at=({}, {})",
reply.tab_id, reply.x, reply.y
);
print_dialog_summaries(&reply.dialogs);
}
}
Ok(())
}
// ---------------------------------------------------------------------------
// bsk fill
// ---------------------------------------------------------------------------
crates/bsk-cli/src/cli/mod.rs
+4 -1
View File
@@ -42,7 +42,7 @@ use crate::cli::evaluate::EvaluateArgs;
use crate::cli::get_html::GetHtmlArgs;
use crate::cli::human_loop::RequestHelpArgs;
use crate::cli::install_skill::InstallSkillArgs;
use crate::cli::interaction::{ClickArgs, FillArgs, PressArgs, SelectArgs};
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs};
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
use crate::cli::network::NetworkArgs;
use crate::cli::observe::ObserveArgs;
@@ -164,6 +164,9 @@ pub enum Command {
/// Click a snapshot ref or CSS selector.
Click(ClickArgs),
/// Hover a snapshot ref or CSS selector.
Hover(HoverArgs),
/// Fill an input / textarea / contenteditable.
Fill(FillArgs),
crates/bsk-cli/src/daemon/ipc.rs
+1
View File
@@ -242,6 +242,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
| Method::ToolNavigateForward
| Method::ToolReload
| Method::ToolClick
| Method::ToolHover
| Method::ToolFill
| Method::ToolPress
| Method::ToolSelect
crates/bsk-cli/src/main.rs
+1
View File
@@ -93,6 +93,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
Command::NavigateForward(args) => cli::navigate::dispatch_navigate_forward(args, format),
Command::Reload(args) => cli::navigate::dispatch_reload(args, format),
Command::Click(args) => cli::interaction::dispatch_click(args, format),
Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
Command::Press(args) => cli::interaction::dispatch_press(args, format),
Command::Select(args) => cli::interaction::dispatch_select(args, format),
crates/bsk-cli/tests/cli_parse.rs
+18
View File
@@ -187,6 +187,24 @@ fn parses_click_count_alias() {
assert_eq!(args.click_count, 2);
}
#[test]
fn parses_hover_with_settle() {
let cli = parse(&[
"bsk",
"hover",
"@e1",
"--session",
"s1",
"--settle",
"300ms",
]);
let Command::Hover(args) = cli.command else {
panic!("expected hover command");
};
assert_eq!(args.target.as_deref(), Some("@e1"));
assert_eq!(args.settle, 300);
}
#[test]
fn rejects_zero_click_count() {
assert!(
crates/bsk-protocol/schema/tool_hover_params.json
+72
View File
@@ -0,0 +1,72 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "HoverParams",
"type": "object",
"required": [
"session_id"
],
"properties": {
"modifiers": {
"type": [
"array",
"null"
],
"items": {
"$ref": "#/definitions/KeyModifier"
}
},
"ref": {
"description": "Optional `@e<N>` ref allocated by the last observation. Mutually exclusive with `selector`.",
"type": [
"string",
"null"
]
},
"selector": {
"type": [
"string",
"null"
]
},
"session_id": {
"type": "string"
},
"settle_ms": {
"description": "Wait after the mouse move so hover-triggered UI can settle.",
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 0.0
},
"tab_id": {
"description": "Target tab. Defaults to the Agent Window's active tab.",
"type": [
"integer",
"null"
],
"format": "int64"
},
"timeout_ms": {
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 1.0
}
},
"definitions": {
"KeyModifier": {
"description": "Keyboard modifier flags. Multiple flags may be combined; the extension folds them into CDP's bitfield (`alt=1, ctrl=2, meta=4, shift=8`).",
"type": "string",
"enum": [
"alt",
"ctrl",
"meta",
"shift"
]
}
}
}
crates/bsk-protocol/schema/tool_hover_result.json
+113
View File
@@ -0,0 +1,113 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "HoverResult",
"type": "object",
"required": [
"tab_id",
"x",
"y"
],
"properties": {
"dialogs": {
"type": "array",
"items": {
"$ref": "#/definitions/JavaScriptDialogInfo"
}
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"used_ref": {
"type": [
"string",
"null"
]
},
"used_selector": {
"type": [
"string",
"null"
]
},
"x": {
"description": "Viewport-relative hover coordinates (CSS pixels).",
"type": "number",
"format": "double"
},
"y": {
"type": "number",
"format": "double"
}
},
"definitions": {
"JavaScriptDialogHandledAction": {
"description": "How the extension resolved the dialog so CDP could continue.",
"type": "string",
"enum": [
"accepted",
"dismissed"
]
},
"JavaScriptDialogInfo": {
"description": "One observed + handled JavaScript dialog during a tool call.",
"type": "object",
"required": [
"handled",
"message",
"sequence",
"tab_id",
"type"
],
"properties": {
"default_prompt": {
"type": [
"string",
"null"
]
},
"handled": {
"$ref": "#/definitions/JavaScriptDialogHandledAction"
},
"has_browser_handler": {
"type": [
"boolean",
"null"
]
},
"message": {
"type": "string"
},
"sequence": {
"description": "Monotonic per-tab sequence for ordering within a session.",
"type": "integer",
"format": "uint64",
"minimum": 0.0
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"type": {
"$ref": "#/definitions/JavaScriptDialogType"
},
"url": {
"type": [
"string",
"null"
]
}
}
},
"JavaScriptDialogType": {
"description": "Native JS dialog kind reported by CDP.",
"type": "string",
"enum": [
"alert",
"confirm",
"prompt",
"beforeunload"
]
}
}
}
crates/bsk-protocol/schema/tool_record_await_result.json
+40
View File
@@ -144,6 +144,46 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
crates/bsk-protocol/schema/tool_record_stop_result.json
+40
View File
@@ -144,6 +144,46 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
crates/bsk-protocol/schema/trace.json
+40
View File
@@ -177,6 +177,46 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
crates/bsk-protocol/schema/trace_step.json
+40
View File
@@ -83,6 +83,46 @@
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
"required": [
"id",
"op",
"page",
"target"
],
"properties": {
"effect": {
"anyOf": [
{
"$ref": "#/definitions/StepEffect"
},
{
"type": "null"
}
]
},
"id": {
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"op": {
"type": "string",
"enum": [
"hover"
]
},
"page": {
"description": "Reference into `pages[]`.",
"type": "string"
},
"target": {
"$ref": "#/definitions/TargetDescriptor"
}
}
},
{
"description": "Fields shared by every step variant (flattened in JSON).",
"type": "object",
crates/bsk-protocol/src/bin/dump-schema.rs
+2
View File
@@ -77,6 +77,8 @@ fn main() {
dump!(ClickParams, "tool_click_params");
dump!(ClickResult, "tool_click_result");
dump!(HoverParams, "tool_hover_params");
dump!(HoverResult, "tool_hover_result");
dump!(FillParams, "tool_fill_params");
dump!(FillResult, "tool_fill_result");
dump!(PressParams, "tool_press_params");
crates/bsk-protocol/src/method.rs
+6 -1
View File
@@ -70,6 +70,8 @@ pub enum Method {
ToolReload,
#[serde(rename = "tool.click")]
ToolClick,
#[serde(rename = "tool.hover")]
ToolHover,
#[serde(rename = "tool.fill")]
ToolFill,
#[serde(rename = "tool.press")]
@@ -161,7 +163,7 @@ impl Method {
// Transient input — no committed browser action, but still page
// input. It must be stopped by pending user interrupts.
Method::ToolObserve => MethodEffect::TransientInput,
Method::ToolHover | Method::ToolObserve => MethodEffect::TransientInput,
// Passive reads — transparent.
// `record_stop` / `record_await` observe / finish a recording
@@ -259,6 +261,7 @@ mod tests {
fn is_mutating_classifies_read_only_tools_as_non_mutating() {
assert!(!Method::ToolTabList.is_mutating());
assert!(!Method::ToolSnapshot.is_mutating());
assert!(!Method::ToolHover.is_mutating());
assert!(!Method::ToolObserve.is_mutating());
assert!(!Method::ToolGetHtml.is_mutating());
assert!(!Method::ToolScreenshot.is_mutating());
@@ -321,6 +324,7 @@ mod tests {
#[test]
fn effect_classifies_observe_as_transient_input() {
assert_eq!(Method::ToolSnapshot.effect(), MethodEffect::PassiveRead);
assert_eq!(Method::ToolHover.effect(), MethodEffect::TransientInput);
assert_eq!(Method::ToolObserve.effect(), MethodEffect::TransientInput);
assert_eq!(Method::ToolClick.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::Cancel.effect(), MethodEffect::ControlPlane);
@@ -329,6 +333,7 @@ mod tests {
#[test]
fn interrupt_gate_includes_transient_input() {
assert!(!Method::ToolSnapshot.requires_interrupt_gate());
assert!(Method::ToolHover.requires_interrupt_gate());
assert!(Method::ToolObserve.requires_interrupt_gate());
assert!(Method::ToolClick.requires_interrupt_gate());
assert!(!Method::Cancel.requires_interrupt_gate());
crates/bsk-protocol/src/tools/interaction.rs
+64
View File
@@ -85,6 +85,52 @@ pub struct ClickResult {
pub dialogs: Vec<JavaScriptDialogInfo>,
}
// ---------------------------------------------------------------------------
// hover
// ---------------------------------------------------------------------------
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct HoverParams {
pub session_id: String,
/// Optional `@e<N>` ref allocated by the last observation.
/// Mutually exclusive with `selector`.
#[serde(
rename = "ref",
alias = "ref_",
default,
skip_serializing_if = "Option::is_none"
)]
pub ref_: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selector: Option<String>,
/// Target tab. Defaults to the Agent Window's active tab.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tab_id: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub modifiers: Option<Vec<KeyModifier>>,
/// Wait after the mouse move so hover-triggered UI can settle.
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 0))]
pub settle_ms: Option<u32>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 1))]
pub timeout_ms: Option<u32>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct HoverResult {
pub tab_id: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_ref: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_selector: Option<String>,
/// Viewport-relative hover coordinates (CSS pixels).
pub x: f64,
pub y: f64,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dialogs: Vec<JavaScriptDialogInfo>,
}
// ---------------------------------------------------------------------------
// fill
// ---------------------------------------------------------------------------
@@ -261,6 +307,24 @@ mod tests {
assert_eq!(p.ref_.as_deref(), Some("e1"));
}
#[test]
fn hover_params_serialise_ref_field_name() {
let p = HoverParams {
session_id: "abcd".into(),
ref_: Some("@e3".into()),
selector: None,
tab_id: Some(42),
modifiers: Some(vec![KeyModifier::Shift]),
settle_ms: Some(200),
timeout_ms: Some(5_000),
};
let v = serde_json::to_value(&p).unwrap();
assert_eq!(v.get("ref").and_then(|v| v.as_str()), Some("@e3"));
assert!(v.get("ref_").is_none());
let round: HoverParams = serde_json::from_value(v).unwrap();
assert_eq!(round, p);
}
#[test]
fn modifiers_render_as_lowercase_strings() {
let v = serde_json::to_value(KeyModifier::Ctrl).unwrap();
crates/bsk-protocol/src/tools/record.rs
+5
View File
@@ -94,6 +94,11 @@ pub enum Step {
common: StepCommon,
target: TargetDescriptor,
},
Hover {
#[serde(flatten)]
common: StepCommon,
target: TargetDescriptor,
},
Fill {
#[serde(flatten)]
common: StepCommon,