figmar/BrowserSkill
1
0
Fork 0
mirror of https://github.com/Tencent/BrowserSkill.git synced 2026-09-10 13:01:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(vom): add type primitive

Browse Source
This commit is contained in:
Ljy-0827
2026-08-31 17:48:45 +08:00
parent e069abaf2e
commit a1904f8141
23 changed files with 1776 additions and 156 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/bsk-protocol/schema/tool_snapshot_result.json
+1 -1
View File
@@ -26,7 +26,7 @@
"format": "int64"
},
"text": {
"description": "Indented aria-snapshot text. Refs are rendered as `@e<N>` so the agent can copy them into subsequent `tool.click` / `tool.fill` selectors.",
"description": "Indented aria-snapshot text. Refs are rendered as `@e<N>` so the agent can copy them into subsequent `tool.click` / `tool.fill` / `tool.type` targets.",
"type": "string"
},
"truncated": {
crates/bsk-protocol/schema/tool_type_params.json
+50
View File
@@ -0,0 +1,50 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "TypeParams",
"type": "object",
"required": [
"session_id",
"text"
],
"properties": {
"focused": {
"description": "Explicitly dispatch to the unique focused text-entry target instead of focusing a ref or selector.",
"type": "boolean"
},
"ref": {
"description": "Target to focus before dispatching text. Exactly one targeting mode must be selected: `ref` / `selector`, or `focused=true`.",
"type": [
"string",
"null"
]
},
"selector": {
"type": [
"string",
"null"
]
},
"session_id": {
"type": "string"
},
"tab_id": {
"type": [
"integer",
"null"
],
"format": "int64"
},
"text": {
"description": "Text to dispatch to the live editing session. Unlike `fill`, `type` does not require the focused DOM editable to retain this text.",
"type": "string"
},
"timeout_ms": {
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 1.0
}
}
}
crates/bsk-protocol/schema/tool_type_result.json
+124
View File
@@ -0,0 +1,124 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "TypeResult",
"type": "object",
"required": [
"tab_id",
"text_length",
"verification"
],
"properties": {
"dialogs": {
"type": "array",
"items": {
"$ref": "#/definitions/JavaScriptDialogInfo"
}
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"text_length": {
"description": "UTF-16 code-unit length of text accepted for dispatch.",
"type": "integer",
"format": "uint32",
"minimum": 0.0
},
"used_ref": {
"type": [
"string",
"null"
]
},
"used_selector": {
"type": [
"string",
"null"
]
},
"verification": {
"$ref": "#/definitions/TypeVerification"
}
},
"definitions": {
"JavaScriptDialogHandledAction": {
"description": "How the extension resolved the dialog so CDP could continue.",
"type": "string",
"enum": [
"accepted",
"dismissed"
]
},
"JavaScriptDialogInfo": {
"description": "One observed + handled JavaScript dialog during a tool call.",
"type": "object",
"required": [
"handled",
"message",
"sequence",
"tab_id",
"type"
],
"properties": {
"default_prompt": {
"type": [
"string",
"null"
]
},
"handled": {
"$ref": "#/definitions/JavaScriptDialogHandledAction"
},
"has_browser_handler": {
"type": [
"boolean",
"null"
]
},
"message": {
"type": "string"
},
"sequence": {
"description": "Monotonic per-tab sequence for ordering within a session.",
"type": "integer",
"format": "uint64",
"minimum": 0.0
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"type": {
"$ref": "#/definitions/JavaScriptDialogType"
},
"url": {
"type": [
"string",
"null"
]
}
}
},
"JavaScriptDialogType": {
"description": "Native JS dialog kind reported by CDP.",
"type": "string",
"enum": [
"alert",
"confirm",
"prompt",
"beforeunload"
]
},
"TypeVerification": {
"oneOf": [
{
"description": "Chromium accepted the complete keyboard/IME dispatch transaction. The caller must observe the application output to verify business state.",
"type": "string",
"enum": [
"dispatched"
]
}
]
}
}
}
crates/bsk-protocol/src/bin/dump-schema.rs
+2
View File
@@ -81,6 +81,8 @@ fn main() {
dump!(HoverResult, "tool_hover_result");
dump!(FillParams, "tool_fill_params");
dump!(FillResult, "tool_fill_result");
dump!(TypeParams, "tool_type_params");
dump!(TypeResult, "tool_type_result");
dump!(PressParams, "tool_press_params");
dump!(PressResult, "tool_press_result");
dump!(SelectParams, "tool_select_params");
crates/bsk-protocol/src/method.rs
+11
View File
@@ -74,6 +74,8 @@ pub enum Method {
ToolHover,
#[serde(rename = "tool.fill")]
ToolFill,
#[serde(rename = "tool.type")]
ToolType,
#[serde(rename = "tool.press")]
ToolPress,
#[serde(rename = "tool.select")]
@@ -154,6 +156,7 @@ impl Method {
| Method::ToolReload
| Method::ToolClick
| Method::ToolFill
| Method::ToolType
| Method::ToolPress
| Method::ToolSelect
| Method::ToolEvaluate
@@ -246,6 +249,13 @@ mod tests {
assert_eq!(serde_json::to_value(method).unwrap(), json!("tool.emulate"));
}
#[test]
fn type_method_round_trips() {
let method: Method = serde_json::from_value(json!("tool.type")).unwrap();
assert_eq!(method, Method::ToolType);
assert_eq!(serde_json::to_value(method).unwrap(), json!("tool.type"));
}
#[test]
fn cancel_params_and_result_round_trip() {
let params: CancelParams = serde_json::from_value(json!({ "rpc_id": "wait-1" })).unwrap();
@@ -284,6 +294,7 @@ mod tests {
assert!(Method::ToolReload.is_mutating());
assert!(Method::ToolClick.is_mutating());
assert!(Method::ToolFill.is_mutating());
assert!(Method::ToolType.is_mutating());
assert!(Method::ToolPress.is_mutating());
assert!(Method::ToolSelect.is_mutating());
assert!(Method::ToolEvaluate.is_mutating());
crates/bsk-protocol/src/tools/interaction.rs
+100 -2
View File
@@ -1,5 +1,5 @@
//! DOM interaction tools (`tool.click`, `tool.fill`, `tool.press`,
//! `tool.select`).
//! DOM interaction tools (`tool.click`, `tool.fill`, `tool.type`,
//! `tool.press`, `tool.select`).
//!
//! Element-targeted tools accept either a snapshot `ref` (`@e<N>` form,
//! normalised against the session's RefStore) **or** a CSS selector
@@ -188,6 +188,60 @@ pub struct FillResult {
pub dialogs: Vec<JavaScriptDialogInfo>,
}
// ---------------------------------------------------------------------------
// type
// ---------------------------------------------------------------------------
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct TypeParams {
pub session_id: String,
/// Text to dispatch to the live editing session. Unlike `fill`, `type`
/// does not require the focused DOM editable to retain this text.
pub text: String,
/// Target to focus before dispatching text. Exactly one targeting mode
/// must be selected: `ref` / `selector`, or `focused=true`.
#[serde(
rename = "ref",
alias = "ref_",
default,
skip_serializing_if = "Option::is_none"
)]
pub ref_: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selector: Option<String>,
/// Explicitly dispatch to the unique focused text-entry target instead
/// of focusing a ref or selector.
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub focused: bool,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tab_id: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 1))]
pub timeout_ms: Option<u32>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
pub enum TypeVerification {
/// Chromium accepted the complete keyboard/IME dispatch transaction.
/// The caller must observe the application output to verify business state.
Dispatched,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct TypeResult {
pub tab_id: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_ref: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_selector: Option<String>,
/// UTF-16 code-unit length of text accepted for dispatch.
pub text_length: u32,
pub verification: TypeVerification,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dialogs: Vec<JavaScriptDialogInfo>,
}
// ---------------------------------------------------------------------------
// press
// ---------------------------------------------------------------------------
@@ -351,6 +405,50 @@ mod tests {
assert_eq!(v, json!("meta"));
}
#[test]
fn type_payloads_distinguish_dispatch_from_value_verification() {
let params = TypeParams {
session_id: "abcd".into(),
text: "测试".into(),
ref_: Some("@e3".into()),
selector: None,
focused: false,
tab_id: Some(4),
timeout_ms: Some(5_000),
};
let encoded = serde_json::to_value(&params).unwrap();
assert_eq!(encoded["ref"], json!("@e3"));
assert_eq!(encoded["text"], json!("测试"));
assert!(encoded.get("focused").is_none());
let focused = TypeParams {
session_id: "abcd".into(),
text: "hello".into(),
ref_: None,
selector: None,
focused: true,
tab_id: None,
timeout_ms: None,
};
assert_eq!(
serde_json::to_value(focused).unwrap()["focused"],
json!(true)
);
let result = TypeResult {
tab_id: 4,
used_ref: Some("e3".into()),
used_selector: None,
text_length: 2,
verification: TypeVerification::Dispatched,
dialogs: vec![],
};
assert_eq!(
serde_json::to_value(result).unwrap()["verification"],
json!("dispatched")
);
}
#[test]
fn press_result_round_trips() {
let r = PressResult {
crates/bsk-protocol/src/tools/observation.rs
+2 -2
View File
@@ -33,8 +33,8 @@ pub struct SnapshotParams {
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct SnapshotResult {
/// Indented aria-snapshot text. Refs are rendered as `@e<N>` so the
/// agent can copy them into subsequent `tool.click` / `tool.fill`
/// selectors.
/// agent can copy them into subsequent `tool.click` / `tool.fill` /
/// `tool.type` targets.
pub text: String,
/// Number of `@e<N>` refs registered for this session by this
/// snapshot. Equivalent to the size of the new ref-store map.