figmar/BrowserSkill
1
0
Fork 0
mirror of https://github.com/Tencent/BrowserSkill.git synced 2026-09-10 13:01:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(vom): canvas position calculation and projection

Browse Source
This commit is contained in:
Ljy-0827
2026-08-31 14:22:34 +08:00
parent d660da87f3
commit 22ca308342
30 changed files with 1433 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/bsk-protocol/src/tools/interaction.rs
+18
View File
@@ -67,6 +67,15 @@ pub struct ClickParams {
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 1))]
pub timeout_ms: Option<u32>,
/// Short-lived capture id returned by a visual Surface screenshot.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub capture_id: Option<String>,
/// X coordinate in the capture image's pixel coordinate space.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub image_x: Option<f64>,
/// Y coordinate in the capture image's pixel coordinate space.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub image_y: Option<f64>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
@@ -80,6 +89,12 @@ pub struct ClickResult {
/// agents can correlate with a follow-up `tool.screenshot`.
pub x: f64,
pub y: f64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub capture_id: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub image_x: Option<f64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub image_y: Option<f64>,
/// Native JS dialogs observed and auto-handled during this call.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dialogs: Vec<JavaScriptDialogInfo>,
@@ -288,6 +303,9 @@ mod tests {
click_count: Some(1),
modifiers: Some(vec![KeyModifier::Ctrl]),
timeout_ms: Some(5_000),
capture_id: Some("sc_test".into()),
image_x: Some(160.0),
image_y: Some(90.0),
};
let v = serde_json::to_value(&p).unwrap();
assert_eq!(v.get("ref").and_then(|v| v.as_str()), Some("@e3"));
crates/bsk-protocol/src/tools/observation.rs
+16
View File
@@ -208,10 +208,20 @@ pub struct ScreenshotResult {
/// Always `"png"` in v0.1; reserved for future JPEG / WebP support.
pub format: String,
pub tab_id: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub capture: Option<SurfaceCaptureInfo>,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dialogs: Vec<JavaScriptDialogInfo>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct SurfaceCaptureInfo {
pub id: String,
pub surface_ref: String,
pub coordinate_space: String,
pub expires_at: u64,
}
#[cfg(test)]
mod tests {
use super::*;
@@ -293,6 +303,12 @@ mod tests {
height: 600,
format: "png".into(),
tab_id: 7,
capture: Some(SurfaceCaptureInfo {
id: "sc_test".into(),
surface_ref: "@e3".into(),
coordinate_space: "capture-image-pixel".into(),
expires_at: 1_700_000_030_000,
}),
dialogs: vec![],
};
let v = serde_json::to_value(&r).unwrap();