figmar/BrowserSkill
1
0
Fork 0
mirror of https://github.com/Tencent/BrowserSkill.git synced 2026-09-10 13:01:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(vom): canvas position calculation and projection

Browse Source
This commit is contained in:
Ljy-0827
2026-08-31 14:22:34 +08:00
parent d660da87f3
commit 22ca308342
30 changed files with 1433 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/bsk-cli/src/cli/interaction.rs
+31
View File
@@ -136,6 +136,18 @@ pub struct ClickArgs {
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
pub timeout: u32,
/// Capture id returned by `bsk screenshot --ref <surface>`.
#[arg(long)]
pub capture: Option<String>,
/// X coordinate in capture-image pixels. Requires `--capture` and `--image-y`.
#[arg(long = "image-x", allow_hyphen_values = true)]
pub image_x: Option<f64>,
/// Y coordinate in capture-image pixels. Requires `--capture` and `--image-x`.
#[arg(long = "image-y", allow_hyphen_values = true)]
pub image_y: Option<f64>,
}
pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
@@ -147,6 +159,14 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
)?;
let modifiers = parse_modifiers(&args.modifiers)
.map_err(|e| CliError::Local(anyhow::anyhow!("--modifiers: {e}")))?;
let point_arg_count = usize::from(args.capture.is_some())
+ usize::from(args.image_x.is_some())
+ usize::from(args.image_y.is_some());
if point_arg_count != 0 && point_arg_count != 3 {
return Err(CliError::Local(anyhow::anyhow!(
"--capture, --image-x, and --image-y must be given together"
)));
}
let params = ClickParams {
session_id: args.session.clone(),
ref_,
@@ -160,6 +180,9 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
Some(modifiers)
},
timeout_ms: Some(args.timeout),
capture_id: args.capture.clone(),
image_x: args.image_x,
image_y: args.image_y,
};
let reply: ClickResult = call(
info.sock_path,
@@ -183,6 +206,14 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
"click ok tab={} target={target} at=({}, {})",
reply.tab_id, reply.x, reply.y
);
if let Some(capture_id) = &reply.capture_id {
println!(
"capture={} image=({}, {}) coordinate_space=capture-image-pixel",
capture_id,
reply.image_x.unwrap_or_default(),
reply.image_y.unwrap_or_default()
);
}
print_dialog_summaries(&reply.dialogs);
}
}
crates/bsk-cli/src/cli/render_error.rs
+61 -2
View File
@@ -50,6 +50,11 @@ pub mod reason {
pub const TAB_NOT_ACTIVE: &str = "tab_not_active";
pub const BORROW_CONFLICT: &str = "borrow_conflict";
pub const SCREENSHOT_CAPTURE_FAILED: &str = "screenshot_capture_failed";
pub const SURFACE_CAPTURE_NOT_FOUND: &str = "surface_capture_not_found";
pub const SURFACE_CAPTURE_EXPIRED: &str = "surface_capture_expired";
pub const SURFACE_CAPTURE_CONSUMED: &str = "surface_capture_consumed";
pub const SURFACE_CAPTURE_STALE: &str = "surface_capture_stale";
pub const SURFACE_COORDINATE_INVALID: &str = "surface_coordinate_invalid";
pub const SESSION_BUSY: &str = crate::rpc_reason::SESSION_BUSY;
pub const RECORD_START_PAGE_UNREACHABLE: &str = "record_start_page_unreachable";
}
@@ -232,7 +237,7 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
(ErrorCode::PermissionDenied, reason::REF_CAPABILITY_DENIED) => RenderInfo {
summary: "snapshot ref does not support this operation",
hint: Some(
"visual surface refs are screenshot-only; inspect `bsk screenshot --ref <ref> --session <id>` with image-understanding capability",
"visual Surface refs require image understanding; inspect `bsk screenshot --ref <ref> --session <id>`, then bind its capture id and image coordinates to `bsk click`",
),
exit_code: base.exit_code,
},
@@ -278,6 +283,41 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
),
exit_code: base.exit_code,
},
(ErrorCode::NotFound, reason::SURFACE_CAPTURE_NOT_FOUND) => RenderInfo {
summary: "surface capture transaction was not found",
hint: Some(
"take a fresh surface screenshot and use its capture id with coordinates from that image",
),
exit_code: base.exit_code,
},
(ErrorCode::PermissionDenied, reason::SURFACE_CAPTURE_EXPIRED) => RenderInfo {
summary: "surface capture transaction has expired",
hint: Some(
"take a fresh surface screenshot, then click promptly with that capture id and image coordinates",
),
exit_code: base.exit_code,
},
(ErrorCode::PermissionDenied, reason::SURFACE_CAPTURE_CONSUMED) => RenderInfo {
summary: "surface capture transaction was already used",
hint: Some(
"each capture id is single-use; take a fresh surface screenshot before another point action",
),
exit_code: base.exit_code,
},
(ErrorCode::PermissionDenied, reason::SURFACE_CAPTURE_STALE) => RenderInfo {
summary: "page state changed since the surface screenshot",
hint: Some(
"take a fresh surface screenshot and choose coordinates from the new image before retrying",
),
exit_code: base.exit_code,
},
(ErrorCode::InvalidParams, reason::SURFACE_COORDINATE_INVALID) => RenderInfo {
summary: "surface image coordinate is invalid",
hint: Some(
"use finite pixel coordinates inside the captured image; its top-left corner is (0, 0)",
),
exit_code: base.exit_code,
},
(
ErrorCode::CdpFailed | ErrorCode::Timeout | ErrorCode::InvalidParams,
reason::RECORD_START_PAGE_UNREACHABLE,
@@ -391,7 +431,7 @@ mod tests {
let info = info_for_error(ErrorCode::PermissionDenied, Some(&data));
assert_eq!(info.summary, "snapshot ref does not support this operation");
assert!(
info.hint.unwrap().contains("screenshot-only"),
info.hint.unwrap().contains("capture id"),
"expected visual-surface-specific hint"
);
assert_eq!(info.exit_code, 1);
@@ -439,6 +479,25 @@ mod tests {
assert_eq!(info.exit_code, 3);
}
#[test]
fn surface_capture_stale_requests_a_fresh_screenshot() {
let data = serde_json::json!({ "reason": reason::SURFACE_CAPTURE_STALE });
let info = info_for_error(ErrorCode::PermissionDenied, Some(&data));
assert_eq!(
info.summary,
"page state changed since the surface screenshot"
);
assert!(info.hint.unwrap().contains("fresh surface screenshot"));
}
#[test]
fn surface_coordinate_invalid_explains_image_pixel_bounds() {
let data = serde_json::json!({ "reason": reason::SURFACE_COORDINATE_INVALID });
let info = info_for_error(ErrorCode::InvalidParams, Some(&data));
assert_eq!(info.summary, "surface image coordinate is invalid");
assert!(info.hint.unwrap().contains("top-left corner is (0, 0)"));
}
#[test]
fn ref_not_found_overrides_not_found_copy() {
let data = serde_json::json!({ "reason": reason::REF_NOT_FOUND });
crates/bsk-cli/src/cli/screenshot.rs
+11 -1
View File
@@ -59,7 +59,7 @@ fn run(sock: PathBuf, args: ScreenshotArgs, format: Format) -> Result<(), CliErr
.map_err(CliError::Local)?;
match format {
Format::Json => {
let json = serde_json::json!({
let mut json = serde_json::json!({
"tab_id": reply.tab_id,
"width": reply.width,
"height": reply.height,
@@ -67,6 +67,10 @@ fn run(sock: PathBuf, args: ScreenshotArgs, format: Format) -> Result<(), CliErr
"path": out_path.to_string_lossy(),
"byte_size": bytes.len(),
});
if let Some(capture) = &reply.capture {
json["capture"] = serde_json::to_value(capture)
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?;
}
println!(
"{}",
serde_json::to_string_pretty(&json)
@@ -75,6 +79,12 @@ fn run(sock: PathBuf, args: ScreenshotArgs, format: Format) -> Result<(), CliErr
}
Format::Human => {
println!("{}", out_path.display());
if let Some(capture) = &reply.capture {
println!(
"capture={} surface={} coordinate_space={} expires_at={}",
capture.id, capture.surface_ref, capture.coordinate_space, capture.expires_at
);
}
print_dialog_summaries(&reply.dialogs);
}
}