mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(vom): canvas position calculation and projection
This commit is contained in:
1 parent
aeece8d7e4
commit
000b374d0c
30 files changed
+1375
-20
No files matched your search
@@ -76,12 +76,24 @@ is cheaper and more precise.
|
||||
When `bsk observe` renders
|
||||
`@eN surface ... [visual-only; requires=image-understanding; ...]`, the ref identifies rendered
|
||||
canvas content that is not represented by the text observation, not an interactable DOM control.
|
||||
Keep using any reliable DOM/AX text and controls that appear alongside it. If you can actually
|
||||
inspect image output, use `bsk screenshot --ref @eN --session <id>` to obtain the visible crop. If
|
||||
you lack multimodal or image-reading capability, do not take a screenshot and pretend to know its
|
||||
contents, and never guess coordinates; tell the user that they need to switch to a model with
|
||||
image-understanding capability. Do not pass a visual surface ref to click, fill, hover, or select;
|
||||
those commands intentionally reject screenshot-only refs.
|
||||
Keep using any reliable DOM/AX text and controls that appear alongside it. If you lack multimodal or
|
||||
image-reading capability, do not take a screenshot and pretend to know its contents, and never guess
|
||||
coordinates; tell the user that they need to switch to a model with image-understanding capability.
|
||||
|
||||
If you can actually inspect image output, use `bsk screenshot --ref @eN --session <id>` to obtain
|
||||
the visible crop and its short-lived, single-use Surface capture id. Only when the task requires a
|
||||
point action and the image provides a clear target, bind that exact screenshot to the click command:
|
||||
|
||||
```bash
|
||||
bsk click @eN --capture <capture-id> --image-x <px> --image-y <px> --session <id>
|
||||
```
|
||||
|
||||
Coordinates are pixels in the returned capture image, not viewport CSS coordinates. Re-observing,
|
||||
navigating, scrolling, resizing, zooming, changing Frame projection, expiry, or reusing the capture
|
||||
makes the action fail; take a new Surface screenshot instead of adjusting or retrying coordinates.
|
||||
A point click does not reveal Canvas semantics and must not be followed by an assumed fill/press
|
||||
workflow. Without all three capture arguments, visual Surface refs remain screenshot-only and
|
||||
click/fill/hover/select reject them.
|
||||
|
||||
Escalate page reading only as needed:
|
||||
|
||||
@@ -135,6 +147,7 @@ Required flags that are easy to get wrong:
|
||||
bsk fill <ref> --value <text> bsk select <ref> --value <option-value>
|
||||
bsk screenshot --out <path> bsk emulate --device <preset-id>
|
||||
bsk upload <ref> --file <path> bsk download <ref> --out <path>
|
||||
bsk click <surface-ref> --capture <id> --image-x <px> --image-y <px>
|
||||
```
|
||||
|
||||
`select` matches an option's `value` attribute, not its visible label. Device preset ids are
|
||||
|
||||
@@ -136,6 +136,18 @@ pub struct ClickArgs {
|
||||
|
||||
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
|
||||
pub timeout: u32,
|
||||
|
||||
/// Capture id returned by `bsk screenshot --ref <surface>`.
|
||||
#[arg(long)]
|
||||
pub capture: Option<String>,
|
||||
|
||||
/// X coordinate in capture-image pixels. Requires `--capture` and `--image-y`.
|
||||
#[arg(long = "image-x", allow_hyphen_values = true)]
|
||||
pub image_x: Option<f64>,
|
||||
|
||||
/// Y coordinate in capture-image pixels. Requires `--capture` and `--image-x`.
|
||||
#[arg(long = "image-y", allow_hyphen_values = true)]
|
||||
pub image_y: Option<f64>,
|
||||
}
|
||||
|
||||
pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
|
||||
@@ -147,6 +159,14 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
|
||||
)?;
|
||||
let modifiers = parse_modifiers(&args.modifiers)
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!("--modifiers: {e}")))?;
|
||||
let point_arg_count = usize::from(args.capture.is_some())
|
||||
+ usize::from(args.image_x.is_some())
|
||||
+ usize::from(args.image_y.is_some());
|
||||
if point_arg_count != 0 && point_arg_count != 3 {
|
||||
return Err(CliError::Local(anyhow::anyhow!(
|
||||
"--capture, --image-x, and --image-y must be given together"
|
||||
)));
|
||||
}
|
||||
let params = ClickParams {
|
||||
session_id: args.session.clone(),
|
||||
ref_,
|
||||
@@ -160,6 +180,9 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
|
||||
Some(modifiers)
|
||||
},
|
||||
timeout_ms: Some(args.timeout),
|
||||
capture_id: args.capture.clone(),
|
||||
image_x: args.image_x,
|
||||
image_y: args.image_y,
|
||||
};
|
||||
let reply: ClickResult = call(
|
||||
info.sock_path,
|
||||
@@ -183,6 +206,14 @@ pub fn dispatch_click(args: ClickArgs, format: Format) -> Result<(), CliError> {
|
||||
"click ok tab={} target={target} at=({}, {})",
|
||||
reply.tab_id, reply.x, reply.y
|
||||
);
|
||||
if let Some(capture_id) = &reply.capture_id {
|
||||
println!(
|
||||
"capture={} image=({}, {}) coordinate_space=capture-image-pixel",
|
||||
capture_id,
|
||||
reply.image_x.unwrap_or_default(),
|
||||
reply.image_y.unwrap_or_default()
|
||||
);
|
||||
}
|
||||
print_dialog_summaries(&reply.dialogs);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -65,6 +65,11 @@ pub mod reason {
|
||||
pub const DOWNLOAD_CAPTURE_FAILED: &str = "download_capture_failed";
|
||||
pub const TRANSFER_OUTCOME_UNKNOWN: &str = "transfer_outcome_unknown";
|
||||
pub const TRANSFER_TIMEOUT: &str = "transfer_timeout";
|
||||
pub const SURFACE_CAPTURE_NOT_FOUND: &str = "surface_capture_not_found";
|
||||
pub const SURFACE_CAPTURE_EXPIRED: &str = "surface_capture_expired";
|
||||
pub const SURFACE_CAPTURE_CONSUMED: &str = "surface_capture_consumed";
|
||||
pub const SURFACE_CAPTURE_STALE: &str = "surface_capture_stale";
|
||||
pub const SURFACE_COORDINATE_INVALID: &str = "surface_coordinate_invalid";
|
||||
pub const SESSION_BUSY: &str = crate::rpc_reason::SESSION_BUSY;
|
||||
pub const RECORD_START_PAGE_UNREACHABLE: &str = "record_start_page_unreachable";
|
||||
}
|
||||
@@ -303,7 +308,7 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
||||
(ErrorCode::PermissionDenied, reason::REF_CAPABILITY_DENIED) => RenderInfo {
|
||||
summary: "snapshot ref does not support this operation",
|
||||
hint: Some(
|
||||
"visual surface refs are screenshot-only; inspect `bsk screenshot --ref <ref> --session <id>` with image-understanding capability",
|
||||
"visual Surface refs require image understanding; inspect `bsk screenshot --ref <ref> --session <id>`, then bind its capture id and image coordinates to `bsk click`",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
@@ -349,6 +354,41 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::NotFound, reason::SURFACE_CAPTURE_NOT_FOUND) => RenderInfo {
|
||||
summary: "surface capture transaction was not found",
|
||||
hint: Some(
|
||||
"take a fresh surface screenshot and use its capture id with coordinates from that image",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::PermissionDenied, reason::SURFACE_CAPTURE_EXPIRED) => RenderInfo {
|
||||
summary: "surface capture transaction has expired",
|
||||
hint: Some(
|
||||
"take a fresh surface screenshot, then click promptly with that capture id and image coordinates",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::PermissionDenied, reason::SURFACE_CAPTURE_CONSUMED) => RenderInfo {
|
||||
summary: "surface capture transaction was already used",
|
||||
hint: Some(
|
||||
"each capture id is single-use; take a fresh surface screenshot before another point action",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::PermissionDenied, reason::SURFACE_CAPTURE_STALE) => RenderInfo {
|
||||
summary: "page state changed since the surface screenshot",
|
||||
hint: Some(
|
||||
"take a fresh surface screenshot and choose coordinates from the new image before retrying",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::InvalidParams, reason::SURFACE_COORDINATE_INVALID) => RenderInfo {
|
||||
summary: "surface image coordinate is invalid",
|
||||
hint: Some(
|
||||
"use finite pixel coordinates inside the captured image; its top-left corner is (0, 0)",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(
|
||||
ErrorCode::CdpFailed | ErrorCode::Timeout | ErrorCode::InvalidParams,
|
||||
reason::RECORD_START_PAGE_UNREACHABLE,
|
||||
@@ -578,7 +618,7 @@ mod tests {
|
||||
let info = info_for_error(ErrorCode::PermissionDenied, Some(&data));
|
||||
assert_eq!(info.summary, "snapshot ref does not support this operation");
|
||||
assert!(
|
||||
info.hint.unwrap().contains("screenshot-only"),
|
||||
info.hint.unwrap().contains("capture id"),
|
||||
"expected visual-surface-specific hint"
|
||||
);
|
||||
assert_eq!(info.exit_code, 1);
|
||||
@@ -684,6 +724,25 @@ mod tests {
|
||||
assert!(info.summary.contains("timed out after browser dispatch"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn surface_capture_stale_requests_a_fresh_screenshot() {
|
||||
let data = serde_json::json!({ "reason": reason::SURFACE_CAPTURE_STALE });
|
||||
let info = info_for_error(ErrorCode::PermissionDenied, Some(&data));
|
||||
assert_eq!(
|
||||
info.summary,
|
||||
"page state changed since the surface screenshot"
|
||||
);
|
||||
assert!(info.hint.unwrap().contains("fresh surface screenshot"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn surface_coordinate_invalid_explains_image_pixel_bounds() {
|
||||
let data = serde_json::json!({ "reason": reason::SURFACE_COORDINATE_INVALID });
|
||||
let info = info_for_error(ErrorCode::InvalidParams, Some(&data));
|
||||
assert_eq!(info.summary, "surface image coordinate is invalid");
|
||||
assert!(info.hint.unwrap().contains("top-left corner is (0, 0)"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ref_not_found_overrides_not_found_copy() {
|
||||
let data = serde_json::json!({ "reason": reason::REF_NOT_FOUND });
|
||||
|
||||
@@ -59,7 +59,7 @@ fn run(sock: PathBuf, args: ScreenshotArgs, format: Format) -> Result<(), CliErr
|
||||
.map_err(CliError::Local)?;
|
||||
match format {
|
||||
Format::Json => {
|
||||
let json = serde_json::json!({
|
||||
let mut json = serde_json::json!({
|
||||
"tab_id": reply.tab_id,
|
||||
"width": reply.width,
|
||||
"height": reply.height,
|
||||
@@ -67,6 +67,10 @@ fn run(sock: PathBuf, args: ScreenshotArgs, format: Format) -> Result<(), CliErr
|
||||
"path": out_path.to_string_lossy(),
|
||||
"byte_size": bytes.len(),
|
||||
});
|
||||
if let Some(capture) = &reply.capture {
|
||||
json["capture"] = serde_json::to_value(capture)
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?;
|
||||
}
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&json)
|
||||
@@ -75,6 +79,12 @@ fn run(sock: PathBuf, args: ScreenshotArgs, format: Format) -> Result<(), CliErr
|
||||
}
|
||||
Format::Human => {
|
||||
println!("{}", out_path.display());
|
||||
if let Some(capture) = &reply.capture {
|
||||
println!(
|
||||
"capture={} surface={} coordinate_space={} expires_at={}",
|
||||
capture.id, capture.surface_ref, capture.coordinate_space, capture.expires_at
|
||||
);
|
||||
}
|
||||
print_dialog_summaries(&reply.dialogs);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -237,6 +237,29 @@ fn parses_click_count_alias() {
|
||||
assert_eq!(args.click_count, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_surface_capture_image_coordinates() {
|
||||
let cli = parse(&[
|
||||
"bsk",
|
||||
"click",
|
||||
"@e1",
|
||||
"--session",
|
||||
"s1",
|
||||
"--capture",
|
||||
"sc_test",
|
||||
"--image-x",
|
||||
"160.5",
|
||||
"--image-y",
|
||||
"-1",
|
||||
]);
|
||||
let Command::Click(args) = cli.command else {
|
||||
panic!("expected click command");
|
||||
};
|
||||
assert_eq!(args.capture.as_deref(), Some("sc_test"));
|
||||
assert_eq!(args.image_x, Some(160.5));
|
||||
assert_eq!(args.image_y, Some(-1.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_hover_with_settle() {
|
||||
let cli = parse(&[
|
||||
|
||||
@@ -238,6 +238,7 @@ async fn screenshot_returns_image_base64_with_dimensions() {
|
||||
height: 600,
|
||||
format: "png".into(),
|
||||
tab_id: 7,
|
||||
capture: None,
|
||||
dialogs: vec![],
|
||||
})
|
||||
.unwrap(),
|
||||
@@ -286,6 +287,7 @@ async fn screenshot_forwards_ref_to_extension() {
|
||||
height: 60,
|
||||
format: "png".into(),
|
||||
tab_id: 7,
|
||||
capture: None,
|
||||
dialogs: vec![],
|
||||
})
|
||||
.unwrap(),
|
||||
|
||||
@@ -354,6 +354,9 @@ async fn click_round_trips_ref_and_modifiers() {
|
||||
used_selector: None,
|
||||
x: 12.5,
|
||||
y: 34.0,
|
||||
capture_id: None,
|
||||
image_x: None,
|
||||
image_y: None,
|
||||
dialogs: vec![],
|
||||
})
|
||||
.unwrap(),
|
||||
@@ -373,6 +376,9 @@ async fn click_round_trips_ref_and_modifiers() {
|
||||
click_count: Some(1),
|
||||
modifiers: Some(vec![KeyModifier::Ctrl, KeyModifier::Shift]),
|
||||
timeout_ms: Some(5_000),
|
||||
capture_id: None,
|
||||
image_x: None,
|
||||
image_y: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -610,6 +616,9 @@ async fn m7_tools_propagate_extension_errors() {
|
||||
click_count: None,
|
||||
modifiers: None,
|
||||
timeout_ms: Some(5_000),
|
||||
capture_id: None,
|
||||
image_x: None,
|
||||
image_y: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
Reference in new issue
Block a user