mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(interaction): add scroll-to-element primitive
This commit is contained in:
@@ -0,0 +1,145 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import type { CdpRunner } from "@/tools/shared";
|
||||
import { handleScrollTo } from "../scroll";
|
||||
|
||||
function fakeAgentWindow(ids: number[]) {
|
||||
let i = 0;
|
||||
return {
|
||||
create: vi.fn(async () => {
|
||||
const id = ids[i++];
|
||||
if (id === undefined) throw new Error("ran out of fake ids");
|
||||
return id;
|
||||
}),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
};
|
||||
}
|
||||
|
||||
function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
||||
const sent: Array<{ tabId: number; method: string; params?: object }> = [];
|
||||
const sendImpl = async (tabId: number, method: string, params?: object) => {
|
||||
sent.push({ tabId, method, params });
|
||||
const handler = handlers[method];
|
||||
if (!handler && method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
if (!handler) throw new Error(`unexpected CDP call ${method}`);
|
||||
return handler(params);
|
||||
};
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(sendImpl) as unknown as CdpRunner["send"],
|
||||
trackSessionTab: vi.fn(),
|
||||
};
|
||||
const tabsApi = {
|
||||
get: vi.fn(
|
||||
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
};
|
||||
return { cdp, tabsApi, sent };
|
||||
}
|
||||
|
||||
describe("handleScrollTo", () => {
|
||||
it("scrolls a ref into view and returns its visible top-viewport bounds", async () => {
|
||||
const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await manager.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, { tabId: 4 });
|
||||
const fake = makeFakeCdp({
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
|
||||
});
|
||||
|
||||
const result = await handleScrollTo(
|
||||
manager,
|
||||
{ session_id: "aa11", ref: "@e3" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
if ("code" in result) throw new Error(`unexpected error: ${JSON.stringify(result)}`);
|
||||
expect(result).toMatchObject({
|
||||
tab_id: 4,
|
||||
used_ref: "e3",
|
||||
x: 10,
|
||||
y: 20,
|
||||
width: 100,
|
||||
height: 40,
|
||||
});
|
||||
expect(fake.sent.map((call) => call.method)).toEqual([
|
||||
"DOM.scrollIntoViewIfNeeded",
|
||||
"DOM.getContentQuads",
|
||||
"Page.getLayoutMetrics",
|
||||
]);
|
||||
});
|
||||
|
||||
it("scrolls OOPIF refs through their parent frame and CDP session", async () => {
|
||||
const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await manager.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
const fake = makeFakeCdp({
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.getBoxModel": () => ({
|
||||
model: { content: [204, 306, 604, 306, 604, 506, 204, 506] },
|
||||
}),
|
||||
});
|
||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child-frame",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 99,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
}));
|
||||
const targetCalls: Array<{ sessionId?: string; method: string }> = [];
|
||||
fake.cdp.sendToTarget = vi.fn(async (target, method) => {
|
||||
targetCalls.push({ sessionId: target.sessionId, method });
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
throw new Error(`unexpected child CDP call ${method}`);
|
||||
}) as CdpRunner["sendToTarget"];
|
||||
|
||||
const result = await handleScrollTo(
|
||||
manager,
|
||||
{ session_id: "aa11", ref: "@e3" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
if ("code" in result) throw new Error(`unexpected error: ${JSON.stringify(result)}`);
|
||||
expect(result).toMatchObject({ x: 224, y: 346, width: 200, height: 80 });
|
||||
expect(targetCalls).toEqual([
|
||||
{ sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" },
|
||||
{ sessionId: "child-session", method: "DOM.getContentQuads" },
|
||||
{ sessionId: "child-session", method: "Page.getLayoutMetrics" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not issue CDP calls after an early cancellation", async () => {
|
||||
const manager = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await manager.start("aa11");
|
||||
const abort = new AbortController();
|
||||
abort.abort();
|
||||
const fake = makeFakeCdp({});
|
||||
|
||||
const result = await handleScrollTo(
|
||||
manager,
|
||||
{ session_id: "aa11", selector: "#target" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi, signal: abort.signal },
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({ code: "cancelled" });
|
||||
expect(fake.cdp.send).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -26,6 +26,7 @@ import type {
|
||||
ResponseFrame,
|
||||
RpcError,
|
||||
ScreenshotParams,
|
||||
ScrollToParams,
|
||||
SelectParams,
|
||||
SnapshotParams,
|
||||
WaitForNavigationParams,
|
||||
@@ -52,6 +53,7 @@ import {
|
||||
handleSnapshot,
|
||||
} from "./observation";
|
||||
import { handleRecordAwait, handleRecordStart, handleRecordStop } from "./record";
|
||||
import { handleScrollTo } from "./scroll";
|
||||
import {
|
||||
handleSessionStart,
|
||||
handleSessionStop,
|
||||
@@ -471,6 +473,17 @@ export class ToolDispatcher {
|
||||
);
|
||||
return this.rememberHover((req.params as HoverParams).session_id, result);
|
||||
}
|
||||
case "tool.scroll_to":
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as ScrollToParams,
|
||||
() =>
|
||||
handleScrollTo(
|
||||
this.sessions,
|
||||
req.params as ScrollToParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
signal,
|
||||
);
|
||||
case "tool.fill":
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as FillParams,
|
||||
@@ -718,6 +731,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
||||
case "tool.reload":
|
||||
case "tool.click":
|
||||
case "tool.hover":
|
||||
case "tool.scroll_to":
|
||||
case "tool.fill":
|
||||
case "tool.press":
|
||||
case "tool.select":
|
||||
|
||||
@@ -128,7 +128,7 @@ async function wait(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
* `RpcError` if the caller supplied neither (or both), or if neither
|
||||
* lookup matched.
|
||||
*/
|
||||
async function resolveBackendNode(
|
||||
export async function resolveBackendNode(
|
||||
cdp: CdpRunner,
|
||||
ctx: SessionContext,
|
||||
target: { tabId: number },
|
||||
|
||||
@@ -0,0 +1,69 @@
|
||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import type { RpcError, ScrollToParams, ScrollToResult } from "@/transport/types";
|
||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||
import { resolveNodeGeometry } from "./frame-geometry";
|
||||
import { resolveBackendNode } from "./interaction";
|
||||
import {
|
||||
type CdpRunner,
|
||||
type ChromeTabsApi,
|
||||
chromeTabsApi,
|
||||
enforceAgentWindow,
|
||||
isRpcError,
|
||||
lookupSession,
|
||||
resolveTargetTab,
|
||||
} from "./shared";
|
||||
|
||||
export interface ScrollToDeps {
|
||||
cdp: CdpRunner;
|
||||
tabsApi: ChromeTabsApi;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null;
|
||||
function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } {
|
||||
if (!defaultDeps) defaultDeps = { cdp: new ChromiumCdp(), tabsApi: chromeTabsApi };
|
||||
return defaultDeps;
|
||||
}
|
||||
|
||||
export async function handleScrollTo(
|
||||
manager: SessionManager,
|
||||
params: ScrollToParams,
|
||||
deps: ScrollToDeps = getDefaultDeps(),
|
||||
): Promise<ScrollToResult | RpcError> {
|
||||
const ctxOrErr = lookupSession(manager, params, "scroll-to");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
if (deps.signal?.aborted) return { code: "cancelled", message: "scroll-to aborted" };
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
const denied = enforceAgentWindow(ctx, target, "scroll-to");
|
||||
if (denied) return denied;
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "scroll-to");
|
||||
if (isRpcError(node)) return node;
|
||||
if (deps.signal?.aborted) return { code: "cancelled", message: "scroll-to aborted" };
|
||||
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const geometry = await resolveNodeGeometry(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
{
|
||||
target: node.cdpTarget,
|
||||
backendNodeId: node.backendNodeId,
|
||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: node.usedRef,
|
||||
used_selector: node.usedSelector,
|
||||
x: geometry.topBounds.x,
|
||||
y: geometry.topBounds.y,
|
||||
width: geometry.topBounds.width,
|
||||
height: geometry.topBounds.height,
|
||||
});
|
||||
}
|
||||
@@ -444,6 +444,25 @@ export interface HoverResult {
|
||||
dialogs?: JavaScriptDialogInfo[];
|
||||
}
|
||||
|
||||
export interface ScrollToParams {
|
||||
session_id: string;
|
||||
ref?: string;
|
||||
selector?: string;
|
||||
tab_id?: number;
|
||||
timeout_ms?: number;
|
||||
}
|
||||
|
||||
export interface ScrollToResult {
|
||||
tab_id: number;
|
||||
used_ref?: string;
|
||||
used_selector?: string;
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
dialogs?: JavaScriptDialogInfo[];
|
||||
}
|
||||
|
||||
export interface FillParams {
|
||||
session_id: string;
|
||||
value: string;
|
||||
|
||||
@@ -202,6 +202,7 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
|
||||
|---------|---------|
|
||||
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||
| `bsk scroll-to <ref-or-selector>` | Scroll an element and its frame owners into the visible viewport |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
||||
|
||||
@@ -25,6 +25,7 @@ pub mod record;
|
||||
pub mod record_state;
|
||||
pub mod render_error;
|
||||
pub mod screenshot;
|
||||
pub mod scroll;
|
||||
pub mod session;
|
||||
pub mod snapshot;
|
||||
pub mod status;
|
||||
@@ -48,6 +49,7 @@ use crate::cli::network::NetworkArgs;
|
||||
use crate::cli::observe::ObserveArgs;
|
||||
use crate::cli::record::RecordCmd;
|
||||
use crate::cli::screenshot::ScreenshotArgs;
|
||||
use crate::cli::scroll::ScrollToArgs;
|
||||
use crate::cli::session::SessionCmd;
|
||||
use crate::cli::snapshot::SnapshotArgs;
|
||||
use crate::cli::tab::TabCmd;
|
||||
@@ -167,6 +169,10 @@ pub enum Command {
|
||||
/// Hover a snapshot ref or CSS selector.
|
||||
Hover(HoverArgs),
|
||||
|
||||
/// Scroll a snapshot ref or CSS selector into the visible viewport.
|
||||
#[command(name = "scroll-to")]
|
||||
ScrollTo(ScrollToArgs),
|
||||
|
||||
/// Fill an input / textarea / contenteditable.
|
||||
Fill(FillArgs),
|
||||
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
//! `bsk scroll-to` — bring a target element into the visible viewport.
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::Context;
|
||||
use bsk_protocol::Method;
|
||||
use bsk_protocol::tools::{ScrollToParams, ScrollToResult};
|
||||
use clap::Args;
|
||||
|
||||
use crate::cli::ensure_daemon::ensure_daemon;
|
||||
use crate::cli::error::{CliError, Format};
|
||||
use crate::cli::interaction::looks_like_ref;
|
||||
use crate::cli::navigate::parse_timeout_ms;
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct ScrollToArgs {
|
||||
/// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used.
|
||||
pub target: Option<String>,
|
||||
|
||||
#[arg(long = "ref")]
|
||||
pub ref_: Option<String>,
|
||||
|
||||
#[arg(long = "selector")]
|
||||
pub selector: Option<String>,
|
||||
|
||||
#[arg(long)]
|
||||
pub session: String,
|
||||
|
||||
#[arg(long = "tab-id")]
|
||||
pub tab_id: Option<i64>,
|
||||
|
||||
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
|
||||
pub timeout: u32,
|
||||
}
|
||||
|
||||
pub fn dispatch(args: ScrollToArgs, format: Format) -> Result<(), CliError> {
|
||||
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||
let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?;
|
||||
let params = ScrollToParams {
|
||||
session_id: args.session,
|
||||
ref_,
|
||||
selector,
|
||||
tab_id: args.tab_id,
|
||||
timeout_ms: Some(args.timeout),
|
||||
};
|
||||
let reply: ScrollToResult = crate::cli::business_rpc::call(
|
||||
info.sock_path,
|
||||
"scroll-to-1",
|
||||
Method::ToolScrollTo,
|
||||
Some(params),
|
||||
ipc_timeout(args.timeout),
|
||||
)?;
|
||||
match format {
|
||||
Format::Json => println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&reply)
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
|
||||
),
|
||||
Format::Human => {
|
||||
let target =
|
||||
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref());
|
||||
println!(
|
||||
"scroll-to ok tab={} target={target} bounds=({}, {}, {}, {})",
|
||||
reply.tab_id, reply.x, reply.y, reply.width, reply.height
|
||||
);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn split_target(
|
||||
positional: Option<String>,
|
||||
explicit_ref: Option<String>,
|
||||
explicit_selector: Option<String>,
|
||||
) -> Result<(Option<String>, Option<String>), CliError> {
|
||||
match (positional, explicit_ref, explicit_selector) {
|
||||
(None, None, None) => Err(CliError::Local(anyhow::anyhow!(
|
||||
"missing target: pass <ref-or-selector>, --ref @eN, or --selector <css>"
|
||||
))),
|
||||
(None, Some(r), None) => Ok((Some(r), None)),
|
||||
(None, None, Some(s)) => Ok((None, Some(s))),
|
||||
(Some(target), None, None) if looks_like_ref(&target) => Ok((Some(target), None)),
|
||||
(Some(target), None, None) => Ok((None, Some(target))),
|
||||
_ => Err(CliError::Local(anyhow::anyhow!(
|
||||
"pass exactly one of: <target>, --ref, or --selector"
|
||||
))),
|
||||
}
|
||||
}
|
||||
|
||||
fn format_used_target(used_ref: Option<&str>, used_selector: Option<&str>) -> String {
|
||||
used_ref
|
||||
.map(|r| format!("@{r}"))
|
||||
.or_else(|| used_selector.map(str::to_string))
|
||||
.unwrap_or_else(|| "?".into())
|
||||
}
|
||||
|
||||
fn ipc_timeout(timeout_ms: u32) -> Duration {
|
||||
Duration::from_millis(u64::from(timeout_ms))
|
||||
.checked_add(Duration::from_secs(15))
|
||||
.unwrap_or(Duration::from_secs(u64::from(timeout_ms / 1_000) + 15))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn split_target_detects_refs_and_selectors() {
|
||||
let (ref_, selector) = split_target(Some("@e3".into()), None, None).unwrap();
|
||||
assert_eq!(ref_.as_deref(), Some("@e3"));
|
||||
assert!(selector.is_none());
|
||||
|
||||
let (ref_, selector) = split_target(Some("#target".into()), None, None).unwrap();
|
||||
assert!(ref_.is_none());
|
||||
assert_eq!(selector.as_deref(), Some("#target"));
|
||||
}
|
||||
}
|
||||
@@ -243,6 +243,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
|
||||
| Method::ToolReload
|
||||
| Method::ToolClick
|
||||
| Method::ToolHover
|
||||
| Method::ToolScrollTo
|
||||
| Method::ToolFill
|
||||
| Method::ToolPress
|
||||
| Method::ToolSelect
|
||||
|
||||
@@ -94,6 +94,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
|
||||
Command::Reload(args) => cli::navigate::dispatch_reload(args, format),
|
||||
Command::Click(args) => cli::interaction::dispatch_click(args, format),
|
||||
Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
|
||||
Command::ScrollTo(args) => cli::scroll::dispatch(args, format),
|
||||
Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
|
||||
Command::Press(args) => cli::interaction::dispatch_press(args, format),
|
||||
Command::Select(args) => cli::interaction::dispatch_select(args, format),
|
||||
|
||||
@@ -206,6 +206,15 @@ fn parses_hover_with_settle() {
|
||||
assert_eq!(args.settle, 300);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_scroll_to_target() {
|
||||
let cli = parse(&["bsk", "scroll-to", "@e2", "--session", "s1"]);
|
||||
let Command::ScrollTo(args) = cli.command else {
|
||||
panic!("expected scroll-to command");
|
||||
};
|
||||
assert_eq!(args.target.as_deref(), Some("@e2"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_zero_click_count() {
|
||||
assert!(
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
//! M7 integration: drive the 7 new tool RPCs (navigate /
|
||||
//! navigate_back / navigate_forward / reload / click / fill / press)
|
||||
//! M7 integration: drive interaction and navigation tool RPCs
|
||||
//! through the IPC + per-session queue + fake extension and assert
|
||||
//! the wire shapes line up. The extension stub mirrors each method's
|
||||
//! `*Result` so we exercise the daemon's serialise → forward →
|
||||
@@ -15,8 +14,8 @@ use bsk_protocol::system::{HandshakeParams, HandshakeResult};
|
||||
use bsk_protocol::tools::{
|
||||
ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, NavigateBackParams,
|
||||
NavigateBackResult, NavigateForwardParams, NavigateForwardResult, NavigateParams,
|
||||
NavigateResult, PressParams, PressResult, ReloadParams, ReloadResult, SelectParams,
|
||||
SelectResult, SessionStartParams, SessionStartResult, WaitUntil,
|
||||
NavigateResult, PressParams, PressResult, ReloadParams, ReloadResult, ScrollToParams,
|
||||
ScrollToResult, SelectParams, SelectResult, SessionStartParams, SessionStartResult, WaitUntil,
|
||||
};
|
||||
use bsk_protocol::{
|
||||
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
|
||||
@@ -382,6 +381,51 @@ async fn click_round_trips_ref_and_modifiers() {
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn scroll_to_round_trips_visible_bounds() {
|
||||
let (handle, sock) = spawn_daemon().await;
|
||||
let mut ws = connect_ext(handle.ws_addr()).await;
|
||||
let _ = do_handshake(&mut ws).await;
|
||||
|
||||
run_extension(ws, |req| {
|
||||
assert_eq!(req.method, Method::ToolScrollTo);
|
||||
let params = req.params.clone().unwrap();
|
||||
let parsed: ScrollToParams = serde_json::from_value(params).unwrap();
|
||||
assert_eq!(parsed.ref_.as_deref(), Some("@e3"));
|
||||
ResponseBody::Ok(
|
||||
serde_json::to_value(ScrollToResult {
|
||||
tab_id: 9,
|
||||
used_ref: Some("e3".into()),
|
||||
used_selector: None,
|
||||
x: 10.0,
|
||||
y: 20.0,
|
||||
width: 100.0,
|
||||
height: 40.0,
|
||||
dialogs: vec![],
|
||||
})
|
||||
.unwrap(),
|
||||
)
|
||||
});
|
||||
|
||||
let session_id = ipc_session_start(&sock).await;
|
||||
let result: ScrollToResult = ipc_tool_call(
|
||||
&sock,
|
||||
Method::ToolScrollTo,
|
||||
ScrollToParams {
|
||||
session_id,
|
||||
ref_: Some("@e3".into()),
|
||||
selector: None,
|
||||
tab_id: None,
|
||||
timeout_ms: Some(5_000),
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("scroll-to ok");
|
||||
assert_eq!(result.used_ref.as_deref(), Some("e3"));
|
||||
assert_eq!(result.width, 100.0);
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn fill_round_trips_clear_before_default() {
|
||||
let (handle, sock) = spawn_daemon().await;
|
||||
|
||||
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "ScrollToParams",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"session_id"
|
||||
],
|
||||
"properties": {
|
||||
"ref": {
|
||||
"description": "Optional `@e<N>` ref allocated by the last observation. Mutually exclusive with `selector`.",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"selector": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"session_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"tab_id": {
|
||||
"description": "Target tab. Defaults to the Agent Window's active tab.",
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "int64"
|
||||
},
|
||||
"timeout_ms": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "uint32",
|
||||
"minimum": 1.0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "ScrollToResult",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"height",
|
||||
"tab_id",
|
||||
"width",
|
||||
"x",
|
||||
"y"
|
||||
],
|
||||
"properties": {
|
||||
"dialogs": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/JavaScriptDialogInfo"
|
||||
}
|
||||
},
|
||||
"height": {
|
||||
"type": "number",
|
||||
"format": "double"
|
||||
},
|
||||
"tab_id": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
},
|
||||
"used_ref": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"used_selector": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"width": {
|
||||
"type": "number",
|
||||
"format": "double"
|
||||
},
|
||||
"x": {
|
||||
"description": "Visible target bounds in top-level viewport CSS pixels.",
|
||||
"type": "number",
|
||||
"format": "double"
|
||||
},
|
||||
"y": {
|
||||
"type": "number",
|
||||
"format": "double"
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"JavaScriptDialogHandledAction": {
|
||||
"description": "How the extension resolved the dialog so CDP could continue.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"accepted",
|
||||
"dismissed"
|
||||
]
|
||||
},
|
||||
"JavaScriptDialogInfo": {
|
||||
"description": "One observed + handled JavaScript dialog during a tool call.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"handled",
|
||||
"message",
|
||||
"sequence",
|
||||
"tab_id",
|
||||
"type"
|
||||
],
|
||||
"properties": {
|
||||
"default_prompt": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"handled": {
|
||||
"$ref": "#/definitions/JavaScriptDialogHandledAction"
|
||||
},
|
||||
"has_browser_handler": {
|
||||
"type": [
|
||||
"boolean",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"message": {
|
||||
"type": "string"
|
||||
},
|
||||
"sequence": {
|
||||
"description": "Monotonic per-tab sequence for ordering within a session.",
|
||||
"type": "integer",
|
||||
"format": "uint64",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"tab_id": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
},
|
||||
"type": {
|
||||
"$ref": "#/definitions/JavaScriptDialogType"
|
||||
},
|
||||
"url": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"JavaScriptDialogType": {
|
||||
"description": "Native JS dialog kind reported by CDP.",
|
||||
"type": "string",
|
||||
"enum": [
|
||||
"alert",
|
||||
"confirm",
|
||||
"prompt",
|
||||
"beforeunload"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -79,6 +79,8 @@ fn main() {
|
||||
dump!(ClickResult, "tool_click_result");
|
||||
dump!(HoverParams, "tool_hover_params");
|
||||
dump!(HoverResult, "tool_hover_result");
|
||||
dump!(ScrollToParams, "tool_scroll_to_params");
|
||||
dump!(ScrollToResult, "tool_scroll_to_result");
|
||||
dump!(FillParams, "tool_fill_params");
|
||||
dump!(FillResult, "tool_fill_result");
|
||||
dump!(PressParams, "tool_press_params");
|
||||
|
||||
@@ -72,6 +72,8 @@ pub enum Method {
|
||||
ToolClick,
|
||||
#[serde(rename = "tool.hover")]
|
||||
ToolHover,
|
||||
#[serde(rename = "tool.scroll_to")]
|
||||
ToolScrollTo,
|
||||
#[serde(rename = "tool.fill")]
|
||||
ToolFill,
|
||||
#[serde(rename = "tool.press")]
|
||||
@@ -153,6 +155,7 @@ impl Method {
|
||||
| Method::ToolNavigateForward
|
||||
| Method::ToolReload
|
||||
| Method::ToolClick
|
||||
| Method::ToolScrollTo
|
||||
| Method::ToolFill
|
||||
| Method::ToolPress
|
||||
| Method::ToolSelect
|
||||
@@ -283,6 +286,7 @@ mod tests {
|
||||
assert!(Method::ToolNavigateForward.is_mutating());
|
||||
assert!(Method::ToolReload.is_mutating());
|
||||
assert!(Method::ToolClick.is_mutating());
|
||||
assert!(Method::ToolScrollTo.is_mutating());
|
||||
assert!(Method::ToolFill.is_mutating());
|
||||
assert!(Method::ToolPress.is_mutating());
|
||||
assert!(Method::ToolSelect.is_mutating());
|
||||
@@ -327,6 +331,7 @@ mod tests {
|
||||
assert_eq!(Method::ToolHover.effect(), MethodEffect::TransientInput);
|
||||
assert_eq!(Method::ToolObserve.effect(), MethodEffect::TransientInput);
|
||||
assert_eq!(Method::ToolClick.effect(), MethodEffect::BrowserMutation);
|
||||
assert_eq!(Method::ToolScrollTo.effect(), MethodEffect::BrowserMutation);
|
||||
assert_eq!(Method::Cancel.effect(), MethodEffect::ControlPlane);
|
||||
}
|
||||
|
||||
@@ -336,6 +341,7 @@ mod tests {
|
||||
assert!(Method::ToolHover.requires_interrupt_gate());
|
||||
assert!(Method::ToolObserve.requires_interrupt_gate());
|
||||
assert!(Method::ToolClick.requires_interrupt_gate());
|
||||
assert!(Method::ToolScrollTo.requires_interrupt_gate());
|
||||
assert!(!Method::Cancel.requires_interrupt_gate());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ pub mod network;
|
||||
pub mod observation;
|
||||
pub mod record;
|
||||
pub mod script;
|
||||
pub mod scroll;
|
||||
pub mod session;
|
||||
pub mod tabs;
|
||||
pub mod waits;
|
||||
@@ -25,6 +26,7 @@ pub use network::*;
|
||||
pub use observation::*;
|
||||
pub use record::*;
|
||||
pub use script::*;
|
||||
pub use scroll::*;
|
||||
pub use session::*;
|
||||
pub use tabs::*;
|
||||
pub use waits::*;
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
//! Element scrolling primitive (`tool.scroll_to`).
|
||||
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
use super::JavaScriptDialogInfo;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct ScrollToParams {
|
||||
pub session_id: String,
|
||||
/// Optional `@e<N>` ref allocated by the last observation.
|
||||
/// Mutually exclusive with `selector`.
|
||||
#[serde(
|
||||
rename = "ref",
|
||||
alias = "ref_",
|
||||
default,
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub ref_: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub selector: Option<String>,
|
||||
/// Target tab. Defaults to the Agent Window's active tab.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub tab_id: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[schemars(range(min = 1))]
|
||||
pub timeout_ms: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct ScrollToResult {
|
||||
pub tab_id: i64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub used_ref: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub used_selector: Option<String>,
|
||||
/// Visible target bounds in top-level viewport CSS pixels.
|
||||
pub x: f64,
|
||||
pub y: f64,
|
||||
pub width: f64,
|
||||
pub height: f64,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn params_serialise_ref_field_name() {
|
||||
let params = ScrollToParams {
|
||||
session_id: "abcd".into(),
|
||||
ref_: Some("@e3".into()),
|
||||
selector: None,
|
||||
tab_id: Some(42),
|
||||
timeout_ms: Some(5_000),
|
||||
};
|
||||
let value = serde_json::to_value(¶ms).unwrap();
|
||||
assert_eq!(value.get("ref").and_then(|v| v.as_str()), Some("@e3"));
|
||||
assert!(value.get("ref_").is_none());
|
||||
let round: ScrollToParams = serde_json::from_value(value).unwrap();
|
||||
assert_eq!(round, params);
|
||||
}
|
||||
}
|
||||
@@ -202,6 +202,7 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
|
||||
|---------|---------|
|
||||
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
||||
| `bsk scroll-to <ref-or-selector>` | Scroll an element and its frame owners into the visible viewport |
|
||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
||||
|
||||
Reference in New Issue
Block a user