Merge pull request #127 from NianJiuZst/codex/scroll-to-element-primitive

feat(interaction): add scroll-to-element primitive
This commit is contained in:
Zhang GH
2026-09-10 11:04:06 +08:00
committed by GitHub
32 changed files with 1731 additions and 24 deletions
+7
View File
@@ -5,6 +5,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/).
Starting from 0.2.0, CLI / Extension / DSH Plugin share the same version number. Starting from 0.2.0, CLI / Extension / DSH Plugin share the same version number.
## [Unreleased]
### Added
- [Scroll-to element primitive](docs/scroll-to.md) across CLI, Extension and DSH Plugin,
with ancestor-clipped visible bounds, iframe support and cooperative cancellation
## [0.2.1] - 2026-09-09 ## [0.2.1] - 2026-09-09
### Changed ### Changed
+3
View File
@@ -207,6 +207,9 @@ through the [plugin](#deepseek-harness-plugin): the agent calls injected
## For Developers ## For Developers
The [scroll-to primitive reference](docs/scroll-to.md) covers its CLI, protocol
and plugin entry points, visible bounds and interruption behavior.
The repository is a Cargo + pnpm workspace: The repository is a Cargo + pnpm workspace:
- `crates/bsk-cli` — `bsk` CLI and local daemon - `crates/bsk-cli` — `bsk` CLI and local daemon
+2
View File
@@ -177,6 +177,8 @@ Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;
## 面向开发者 ## 面向开发者
[scroll-to 原语说明](docs/scroll-to.md)介绍 CLI、协议和插件入口,以及可见区域、错误和中断语义。
本仓库是 Cargo + pnpm workspace: 本仓库是 Cargo + pnpm workspace:
- `crates/bsk-cli` — `bsk` CLI 与本地 daemon - `crates/bsk-cli` — `bsk` CLI 与本地 daemon
@@ -602,6 +602,7 @@ describe("ToolDispatcher", () => {
it.each([ it.each([
"focus", "focus",
"blur", "blur",
"scroll_to",
] as const)("routes %s with hover cleanup and cooperative cancellation", async (action) => { ] as const)("routes %s with hover cleanup and cooperative cancellation", async (action) => {
const tab = { id: 7, windowId: 4242, active: true }; const tab = { id: 7, windowId: 4242, active: true };
vi.stubGlobal("chrome", { vi.stubGlobal("chrome", {
@@ -654,7 +655,15 @@ describe("ToolDispatcher", () => {
}); });
expect(cdp.send).not.toHaveBeenCalledWith(7, "DOM.focus", expect.anything()); expect(cdp.send).not.toHaveBeenCalledWith(7, "DOM.focus", expect.anything());
expect(cdp.send).not.toHaveBeenCalledWith(7, "Runtime.callFunctionOn", expect.anything()); expect(cdp.send).not.toHaveBeenCalledWith(7, "Runtime.callFunctionOn", expect.anything());
expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObject", { objectId: "focus-target" }); if (action === "scroll_to") {
expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", {
objectGroup: expect.any(String),
});
} else {
expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObject", {
objectId: "focus-target",
});
}
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith( expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
7, 7,
expect.objectContaining({ enabled: false }), expect.objectContaining({ enabled: false }),
@@ -0,0 +1,348 @@
// @vitest-environment node
// Opt in with BSK_SCROLL_CHROME=/path/to/chrome. Uses an isolated browser/profile.
import { describe, expect, it } from "vitest";
import type { CdpFrame, CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
import { SessionManager } from "@/session-manager/manager";
import type { ViewportRect } from "../geometry";
import { handleScrollTo } from "../scroll";
import type { CdpRunner } from "../shared";
type Send = <T = Record<string, unknown>>(
method: string,
params?: object,
sessionId?: string,
) => Promise<T>;
type Tree = { frame: { id: string; parentId?: string; name?: string }; childFrames?: Tree[] };
async function withBrowser(
run: (h: Awaited<ReturnType<typeof harness>>) => Promise<void>,
zoom = 1,
) {
const { withChrome } = await import(
new URL(
"../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs",
import.meta.url,
).href
);
await withChrome(
{ executable: process.env.BSK_SCROLL_CHROME, deviceScale: 1, zoom },
async (send: Send) => run(await harness(send)),
);
}
async function harness(send: Send) {
const { targetId } = await send<{ targetId: string }>("Target.createTarget", {
url: "about:blank",
});
const { sessionId: rootSession } = await send<{ sessionId: string }>("Target.attachToTarget", {
targetId,
flatten: true,
});
await send("Page.bringToFront", {}, rootSession);
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("aa11");
const calls: { method: string; sessionId?: string }[] = [];
const after = { command: (_method: string) => {} };
const sessionFor = (target: CdpTarget) => target.sessionId ?? rootSession;
const forward = async <T>(target: CdpTarget, method: string, params?: object): Promise<T> => {
calls.push({ method, sessionId: target.sessionId });
const result = await send<T>(method, params, sessionFor(target));
after.command(method);
return result;
};
const cdp: CdpRunner = {
send: (_tab, method, params) => forward({ tabId: 4 }, method, params),
sendToTarget: forward,
};
const evaluate = async <T>(expression: string, frame?: CdpFrame): Promise<T> => {
const session = frame ? sessionFor(frame.target) : rootSession;
const world = frame
? await send<{ executionContextId: number }>(
"Page.createIsolatedWorld",
{
frameId: frame.frameId,
worldName: "scroll-oracle",
},
session,
)
: undefined;
const reply = await send<{ result: { value: T }; exceptionDetails?: unknown }>(
"Runtime.evaluate",
{
expression,
contextId: world?.executionContextId,
returnByValue: true,
awaitPromise: true,
},
session,
);
expect(reply.exceptionDetails).toBeUndefined();
return reply.result.value;
};
const rememberRef = (frame: CdpFrame, backendNodeId: number) =>
ctx.refStore.set("e1", backendNodeId, {
tabId: 4,
frameId: frame.frameId,
cdpSessionId: frame.target.sessionId,
});
const ref = async (frame: CdpFrame, selector = "#probe") => {
const { executionContextId } = await send<{ executionContextId: number }>(
"Page.createIsolatedWorld",
{
frameId: frame.frameId,
worldName: "scroll-oracle",
},
sessionFor(frame.target),
);
const { result } = await send<{ result: { objectId: string } }>(
"Runtime.evaluate",
{
expression: `document.querySelector(${JSON.stringify(selector)})`,
contextId: executionContextId,
},
sessionFor(frame.target),
);
const { node } = await send<{ node: { backendNodeId: number } }>(
"DOM.describeNode",
{ objectId: result.objectId },
sessionFor(frame.target),
);
await send("Runtime.releaseObject", { objectId: result.objectId }, sessionFor(frame.target));
rememberRef(frame, node.backendNodeId);
};
const scroll = (target: { selector: string } | { ref: string }, signal?: AbortSignal) =>
handleScrollTo(
manager,
{ session_id: "aa11", ...target },
{
cdp,
signal,
tabsApi: {
get: async (id) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab,
query: async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab],
},
},
);
const loadFrames = async () => {
const { targetInfos } = await send<{ targetInfos: { targetId: string; type: string }[] }>(
"Target.getTargets",
);
const sessions = [rootSession];
for (const target of targetInfos.filter((t) => t.type === "iframe")) {
const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", {
targetId: target.targetId,
flatten: true,
});
sessions.push(sessionId);
}
const frames: (CdpFrame & { name: string })[] = [];
for (const session of sessions) {
const { frameTree } = await send<{ frameTree: Tree }>("Page.getFrameTree", {}, session);
const target = { tabId: 4, ...(session !== rootSession ? { sessionId: session } : {}) };
const visit = (tree: Tree, parentFrameId = tree.frame.parentId) => {
frames.push({ frameId: tree.frame.id, parentFrameId, target, name: tree.frame.name ?? "" });
for (const child of tree.childFrames ?? []) visit(child, tree.frame.id);
};
visit(frameTree);
}
for (const frame of frames) {
if (!frame.parentFrameId) continue;
const parent = frames.find((f) => f.frameId === frame.parentFrameId)!;
const owner = await send<{ backendNodeId: number }>(
"DOM.getFrameOwner",
{ frameId: frame.frameId },
sessionFor(parent.target),
);
frame.ownerBackendNodeId = owner.backendNodeId;
}
const graph: CdpFrameGraph = { rootFrameId: frames[0].frameId, frames };
cdp.getFrameGraph = async () => graph;
return frames;
};
return { send, rootSession, evaluate, ref, rememberRef, scroll, loadFrames, calls, after };
}
function intersection(a: ViewportRect, b: ViewportRect): ViewportRect | null {
const x = Math.max(a.x, b.x),
y = Math.max(a.y, b.y);
const right = Math.min(a.x + a.width, b.x + b.width),
bottom = Math.min(a.y + a.height, b.y + b.height);
return right > x && bottom > y ? { x, y, width: right - x, height: bottom - y } : null;
}
function expectRect(actual: unknown, expected: ViewportRect) {
expect(actual).not.toHaveProperty("code");
for (const key of ["x", "y", "width", "height"] as const)
expect(Math.abs((actual as ViewportRect)[key] - expected[key]), key).toBeLessThan(2);
}
describe.skipIf(!process.env.BSK_SCROLL_CHROME)("real browser scroll-to", () => {
it("scrolls ordinary elements and reports the clipped portion of nested scrollers", async () => {
await withBrowser(async (h) => {
for (const html of [
'<div style="height:2200px"></div><div id="target" style="width:100px;height:40px"></div>',
'<div style="height:200px"></div><div id="clip" style="height:80px;width:200px;overflow:auto"><div id="target" style="width:100px;height:400px"></div></div>',
'<div style="height:200px"></div><div id="clip" style="height:80px;width:200px;overflow:auto;transform:scale(1.25);transform-origin:0 0"><div id="target" style="width:100px;height:400px"></div></div>',
]) {
await h.evaluate(`document.body.innerHTML = ${JSON.stringify(html)}`);
const result = await h.scroll({ selector: "#target" });
const oracle = await h.evaluate<{ rect: ViewportRect; clip?: ViewportRect }>(`({
rect: document.querySelector('#target').getBoundingClientRect().toJSON(),
clip: document.querySelector('#clip')?.getBoundingClientRect().toJSON()
})`);
expectRect(result, oracle.clip ? intersection(oracle.rect, oracle.clip)! : oracle.rect);
}
});
});
it.each([
"visibility:hidden",
"opacity:0",
"display:none",
"content-visibility:hidden",
])("rejects %s targets", async (style) => {
await withBrowser(async (h) => {
await h.evaluate(
`document.body.innerHTML = '<div style="${style}"><button id="target">Target</button></div>'`,
);
expect(await h.scroll({ selector: "#target" })).toMatchObject({
code: "permission_denied",
data: { reason: "element_not_visible" },
});
});
});
it("rejects targets fully outside an overflow:clip ancestor", async () => {
await withBrowser(async (h) => {
await h.evaluate(
`document.body.innerHTML = '<div style="height:80px;overflow:clip"><div style="height:150px"></div><button id="target">Target</button></div>'`,
);
expect(await h.scroll({ selector: "#target" })).toMatchObject({ code: "permission_denied" });
});
});
it.each(["open", "closed"])("clips %s shadow-root refs", async (mode) => {
await withBrowser(async (h) => {
await h.evaluate(`(() => {
const host = document.createElement('div'); document.body.append(host);
const root = host.attachShadow({mode:'${mode}'});
root.innerHTML = '<div style="height:60px;overflow:auto"><button id="probe" style="height:200px;width:100px">Target</button></div>';
window.probe = root.querySelector('#probe');
})()`);
const frames = await h.loadFrames();
const { result } = await h.send<{ result: { objectId: string } }>(
"Runtime.evaluate",
{ expression: "window.probe" },
h.rootSession,
);
// Build the ref from the actual closed-root node, as an observation does.
const { node } = await h.send<{ node: { backendNodeId: number } }>(
"DOM.describeNode",
{ objectId: result.objectId },
h.rootSession,
);
await h.send("Runtime.releaseObject", { objectId: result.objectId }, h.rootSession);
h.rememberRef(frames[0], node.backendNodeId);
expect(await h.scroll({ ref: "e1" })).toMatchObject({ height: 60 });
});
});
it("cancels after scrolling and releases the measurement object without follow-up reads", async () => {
await withBrowser(async (h) => {
await h.evaluate("document.body.innerHTML = '<button id=target>Target</button>'");
const abort = new AbortController();
h.after.command = (method) => {
if (method === "DOM.resolveNode") abort.abort();
};
expect(await h.scroll({ selector: "#target" }, abort.signal)).toMatchObject({
code: "cancelled",
});
expect(h.calls.at(-1)!.method).toBe("Runtime.releaseObjectGroup");
expect(h.calls.some((call) => call.method === "Runtime.callFunctionOn")).toBe(false);
});
});
it.each([
{ fixture: "snapshot-coordinates", zoom: 1 },
{ fixture: "snapshot-coordinates", zoom: 1.25 },
{ fixture: "oopif-scrollbars", zoom: 1 },
{ fixture: "oopif-scrollbars", zoom: 0.8 },
])("scrolls nested frames: $fixture at zoom $zoom", async ({ fixture, zoom }) => {
const { createEvalServer } = await import(
new URL("../../../../../evals/browser/lib/server.mjs", import.meta.url).href
);
const server = createEvalServer();
const { baseUrl } = await server.start();
try {
await withBrowser(async (h) => {
await h.send("Page.navigate", { url: `${baseUrl}/${fixture}?run=scroll` }, h.rootSession);
await expect
.poll(
() =>
server
.snapshot("scroll")
.events.some(
(e: { type: string; data: { root?: boolean } }) =>
e.type === "geometry.ready" && e.data.root,
),
{ timeout: 10_000 },
)
.toBe(true);
const frames = await h.loadFrames();
expect(frames.length).toBe(fixture === "snapshot-coordinates" ? 5 : 3);
for (const frame of frames) {
await h.ref(frame);
const result = await h.scroll({ ref: "e1" });
// Independent DOM oracle: CSS border boxes plus this fixture's known
// axis-aligned iframe scales. Re-read after each scroll changes layout.
let rect = await h.evaluate<ViewportRect>(
"document.querySelector('#probe').getBoundingClientRect().toJSON()",
frame,
);
let current = frame;
while (true) {
const viewport = await h.evaluate<ViewportRect>(
"({ x:0, y:0, width:document.documentElement.clientWidth, height:document.documentElement.clientHeight })",
current,
);
rect = intersection(rect, viewport)!;
if (!current.parentFrameId) break;
const parent = frames.find((f) => f.frameId === current.parentFrameId)!;
const owner = await h.evaluate<{ x: number; y: number; scale: number }>(
`(() => {
const iframe = document.querySelector('iframe[name="${current.name}"]');
const r = iframe.getBoundingClientRect(); const s = getComputedStyle(iframe);
const scale = r.width / iframe.offsetWidth;
return { x:r.x + (iframe.clientLeft + parseFloat(s.paddingLeft))*scale,
y:r.y + (iframe.clientTop + parseFloat(s.paddingTop))*scale, scale };
})()`,
parent,
);
rect = {
x: owner.x + rect.x * owner.scale,
y: owner.y + rect.y * owner.scale,
width: rect.width * owner.scale,
height: rect.height * owner.scale,
};
current = parent;
}
expectRect(result, rect);
}
const child = frames.find((frame) => frame.parentFrameId === frames[0].frameId)!;
await h.ref(child);
await h.evaluate(
`document.querySelector('iframe[name="${child.name}"]').style.opacity = '0'`,
);
expect(await h.scroll({ ref: "e1" })).toMatchObject({ code: "permission_denied" });
}, zoom);
} finally {
await server.stop();
}
}, 30_000);
});
@@ -0,0 +1,310 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
import { SessionManager } from "@/session-manager/manager";
import type { JavaScriptDialogInfo, ScrollToParams } from "@/transport/types";
import type { ViewportRect } from "../geometry";
import { handleScrollTo } from "../scroll";
import type { CdpRunner } from "../shared";
interface Call {
target: CdpTarget;
method: string;
params?: Record<string, unknown>;
aborted: boolean;
}
async function fixture(frame: "top" | "same-target" | "oopif" = "top") {
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("aa11");
ctx.refStore.set("e3", 1234, {
tabId: 4,
...(frame !== "top" ? { frameId: "child" } : {}),
...(frame === "oopif" ? { cdpSessionId: "child-session" } : {}),
});
const abort = new AbortController();
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 99,
target: { tabId: 4, ...(frame === "oopif" ? { sessionId: "child-session" } : {}) },
},
],
};
const rectangles = new Map<number, ViewportRect | null>([
[1234, { x: 10, y: 20, width: 100, height: 40 }],
[99, { x: 204, y: 306, width: 400, height: 200 }],
]);
const calls: Call[] = [];
const hooks = {
after: (_call: Call) => {},
reply: (_call: Call): object | undefined => undefined,
};
const send = async <T>(target: CdpTarget, method: string, params?: object): Promise<T> => {
const call = {
target,
method,
params: params as Call["params"],
aborted: abort.signal.aborted,
};
calls.push(call);
const override = hooks.reply(call);
hooks.after(call);
if (override) return override as T;
const args = call.params ?? {};
const replies: Record<string, unknown> = {
"DOM.getDocument": { root: { nodeId: 1 } },
"DOM.querySelector": { nodeId: 2 },
"DOM.describeNode": { node: { backendNodeId: 1234 } },
"DOM.scrollIntoViewIfNeeded": {},
"DOM.resolveNode": { object: { objectId: String(args.backendNodeId) } },
"Runtime.callFunctionOn": {
result: {
value: String(args.functionDeclaration).includes("IntersectionObserver")
? rectangles.get(Number(args.objectId))
: { width: 200, height: 100 },
},
},
"Runtime.releaseObject": {},
"Runtime.releaseObjectGroup": {},
"Runtime.evaluate": { result: { value: { width: 200, height: 100 } } },
"Page.getLayoutMetrics": {
cssLayoutViewport: {
clientWidth: target.sessionId ? 185 : 1280,
clientHeight: target.sessionId ? 89 : 720,
},
},
"DOM.getBoxModel": { model: { content: [204, 306, 604, 306, 604, 506, 204, 506] } },
};
if (!(method in replies)) throw new Error(`unexpected CDP call ${method}`);
return replies[method] as T;
};
const cdp: CdpRunner = {
send: (tabId, method, params) => send({ tabId }, method, params),
sendToTarget: send,
getFrameGraph: vi.fn(async () => {
hooks.after({ target: { tabId: 4 }, method: "getFrameGraph", aborted: abort.signal.aborted });
return graph;
}),
trackSessionTab: vi.fn(),
};
const tabsApi = {
get: vi.fn(async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
};
const run = (params: Partial<ScrollToParams> = {}) =>
handleScrollTo(
manager,
{ session_id: "aa11", ref: "@e3", ...params },
{ cdp, tabsApi, signal: abort.signal },
);
return { ctx, cdp, graph, rectangles, calls, hooks, abort, tabsApi, run };
}
afterEach(() => vi.restoreAllMocks());
describe("handleScrollTo", () => {
it("scrolls a ref and returns renderer-clipped bounds", async () => {
const f = await fixture();
expect(await f.run()).toMatchObject({
tab_id: 4,
used_ref: "e3",
x: 10,
y: 20,
width: 100,
height: 40,
});
expect(f.calls[0]).toMatchObject({
method: "DOM.scrollIntoViewIfNeeded",
params: { backendNodeId: 1234 },
});
const group = f.calls.find((call) => call.method === "DOM.resolveNode")!.params!.objectGroup;
expect(f.calls.at(-1)).toMatchObject({
method: "Runtime.releaseObjectGroup",
params: { objectGroup: group },
});
});
it("resolves a selector and tracks the attached tab even when it is missing", async () => {
const f = await fixture();
expect(await f.run({ ref: undefined, selector: "#target" })).toMatchObject({
used_selector: "#target",
});
f.hooks.reply = ({ method }) => (method === "DOM.querySelector" ? { nodeId: 0 } : undefined);
expect(await f.run({ ref: undefined, selector: "#missing" })).toMatchObject({
code: "not_found",
data: { reason: "selector_not_found" },
});
expect(f.cdp.trackSessionTab).toHaveBeenCalledWith("aa11", 4);
});
it.each([
"same-target",
"oopif",
] as const)("projects %s refs and scrolls their owners", async (frame) => {
const f = await fixture(frame);
expect(await f.run()).toMatchObject({ x: 224, y: 346, width: 200, height: 80 });
expect(
f.calls
.filter((call) => call.method === "DOM.scrollIntoViewIfNeeded")
.map((call) => call.params!.backendNodeId),
).toEqual([99, 1234]);
expect(f.cdp.getFrameGraph).toHaveBeenCalledTimes(1);
if (frame === "oopif")
expect(f.calls.find((call) => call.params?.backendNodeId === 1234)!.target.sessionId).toBe(
"child-session",
);
});
it("clips OOPIF bounds to occupied scrollbars without changing the frame scale", async () => {
const f = await fixture("oopif");
f.rectangles.set(1234, { x: 180, y: 80, width: 5, height: 9 });
expect(await f.run()).toMatchObject({ x: 564, y: 466, width: 10, height: 18 });
});
it("clips against the iframe owner's ordinary DOM ancestors", async () => {
const f = await fixture("oopif");
f.rectangles.set(99, { x: 204, y: 306, width: 400, height: 80 });
expect(await f.run()).toMatchObject({ x: 224, y: 346, width: 200, height: 40 });
});
it.each([1234, 99])("rejects a hidden or fully clipped node %i", async (node) => {
const f = await fixture("oopif");
f.rectangles.set(node, null);
expect(await f.run()).toMatchObject({
code: "permission_denied",
data: { reason: "element_not_visible" },
});
expect(f.calls.at(-1)!.method).toBe("Runtime.releaseObjectGroup");
});
it.each([
{ ref: undefined },
{ selector: "#target" },
{ timeout_ms: 0 },
])("rejects invalid params %j", async (params) => {
const f = await fixture();
expect(await f.run(params)).toMatchObject({ code: "invalid_params" });
expect(f.calls).toEqual([]);
});
it.each(["stale", "other-tab"])("rejects %s refs before scrolling", async (mode) => {
const f = await fixture();
if (mode === "stale") f.ctx.refStore.clear();
else f.ctx.refStore.set("e3", 1234, { tabId: 8 });
expect(await f.run()).toMatchObject({ code: "not_found", data: { reason: "ref_not_found" } });
expect(f.calls).toEqual([]);
});
it("enforces the Agent Window boundary", async () => {
const f = await fixture();
f.tabsApi.query.mockResolvedValue([{ id: 4, windowId: 200, active: true } as chrome.tabs.Tab]);
f.tabsApi.get.mockResolvedValue({ id: 4, windowId: 200, active: true } as chrome.tabs.Tab);
expect(await f.run({ tab_id: 4 })).toMatchObject({ code: "permission_denied" });
expect(f.calls).toEqual([]);
});
it.each([
"getFrameGraph",
"DOM.scrollIntoViewIfNeeded",
"DOM.resolveNode",
"Runtime.callFunctionOn",
"Page.getLayoutMetrics",
"Runtime.evaluate",
"DOM.getBoxModel",
])("honors cancellation during %s and only cleans up afterwards", async (method) => {
const f = await fixture("oopif");
f.hooks.after = (call) => {
if (call.method === method) f.abort.abort();
};
expect(await f.run()).toMatchObject({ code: "cancelled" });
expect(
f.calls.filter((call) => call.aborted && call.method !== "Runtime.releaseObjectGroup"),
).toEqual([]);
});
it.each([
"DOM.getDocument",
"DOM.querySelector",
"DOM.describeNode",
])("cancels selector resolution during %s", async (method) => {
const f = await fixture();
f.hooks.after = (call) => {
if (call.method === method) f.abort.abort();
};
expect(await f.run({ ref: undefined, selector: "#target" })).toMatchObject({
code: "cancelled",
});
expect(f.calls.filter((call) => call.aborted)).toEqual([]);
});
it("does not start after early cancellation", async () => {
const f = await fixture();
f.abort.abort();
expect(await f.run()).toMatchObject({ code: "cancelled" });
expect(f.calls).toEqual([]);
expect(f.tabsApi.query).not.toHaveBeenCalled();
});
it("does not issue fallback scrolling after cancellation", async () => {
const f = await fixture();
f.hooks.reply = ({ method }) => {
if (method === "DOM.scrollIntoViewIfNeeded") {
f.abort.abort();
throw new Error("scroll failed");
}
return undefined;
};
expect(await f.run()).toMatchObject({ code: "cancelled" });
expect(f.calls.map((call) => call.method)).toEqual(["DOM.scrollIntoViewIfNeeded"]);
});
it("stops target scrolling after a deadline expires while scrolling its owner", async () => {
const f = await fixture("oopif");
const now = vi.spyOn(Date, "now").mockReturnValue(100);
f.hooks.after = ({ method }) => {
if (method === "DOM.scrollIntoViewIfNeeded") now.mockReturnValue(201);
};
expect(await f.run({ timeout_ms: 100 })).toMatchObject({ code: "timeout" });
expect(f.calls.filter((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toHaveLength(1);
});
it("reports measurement exceptions and releases objects even if cleanup fails", async () => {
const f = await fixture();
f.hooks.reply = ({ method }) => {
if (method === "Runtime.callFunctionOn")
return { exceptionDetails: { text: "measurement failed" } };
if (method === "Runtime.releaseObjectGroup") throw new Error("target closed");
return undefined;
};
expect(await f.run()).toMatchObject({ code: "cdp_failed", message: "measurement failed" });
expect(f.calls.at(-1)!.method).toBe("Runtime.releaseObjectGroup");
});
it("attaches dialogs observed during scrolling", async () => {
const f = await fixture();
f.cdp.dialogCursor = () => 5;
const dialogs: JavaScriptDialogInfo[] = [
{
sequence: 6,
tab_id: 4,
type: "alert",
message: "scrolled",
handled: "accepted",
url: "https://example.test",
},
];
f.cdp.dialogsSince = vi.fn(() => dialogs);
expect(await f.run()).toHaveProperty("dialogs", dialogs);
expect(f.cdp.dialogsSince).toHaveBeenCalledWith(4, 5);
});
});
+14
View File
@@ -29,6 +29,7 @@ import type {
ResponseFrame, ResponseFrame,
RpcError, RpcError,
ScreenshotParams, ScreenshotParams,
ScrollToParams,
SelectParams, SelectParams,
SnapshotParams, SnapshotParams,
UploadParams, UploadParams,
@@ -71,6 +72,7 @@ import {
handleRecordStop, handleRecordStop,
type RecordRuntimeDeps, type RecordRuntimeDeps,
} from "./record"; } from "./record";
import { handleScrollTo } from "./scroll";
import { import {
handleSessionStart, handleSessionStart,
handleSessionStop, handleSessionStop,
@@ -525,6 +527,17 @@ export class ToolDispatcher {
); );
return this.rememberHover((req.params as HoverParams).session_id, result); return this.rememberHover((req.params as HoverParams).session_id, result);
} }
case "tool.scroll_to":
return this.withHoverReleaseForRequest(
req.params as ScrollToParams,
() =>
handleScrollTo(
this.sessions,
req.params as ScrollToParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
),
signal,
);
case "tool.focus": case "tool.focus":
return this.withHoverReleaseForRequest( return this.withHoverReleaseForRequest(
req.params as FocusParams, req.params as FocusParams,
@@ -817,6 +830,7 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
case "tool.reload": case "tool.reload":
case "tool.click": case "tool.click":
case "tool.hover": case "tool.hover":
case "tool.scroll_to":
case "tool.focus": case "tool.focus":
case "tool.blur": case "tool.blur":
case "tool.fill": case "tool.fill":
@@ -0,0 +1,142 @@
import { cdpTargetKey } from "@/browser-driver/frame-graph";
import type { NodeAddress } from "./frame-geometry";
import {
clipPolygon,
projectRegionToViewport,
type Region,
rectPolygon,
regionBounds,
type ViewportRect,
} from "./geometry";
import { GeometryContext } from "./geometry/frame-context";
import { type CdpRunner, sendToCdpTarget } from "./shared";
// Ask the renderer to apply containing-block clips (including shadow DOM),
// instead of treating layout quads as visible pixels. Each document is measured
// locally; the existing frame projections supply the top-viewport coordinates.
// IntersectionObserver reports a bounding rectangle, not an occlusion/hit test.
const VISIBLE_RECT = `function(timeoutMs) {
const element = this;
const visible = () => element.isConnected && element.checkVisibility({
checkOpacity: true, checkVisibilityCSS: true, contentVisibilityAuto: true
});
if (typeof element.checkVisibility !== 'function' || !visible()) return null;
return new Promise((resolve, reject) => {
const observer = new IntersectionObserver(([entry]) => {
clearTimeout(timer);
observer.disconnect();
const rect = entry.intersectionRect;
resolve(visible() && entry.isIntersecting && rect.width > 0 && rect.height > 0
? { x: rect.x, y: rect.y, width: rect.width, height: rect.height }
: null);
}, { root: element.ownerDocument });
const timer = setTimeout(() => {
observer.disconnect();
reject(new Error('scroll-to visibility measurement timed out'));
}, timeoutMs);
observer.observe(element);
});
}`;
async function visibleRect(
cdp: CdpRunner,
address: NodeAddress,
deadline: number,
): Promise<ViewportRect | null> {
const resolved = await sendToCdpTarget<{ object?: { objectId?: string } }>(
cdp,
address.target,
"DOM.resolveNode",
{ backendNodeId: address.backendNodeId },
);
if (!resolved.object?.objectId) throw new Error("scroll-to could not resolve the target element");
const reply = await sendToCdpTarget<{
result?: { value?: ViewportRect | null };
exceptionDetails?: { text?: string; exception?: { description?: string } };
}>(cdp, address.target, "Runtime.callFunctionOn", {
objectId: resolved.object.objectId,
functionDeclaration: VISIBLE_RECT,
// Bound this read even if a background document stops rendering. Its
// observer always disconnects; cancellation cannot leave a live observer.
arguments: [{ value: Math.max(1, Math.min(1_000, deadline - Date.now())) }],
awaitPromise: true,
returnByValue: true,
});
if (reply.exceptionDetails) {
const details = reply.exceptionDetails;
throw new Error(details.exception?.description ?? details.text ?? "visibility script failed");
}
const rect = reply.result?.value;
if (rect === null) return null;
if (
!rect ||
![rect.x, rect.y, rect.width, rect.height].every(Number.isFinite) ||
rect.width <= 0 ||
rect.height <= 0
) {
throw new Error("scroll-to visibility measurement returned invalid bounds");
}
return rect;
}
async function projectRect(
context: GeometryContext,
address: NodeAddress,
rect: ViewportRect,
): Promise<Region> {
let region: Region = [rectPolygon({ x: rect.x, y: rect.y, w: rect.width, h: rect.height })];
if (!address.frameId) return region;
const frame = await context.frame(address.frameId);
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(address.target)) {
throw new Error("scroll-to frame no longer belongs to its target");
}
const parent = frame.parentFrameId ? await context.frame(frame.parentFrameId) : undefined;
if (parent && cdpTargetKey(parent.target) === cdpTargetKey(frame.target)) {
// DOM rectangles are document-local. Same-target iframe quads are already
// target-local, so cross this one boundary before the target projection.
const viewport = await context.viewport(frame.target);
if (frame.ownerBackendNodeId === undefined || !viewport) {
throw new Error("scroll-to could not resolve the frame viewport");
}
const local = await context.snapshotProjection(address, frame.ownerBackendNodeId, [], viewport);
if (local.status !== "available") throw new Error("scroll-to could not project the frame");
region = projectRegionToViewport(region, local.projection.geometry);
}
const projection = await context.targetProjection(frame.frameId);
if (!projection) throw new Error("scroll-to could not project the target viewport");
return projectRegionToViewport(region, projection);
}
/** Only scroll-to needs renderer-clipped bounds; other interaction geometry is unchanged. */
export async function scrollVisibleBounds(
cdp: CdpRunner,
tabId: number,
address: NodeAddress,
deadline: number,
): Promise<ViewportRect | null> {
const context = new GeometryContext(cdp, tabId);
const rect = await visibleRect(cdp, address, deadline);
if (!rect) return null;
let region = await projectRect(context, address, rect);
if (address.frameId) {
const ancestry = await context.ancestry(address.frameId);
if (!ancestry) throw new Error("scroll-to could not resolve the frame ancestry");
for (const child of ancestry) {
const parent = child.parentFrameId ? await context.frame(child.parentFrameId) : undefined;
if (!parent || child.ownerBackendNodeId === undefined) {
throw new Error("scroll-to could not resolve the frame owner");
}
const owner = {
target: parent.target,
frameId: parent.frameId,
backendNodeId: child.ownerBackendNodeId,
};
// A visible child document can still have a hidden or clipped iframe.
const ownerRect = await visibleRect(cdp, owner, deadline);
if (!ownerRect) return null;
const clips = await projectRect(context, owner, ownerRect);
region = region.flatMap((polygon) => clips.map((clip) => clipPolygon(polygon, clip)));
}
}
return regionBounds(region);
}
+137
View File
@@ -0,0 +1,137 @@
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
import type { SessionManager } from "@/session-manager/manager";
import type { RpcError, ScrollToParams, ScrollToResult } from "@/transport/types";
import { attachDialogs, markDialogCursor } from "./dialogs";
import { cdpError, rpcError } from "./errors";
import { scrollElementAndFramesIntoView } from "./frame-geometry";
import { resolveBackendNode } from "./interaction";
import { scrollVisibleBounds } from "./scroll-visibility";
import {
type CdpRunner,
type ChromeTabsApi,
chromeTabsApi,
enforceAgentWindow,
isRpcError,
lookupSession,
resolveTargetTab,
sendToCdpTarget,
} from "./shared";
export interface ScrollToDeps {
cdp: CdpRunner;
tabsApi: ChromeTabsApi;
signal?: AbortSignal;
}
let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null;
function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } {
if (!defaultDeps) defaultDeps = { cdp: new ChromiumCdp(), tabsApi: chromeTabsApi };
return defaultDeps;
}
export async function handleScrollTo(
manager: SessionManager,
params: ScrollToParams,
deps: ScrollToDeps = getDefaultDeps(),
): Promise<ScrollToResult | RpcError> {
const ctxOrErr = lookupSession(manager, params, "scroll-to");
if (isRpcError(ctxOrErr)) return ctxOrErr;
const ctx = ctxOrErr;
const timeout = params.timeout_ms ?? 30_000;
if (!Number.isInteger(timeout) || timeout <= 0) {
return { code: "invalid_params", message: "timeout_ms must be a positive integer" };
}
const deadline = Date.now() + timeout;
const checkActive = () => {
if (deps.signal?.aborted) throw new DOMException("scroll-to aborted", "AbortError");
if (Date.now() >= deadline) throw new DOMException("scroll-to timed out", "TimeoutError");
};
const objectGroup = `bsk-scroll-${crypto.randomUUID()}`;
const objectTargets = new Map<string, CdpTarget>();
const send = async <T>(target: CdpTarget, method: string, args?: object): Promise<T> => {
checkActive();
// Also own objects allocated by the shared scroll fallback. Record the
// target before awaiting so cancellation cannot skip their cleanup.
if (method === "DOM.resolveNode") objectTargets.set(cdpTargetKey(target), target);
const result = await sendToCdpTarget<T>(
deps.cdp,
target,
method,
method === "DOM.resolveNode" ? { ...args, objectGroup } : args,
);
checkActive();
return result;
};
let graph: ReturnType<NonNullable<CdpRunner["getFrameGraph"]>> | undefined;
const cdp: CdpRunner = {
send: (tabId, method, args) => send({ tabId }, method, args),
sendToTarget: send,
trackSessionTab: deps.cdp.trackSessionTab?.bind(deps.cdp),
getFrameGraph: deps.cdp.getFrameGraph
? async (tabId) => {
checkActive();
graph ??= deps.cdp.getFrameGraph!(tabId);
const result = await graph;
checkActive();
return result;
}
: undefined,
};
try {
checkActive();
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
checkActive();
if (isRpcError(target)) return target;
const denied = enforceAgentWindow(ctx, target, "scroll-to");
if (denied) return denied;
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
const node = await resolveBackendNode(cdp, ctx, target, params, "scroll-to");
checkActive();
if (isRpcError(node)) return node;
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
const scrollError = await scrollElementAndFramesIntoView(
cdp,
target.tabId,
node.cdpTarget,
node.backendNodeId,
node.frameId,
);
checkActive();
if (scrollError) return scrollError;
const bounds = await scrollVisibleBounds(
cdp,
target.tabId,
{
target: node.cdpTarget,
backendNodeId: node.backendNodeId,
frameId: node.frameId,
},
deadline,
);
checkActive();
if (!bounds)
return rpcError(
"permission_denied",
"element_not_visible",
"scroll-to target has no visible area",
);
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
tab_id: target.tabId,
used_ref: node.usedRef,
used_selector: node.usedSelector,
...bounds,
});
} catch (error) {
if (deps.signal?.aborted) return { code: "cancelled", message: "scroll-to aborted" };
if (Date.now() >= deadline) return { code: "timeout", message: "scroll-to timed out" };
return cdpError(error);
} finally {
// Cleanup bypasses the cancellation guard and cannot replace the result.
for (const target of objectTargets.values()) {
await sendToCdpTarget(deps.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }).catch(
() => {},
);
}
}
}
+20
View File
@@ -470,6 +470,26 @@ export interface HoverResult {
dialogs?: JavaScriptDialogInfo[]; dialogs?: JavaScriptDialogInfo[];
} }
export interface ScrollToParams {
session_id: string;
ref?: string;
selector?: string;
tab_id?: number;
timeout_ms?: number;
}
export interface ScrollToResult {
tab_id: number;
used_ref?: string;
used_selector?: string;
/** Clipped border-box bounds in top-level viewport CSS pixels; not an occlusion test. */
x: number;
y: number;
width: number;
height: number;
dialogs?: JavaScriptDialogInfo[];
}
export interface FocusParams { export interface FocusParams {
session_id: string; session_id: string;
ref?: string; ref?: string;
+9 -2
View File
@@ -55,10 +55,17 @@ Use this default loop:
```text ```text
bsk navigate <url> --session <id> bsk navigate <url> --session <id>
bsk observe --session <id> bsk observe --session <id>
bsk click|hover|focus|blur|fill|select|press ... --session <id> bsk click|hover|scroll-to|focus|blur|fill|select|press ... --session <id>
bsk observe --session <id> # after navigation or a meaningful DOM change bsk observe --session <id> # after navigation or a meaningful DOM change
``` ```
`bsk scroll-to <ref-or-selector> --session <id>` scrolls an element and its frame owners into view.
Use a fresh element ref for iframe/shadow-root targets; CSS selectors search the main document.
The result is the visible border-box portion's bounds in top-level viewport CSS pixels after
ancestor clipping. Partial visibility is enough; hidden or fully clipped targets fail with
`permission_denied` and `data.reason=element_not_visible`. This does not test occlusion by other elements.
For a specific tab or deadline: `bsk scroll-to @e3 --session <id> --tab-id 42 --timeout 5s`.
`bsk focus <ref>` explicitly focuses a target; `bsk blur <ref>` removes focus and reports whether `bsk focus <ref>` explicitly focuses a target; `bsk blur <ref>` removes focus and reports whether
it was focused. Use these for UI states triggered by focus changes. it was focused. Use these for UI states triggered by focus changes.
@@ -117,7 +124,7 @@ This list of names is complete. Never invent a command outside it; read
session start|stop|list browsers status doctor update logs session start|stop|list browsers status doctor update logs
navigate navigate-back navigate-forward reload wait-for-navigation wait-ms navigate navigate-back navigate-forward reload wait-for-navigation wait-ms
observe snapshot get-html screenshot console network observe snapshot get-html screenshot console network
click hover focus blur fill select press evaluate click hover scroll-to focus blur fill select press evaluate
tab list|create|close|select|borrow|return window resize emulate tab list|create|close|select|borrow|return window resize emulate
upload download request-help record start|stop upload download request-help record start|stop
``` ```
+6
View File
@@ -28,6 +28,7 @@ pub mod record_recovery;
pub mod record_state; pub mod record_state;
pub mod render_error; pub mod render_error;
pub mod screenshot; pub mod screenshot;
pub mod scroll;
pub mod session; pub mod session;
pub mod snapshot; pub mod snapshot;
pub mod status; pub mod status;
@@ -55,6 +56,7 @@ use crate::cli::network::NetworkArgs;
use crate::cli::observe::ObserveArgs; use crate::cli::observe::ObserveArgs;
use crate::cli::record::RecordCmd; use crate::cli::record::RecordCmd;
use crate::cli::screenshot::ScreenshotArgs; use crate::cli::screenshot::ScreenshotArgs;
use crate::cli::scroll::ScrollToArgs;
use crate::cli::session::SessionCmd; use crate::cli::session::SessionCmd;
use crate::cli::snapshot::SnapshotArgs; use crate::cli::snapshot::SnapshotArgs;
use crate::cli::tab::TabCmd; use crate::cli::tab::TabCmd;
@@ -175,6 +177,10 @@ pub enum Command {
/// Hover a snapshot ref or CSS selector. /// Hover a snapshot ref or CSS selector.
Hover(HoverArgs), Hover(HoverArgs),
/// Scroll a snapshot ref or CSS selector into the visible viewport.
#[command(name = "scroll-to")]
ScrollTo(ScrollToArgs),
/// Focus a snapshot ref or CSS selector. /// Focus a snapshot ref or CSS selector.
Focus(FocusArgs), Focus(FocusArgs),
+100
View File
@@ -0,0 +1,100 @@
//! `bsk scroll-to` — bring a target element into the visible viewport.
use std::time::Duration;
use anyhow::Context;
use bsk_protocol::Method;
use bsk_protocol::tools::{ScrollToParams, ScrollToResult};
use clap::Args;
use crate::cli::dialogs::print_dialog_summaries;
use crate::cli::ensure_daemon::ensure_daemon;
use crate::cli::error::{CliError, Format};
use crate::cli::interaction::split_target;
use crate::cli::navigate::parse_timeout_ms;
#[derive(Debug, Clone, Args)]
pub struct ScrollToArgs {
/// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used.
pub target: Option<String>,
#[arg(long = "ref")]
pub ref_: Option<String>,
#[arg(long = "selector")]
pub selector: Option<String>,
#[arg(long)]
pub session: String,
#[arg(long = "tab-id")]
pub tab_id: Option<i64>,
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
pub timeout: u32,
}
pub fn dispatch(args: ScrollToArgs, format: Format) -> Result<(), CliError> {
let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?;
let info = ensure_daemon().context("ensure daemon is running")?;
let params = ScrollToParams {
session_id: args.session,
ref_,
selector,
tab_id: args.tab_id,
timeout_ms: Some(args.timeout),
};
let reply: ScrollToResult = crate::cli::business_rpc::call(
info.sock_path,
"scroll-to-1",
Method::ToolScrollTo,
Some(params),
ipc_timeout(args.timeout),
)?;
match format {
Format::Json => println!(
"{}",
serde_json::to_string_pretty(&reply)
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
),
Format::Human => {
let target =
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref());
println!(
"scroll-to ok tab={} target={target} bounds=({}, {}, {}, {})",
reply.tab_id, reply.x, reply.y, reply.width, reply.height
);
print_dialog_summaries(&reply.dialogs);
}
}
Ok(())
}
fn format_used_target(used_ref: Option<&str>, used_selector: Option<&str>) -> String {
used_ref
.map(|r| format!("@{r}"))
.or_else(|| used_selector.map(str::to_string))
.unwrap_or_else(|| "?".into())
}
fn ipc_timeout(timeout_ms: u32) -> Duration {
Duration::from_millis(u64::from(timeout_ms))
.checked_add(Duration::from_secs(15))
.unwrap_or(Duration::from_secs(u64::from(timeout_ms / 1_000) + 15))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn split_target_detects_refs_and_selectors() {
let (ref_, selector) = split_target(Some("@e3".into()), None, None).unwrap();
assert_eq!(ref_.as_deref(), Some("@e3"));
assert!(selector.is_none());
let (ref_, selector) = split_target(Some("#target".into()), None, None).unwrap();
assert!(ref_.is_none());
assert_eq!(selector.as_deref(), Some("#target"));
}
}
+1
View File
@@ -253,6 +253,7 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
| Method::ToolReload | Method::ToolReload
| Method::ToolClick | Method::ToolClick
| Method::ToolHover | Method::ToolHover
| Method::ToolScrollTo
| Method::ToolFocus | Method::ToolFocus
| Method::ToolBlur | Method::ToolBlur
| Method::ToolFill | Method::ToolFill
+1
View File
@@ -94,6 +94,7 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
Command::Reload(args) => cli::navigate::dispatch_reload(args, format), Command::Reload(args) => cli::navigate::dispatch_reload(args, format),
Command::Click(args) => cli::interaction::dispatch_click(args, format), Command::Click(args) => cli::interaction::dispatch_click(args, format),
Command::Hover(args) => cli::interaction::dispatch_hover(args, format), Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
Command::ScrollTo(args) => cli::scroll::dispatch(args, format),
Command::Focus(args) => cli::interaction::dispatch_focus(args, format), Command::Focus(args) => cli::interaction::dispatch_focus(args, format),
Command::Blur(args) => cli::interaction::dispatch_blur(args, format), Command::Blur(args) => cli::interaction::dispatch_blur(args, format),
Command::Fill(args) => cli::interaction::dispatch_fill(args, format), Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
+41
View File
@@ -255,6 +255,47 @@ fn parses_hover_with_settle() {
assert_eq!(args.settle, 300); assert_eq!(args.settle, 300);
} }
#[test]
fn parses_scroll_to_target() {
let cli = parse(&["bsk", "scroll-to", "@e2", "--session", "s1"]);
let Command::ScrollTo(args) = cli.command else {
panic!("expected scroll-to command");
};
assert_eq!(args.target.as_deref(), Some("@e2"));
}
#[test]
fn parses_scroll_to_explicit_target_tab_and_timeout() {
for flag in ["--ref", "--selector"] {
let cli = parse(&[
"bsk",
"scroll-to",
flag,
"e3",
"--session",
"s1",
"--tab-id",
"42",
"--timeout",
"5s",
]);
let Command::ScrollTo(args) = cli.command else {
panic!("expected scroll-to command");
};
assert_eq!(args.tab_id, Some(42));
assert_eq!(args.timeout, 5_000);
assert_eq!(
if flag == "--ref" {
args.ref_
} else {
args.selector
}
.as_deref(),
Some("e3")
);
}
}
#[test] #[test]
fn parses_focus_target() { fn parses_focus_target() {
let cli = parse(&["bsk", "focus", "@e2", "--session", "s1"]); let cli = parse(&["bsk", "focus", "@e2", "--session", "s1"]);
+23 -13
View File
@@ -293,6 +293,15 @@ async fn session_user_interrupt_event_cancels_inflight_with_user_aborted() {
/// catch a tool dispatched arbitrarily later. /// catch a tool dispatched arbitrarily later.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)] #[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle() { async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle() {
assert_idle_interrupt_rejects(Method::ToolClick).await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn user_interrupt_rejects_scroll_to_before_forwarding() {
assert_idle_interrupt_rejects(Method::ToolScrollTo).await;
}
async fn assert_idle_interrupt_rejects(method: Method) {
let (handle, sock) = spawn_daemon().await; let (handle, sock) = spawn_daemon().await;
let mut ws = connect_ext(handle.ws_addr()).await; let mut ws = connect_ext(handle.ws_addr()).await;
let _ = handshake_as_ext(&mut ws).await; let _ = handshake_as_ext(&mut ws).await;
@@ -301,10 +310,11 @@ async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle()
// Fake extension: answer tool.session_start, observe other // Fake extension: answer tool.session_start, observe other
// requests but never reply. We expect the daemon to reject the // requests but never reply. We expect the daemon to reject the
// tool call BEFORE it ever forwards a tool.click frame. // tool call BEFORE it ever forwards a browser-input frame.
let ws_sink_for_responder = Arc::clone(&ws_sink); let ws_sink_for_responder = Arc::clone(&ws_sink);
let observed_click_count = Arc::new(std::sync::atomic::AtomicUsize::new(0)); let observed_call_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let observed_click_count_clone = Arc::clone(&observed_click_count); let observed_call_count_clone = Arc::clone(&observed_call_count);
let forwarded_method = method.clone();
let responder = tokio::spawn(async move { let responder = tokio::spawn(async move {
let mut ws_stream = ws_stream; let mut ws_stream = ws_stream;
while let Some(Ok(msg)) = ws_stream.next().await { while let Some(Ok(msg)) = ws_stream.next().await {
@@ -329,8 +339,8 @@ async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle()
g.send(Message::Text(serde_json::to_string(&reply).unwrap())) g.send(Message::Text(serde_json::to_string(&reply).unwrap()))
.await .await
.unwrap(); .unwrap();
} else if req.method == Method::ToolClick { } else if req.method == forwarded_method {
observed_click_count_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst); observed_call_count_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
// Deliberately do NOT reply — daemon should reject before // Deliberately do NOT reply — daemon should reject before
// ever forwarding to us. // ever forwarding to us.
let _ = req; let _ = req;
@@ -377,12 +387,12 @@ async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle()
let state = handle.state(); let state = handle.state();
wait_for_session_interrupt_pending(&state, &start.session_id).await; wait_for_session_interrupt_pending(&state, &start.session_id).await;
// The CLI now issues a tool.click (mutating). The daemon must // The CLI now issues a browser-input (mutating). The daemon must
// reject it WITHOUT forwarding to the extension. // reject it WITHOUT forwarding to the extension.
let click_outcome = ipc let outcome = ipc
.call::<_, serde_json::Value>( .call::<_, serde_json::Value>(
"click-after-interrupt", "input-after-interrupt",
Method::ToolClick, method,
Some(json!({ Some(json!({
"session_id": start.session_id, "session_id": start.session_id,
"ref": "fake-ref-1", "ref": "fake-ref-1",
@@ -391,7 +401,7 @@ async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle()
) )
.await .await
.unwrap(); .unwrap();
let err = click_outcome.expect_err("tool.click must be rejected"); let err = outcome.expect_err("browser-input must be rejected");
assert_eq!( assert_eq!(
err.code, err.code,
ErrorCode::UserAborted, ErrorCode::UserAborted,
@@ -399,11 +409,11 @@ async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle()
err err
); );
// Crucially: the extension never saw a tool.click frame. // Crucially: the extension never saw a browser-input frame.
assert_eq!( assert_eq!(
observed_click_count.load(std::sync::atomic::Ordering::SeqCst), observed_call_count.load(std::sync::atomic::Ordering::SeqCst),
0, 0,
"tool.click must NOT have been forwarded to the extension" "browser-input must NOT have been forwarded to the extension"
); );
responder.abort(); responder.abort();
+46
View File
@@ -18,6 +18,7 @@ use bsk_protocol::tools::{
PressResult, ReloadParams, ReloadResult, SelectParams, SelectResult, SessionStartParams, PressResult, ReloadParams, ReloadResult, SelectParams, SelectResult, SessionStartParams,
SessionStartResult, WaitUntil, SessionStartResult, WaitUntil,
}; };
use bsk_protocol::tools::{ScrollToParams, ScrollToResult};
use bsk_protocol::{ use bsk_protocol::{
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError, BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
}; };
@@ -468,6 +469,51 @@ async fn blur_round_trips_previous_focus_state() {
handle.shutdown().await; handle.shutdown().await;
} }
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn scroll_to_round_trips_visible_bounds() {
let (handle, sock) = spawn_daemon().await;
let mut ws = connect_ext(handle.ws_addr()).await;
let _ = do_handshake(&mut ws).await;
run_extension(ws, |req| {
assert_eq!(req.method, Method::ToolScrollTo);
let params = req.params.clone().unwrap();
let parsed: ScrollToParams = serde_json::from_value(params).unwrap();
assert_eq!(parsed.ref_.as_deref(), Some("@e3"));
ResponseBody::Ok(
serde_json::to_value(ScrollToResult {
tab_id: 9,
used_ref: Some("e3".into()),
used_selector: None,
x: 10.0,
y: 20.0,
width: 100.0,
height: 40.0,
dialogs: vec![],
})
.unwrap(),
)
});
let session_id = ipc_session_start(&sock).await;
let result: ScrollToResult = ipc_tool_call(
&sock,
Method::ToolScrollTo,
ScrollToParams {
session_id,
ref_: Some("@e3".into()),
selector: None,
tab_id: None,
timeout_ms: Some(5_000),
},
)
.await
.expect("scroll-to ok");
assert_eq!(result.used_ref.as_deref(), Some("e3"));
assert_eq!(result.width, 100.0);
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn fill_round_trips_clear_before_default() { async fn fill_round_trips_clear_before_default() {
let (handle, sock) = spawn_daemon().await; let (handle, sock) = spawn_daemon().await;
@@ -0,0 +1,42 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "ScrollToParams",
"type": "object",
"required": [
"session_id"
],
"properties": {
"ref": {
"description": "Optional `@e<N>` ref allocated by the last observation. Mutually exclusive with `selector`.",
"type": [
"string",
"null"
]
},
"selector": {
"type": [
"string",
"null"
]
},
"session_id": {
"type": "string"
},
"tab_id": {
"description": "Target tab. Defaults to the Agent Window's active tab.",
"type": [
"integer",
"null"
],
"format": "int64"
},
"timeout_ms": {
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 1.0
}
}
}
@@ -0,0 +1,123 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "ScrollToResult",
"type": "object",
"required": [
"height",
"tab_id",
"width",
"x",
"y"
],
"properties": {
"dialogs": {
"type": "array",
"items": {
"$ref": "#/definitions/JavaScriptDialogInfo"
}
},
"height": {
"type": "number",
"format": "double"
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"used_ref": {
"type": [
"string",
"null"
]
},
"used_selector": {
"type": [
"string",
"null"
]
},
"width": {
"type": "number",
"format": "double"
},
"x": {
"description": "Bounding rectangle of the element's visible border-box portion, in top-level viewport CSS pixels, after ancestor and viewport clipping. Partial visibility is sufficient. This is not an occlusion or hit test.",
"type": "number",
"format": "double"
},
"y": {
"type": "number",
"format": "double"
}
},
"definitions": {
"JavaScriptDialogHandledAction": {
"description": "How the extension resolved the dialog so CDP could continue.",
"type": "string",
"enum": [
"accepted",
"dismissed"
]
},
"JavaScriptDialogInfo": {
"description": "One observed + handled JavaScript dialog during a tool call.",
"type": "object",
"required": [
"handled",
"message",
"sequence",
"tab_id",
"type"
],
"properties": {
"default_prompt": {
"type": [
"string",
"null"
]
},
"handled": {
"$ref": "#/definitions/JavaScriptDialogHandledAction"
},
"has_browser_handler": {
"type": [
"boolean",
"null"
]
},
"message": {
"type": "string"
},
"sequence": {
"description": "Monotonic per-tab sequence for ordering within a session.",
"type": "integer",
"format": "uint64",
"minimum": 0.0
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"type": {
"$ref": "#/definitions/JavaScriptDialogType"
},
"url": {
"type": [
"string",
"null"
]
}
}
},
"JavaScriptDialogType": {
"description": "Native JS dialog kind reported by CDP.",
"type": "string",
"enum": [
"alert",
"confirm",
"prompt",
"beforeunload"
]
}
}
}
@@ -79,6 +79,8 @@ fn main() {
dump!(ClickResult, "tool_click_result"); dump!(ClickResult, "tool_click_result");
dump!(HoverParams, "tool_hover_params"); dump!(HoverParams, "tool_hover_params");
dump!(HoverResult, "tool_hover_result"); dump!(HoverResult, "tool_hover_result");
dump!(ScrollToParams, "tool_scroll_to_params");
dump!(ScrollToResult, "tool_scroll_to_result");
dump!(FocusParams, "tool_focus_params"); dump!(FocusParams, "tool_focus_params");
dump!(FocusResult, "tool_focus_result"); dump!(FocusResult, "tool_focus_result");
dump!(BlurParams, "tool_blur_params"); dump!(BlurParams, "tool_blur_params");
+6
View File
@@ -72,6 +72,8 @@ pub enum Method {
ToolClick, ToolClick,
#[serde(rename = "tool.hover")] #[serde(rename = "tool.hover")]
ToolHover, ToolHover,
#[serde(rename = "tool.scroll_to")]
ToolScrollTo,
#[serde(rename = "tool.focus")] #[serde(rename = "tool.focus")]
ToolFocus, ToolFocus,
#[serde(rename = "tool.blur")] #[serde(rename = "tool.blur")]
@@ -172,6 +174,7 @@ impl Method {
| Method::ToolNavigateForward | Method::ToolNavigateForward
| Method::ToolReload | Method::ToolReload
| Method::ToolClick | Method::ToolClick
| Method::ToolScrollTo
| Method::ToolFocus | Method::ToolFocus
| Method::ToolBlur | Method::ToolBlur
| Method::ToolFill | Method::ToolFill
@@ -311,6 +314,7 @@ mod tests {
assert!(Method::ToolNavigateForward.is_mutating()); assert!(Method::ToolNavigateForward.is_mutating());
assert!(Method::ToolReload.is_mutating()); assert!(Method::ToolReload.is_mutating());
assert!(Method::ToolClick.is_mutating()); assert!(Method::ToolClick.is_mutating());
assert!(Method::ToolScrollTo.is_mutating());
assert!(Method::ToolFocus.is_mutating()); assert!(Method::ToolFocus.is_mutating());
assert!(Method::ToolBlur.is_mutating()); assert!(Method::ToolBlur.is_mutating());
assert!(Method::ToolFill.is_mutating()); assert!(Method::ToolFill.is_mutating());
@@ -357,6 +361,7 @@ mod tests {
assert_eq!(Method::ToolHover.effect(), MethodEffect::TransientInput); assert_eq!(Method::ToolHover.effect(), MethodEffect::TransientInput);
assert_eq!(Method::ToolObserve.effect(), MethodEffect::TransientInput); assert_eq!(Method::ToolObserve.effect(), MethodEffect::TransientInput);
assert_eq!(Method::ToolClick.effect(), MethodEffect::BrowserMutation); assert_eq!(Method::ToolClick.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::ToolScrollTo.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::ToolFocus.effect(), MethodEffect::BrowserMutation); assert_eq!(Method::ToolFocus.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::ToolBlur.effect(), MethodEffect::BrowserMutation); assert_eq!(Method::ToolBlur.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::Cancel.effect(), MethodEffect::ControlPlane); assert_eq!(Method::Cancel.effect(), MethodEffect::ControlPlane);
@@ -368,6 +373,7 @@ mod tests {
assert!(Method::ToolHover.requires_interrupt_gate()); assert!(Method::ToolHover.requires_interrupt_gate());
assert!(Method::ToolObserve.requires_interrupt_gate()); assert!(Method::ToolObserve.requires_interrupt_gate());
assert!(Method::ToolClick.requires_interrupt_gate()); assert!(Method::ToolClick.requires_interrupt_gate());
assert!(Method::ToolScrollTo.requires_interrupt_gate());
assert!(Method::ToolFocus.requires_interrupt_gate()); assert!(Method::ToolFocus.requires_interrupt_gate());
assert!(Method::ToolBlur.requires_interrupt_gate()); assert!(Method::ToolBlur.requires_interrupt_gate());
assert!(!Method::Cancel.requires_interrupt_gate()); assert!(!Method::Cancel.requires_interrupt_gate());
+2
View File
@@ -14,6 +14,7 @@ mod record_common;
mod record_v2; mod record_v2;
mod record_v3; mod record_v3;
pub mod script; pub mod script;
pub mod scroll;
pub mod session; pub mod session;
pub mod tabs; pub mod tabs;
pub mod waits; pub mod waits;
@@ -30,6 +31,7 @@ pub use network::*;
pub use observation::*; pub use observation::*;
pub use record::*; pub use record::*;
pub use script::*; pub use script::*;
pub use scroll::*;
pub use session::*; pub use session::*;
pub use tabs::*; pub use tabs::*;
pub use waits::*; pub use waits::*;
+67
View File
@@ -0,0 +1,67 @@
//! Element scrolling primitive (`tool.scroll_to`).
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use super::JavaScriptDialogInfo;
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct ScrollToParams {
pub session_id: String,
/// Optional `@e<N>` ref allocated by the last observation.
/// Mutually exclusive with `selector`.
#[serde(
rename = "ref",
alias = "ref_",
default,
skip_serializing_if = "Option::is_none"
)]
pub ref_: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selector: Option<String>,
/// Target tab. Defaults to the Agent Window's active tab.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tab_id: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 1))]
pub timeout_ms: Option<u32>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct ScrollToResult {
pub tab_id: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_ref: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_selector: Option<String>,
/// Bounding rectangle of the element's visible border-box portion, in
/// top-level viewport CSS pixels, after ancestor and viewport clipping.
/// Partial visibility is sufficient. This is not an occlusion or hit test.
pub x: f64,
pub y: f64,
pub width: f64,
pub height: f64,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dialogs: Vec<JavaScriptDialogInfo>,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn params_serialise_ref_field_name() {
let params = ScrollToParams {
session_id: "abcd".into(),
ref_: Some("@e3".into()),
selector: None,
tab_id: Some(42),
timeout_ms: Some(5_000),
};
let value = serde_json::to_value(&params).unwrap();
assert_eq!(value.get("ref").and_then(|v| v.as_str()), Some("@e3"));
assert!(value.get("ref_").is_none());
let round: ScrollToParams = serde_json::from_value(value).unwrap();
assert_eq!(round, params);
}
}
+5
View File
@@ -84,6 +84,11 @@ Shared Rust types + JSON Schema generation. TypeScript mirrors frame shapes in
4. Extension dispatcher validates sandbox rules, invokes CDP via `BrowserDriver`. 4. Extension dispatcher validates sandbox rules, invokes CDP via `BrowserDriver`.
5. Response travels CLI ← daemon ← extension; CLI prints result and exits. 5. Response travels CLI ← daemon ← extension; CLI prints result and exits.
The [scroll-to primitive reference](scroll-to.md) documents `tool.scroll_to`,
including its CLI/plugin mappings, visible-bounds contract and cancellation
behavior. It follows the same routing path and is classified as a browser
mutation for session queueing and user-interruption gating.
## Session and sandbox model ## Session and sandbox model
- **Session** = opaque ID (4 lowercase letters in v0.1) + dedicated **Agent Window** - **Session** = opaque ID (4 lowercase letters in v0.1) + dedicated **Agent Window**
+131
View File
@@ -0,0 +1,131 @@
# Scroll-to primitive
`scroll-to` brings an existing element and its containing frames into view, then
returns the visible portion's bounds. Use it to reveal a target before inspecting
the page or taking a screenshot. It accepts an element target rather than a pixel
distance or scroll direction.
| Entry point | Name |
| --- | --- |
| CLI | `bsk scroll-to` |
| Wire protocol | `tool.scroll_to` |
| DeepSeek Harness | `browser_interact` with `action: "scroll-to"` |
## CLI
Use a session you started and a fresh ref from an observation. The ids below are
examples; substitute the session, tab and ref returned by your own calls.
```sh
bsk scroll-to @e3 --session abcd --tab-id 42 --timeout 5s --json
bsk scroll-to --selector '#details' --session abcd
```
Supply exactly one target: a positional ref or selector, `--ref`, or `--selector`.
Refs accept `@e3` and `e3`. CSS selectors search only the main document; use refs
for elements in iframes (including out-of-process iframes) or shadow roots.
Refs belong to one session and tab and may become stale after page changes.
`--session` is required. `--tab-id` defaults to the Agent Window's active tab.
`--timeout` defaults to `30s` and must be positive; durations such as `5000ms` and
`5s` are accepted. The command operates on Agent Window tabs, including user tabs
explicitly borrowed into that window.
Example JSON output:
```json
{
"tab_id": 42,
"used_ref": "e3",
"x": 10,
"y": 20,
"width": 100,
"height": 80
}
```
`used_ref` or `used_selector` identifies the resolved target. The optional
`dialogs` array reports JavaScript dialogs handled during the call.
## Result contract
`x` and `y` are the upper-left corner, and `width` and `height` are the size of
the visible border-box portion's bounding rectangle. All four values use
**top-level viewport CSS pixels**, after scrolling and clipping by ancestor
containers and frame viewports. They are not document or screenshot pixel
coordinates.
Partial visibility is sufficient: a 400px-high element clipped by an 80px-high
scroll container can succeed with `height: 80`. Hidden or fully clipped targets
fail. The rectangle is not an occlusion or hit test: another element can cover
the target, and the rectangle's center is not a guaranteed click point. Layout
can change after the result; use fresh refs for subsequent interactions.
## Protocol and plugin
The CLI sends a normal request through the daemon to the extension:
```json
{
"id": "scroll-1",
"method": "tool.scroll_to",
"params": {
"session_id": "abcd",
"ref": "@e3",
"tab_id": 42,
"timeout_ms": 5000
}
}
```
Protocol callers must supply exactly one nonempty `ref` or `selector`.
`tab_id` is optional; `timeout_ms` is a positive integer and defaults to 30000.
These target constraints are enforced by the handler. See the generated
[parameter schema](../crates/bsk-protocol/schema/tool_scroll_to_params.json) and
[result schema](../crates/bsk-protocol/schema/tool_scroll_to_result.json) for field
types. The wire response wraps the result above in `{ "id": "scroll-1", "result": ... }`.
The plugin exposes the same action with its usual camelCase parameters:
```json
{
"action": "scroll-to",
"session": "abcd",
"target": "@e3",
"tabId": 42,
"timeoutMs": 5000
}
```
Call this through `browser_interact` using a plugin-owned session. Its result
contains `session`, `tabId`, `x`, `y`, `width` and `height` with the same bounds
semantics. Omitted `session` uses the plugin's current session.
## Errors and interruption
The wire response uses the normal `error.code`, `error.message` and optional
`error.data.reason` fields. Common cases are:
| Code | `data.reason` | Meaning |
| --- | --- | --- |
| `invalid_params` | — | Missing/conflicting target or invalid timeout/tab id |
| `not_found` | `ref_not_found` | Ref is unknown, expired or belongs to another tab |
| `not_found` | `selector_not_found` | Main-document selector matched no element |
| `permission_denied` | `agent_window_scope` | Target tab has not been borrowed into the Agent Window |
| `permission_denied` | `element_not_visible` | No visible area remains after scrolling |
| `cancelled` | — | The call was cancelled |
| `timeout` | — | The action deadline expired |
| `cdp_failed` | varies | Browser command, frame geometry or visibility measurement failed |
The protocol classifies scroll-to as a browser mutation, so it uses the existing
per-session queue and user-interruption gate. The extension releases any retained
hover state before dispatch. Cancellation and deadline checks surround each CDP
step, including frame lookup, scrolling and measurement. Once detected, they
prevent further scrolling and a successful response; remote-object cleanup still
runs. Already performed scrolling is not rolled back, so inspect the current
page before retrying an interrupted call.
The implementation is scoped to scroll-to: it reuses frame scrolling and
coordinate projection, with a separate visibility measurement that accounts for
ancestor clipping and hidden frame owners. Shared click, hover and screenshot
geometry is unchanged.
+4 -1
View File
@@ -54,10 +54,13 @@ the `bsk` CLI and browser extension separately when a release requires it.
| `browser_session` | `start`, `stop`, `list` | Manage plugin-owned Agent Window sessions. | | `browser_session` | `start`, `stop`, `list` | Manage plugin-owned Agent Window sessions. |
| `browser_page` | `navigate`, `back`, `forward`, `reload`, `wait` | Navigate the active tab and wait for page lifecycle events. | | `browser_page` | `navigate`, `back`, `forward`, `reload`, `wait` | Navigate the active tab and wait for page lifecycle events. |
| `browser_inspect` | `observe`, `snapshot`, `html`, `screenshot`, `console`, `network` | Read semantic or diagnostic page state and capture screenshots. | | `browser_inspect` | `observe`, `snapshot`, `html`, `screenshot`, `console`, `network` | Read semantic or diagnostic page state and capture screenshots. |
| `browser_interact` | `click`, `hover`, `focus`, `blur`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. | | `browser_interact` | `click`, `hover`, `scroll-to`, `focus`, `blur`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. |
| `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. | | `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. |
| `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. | | `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. |
For `browser_interact` with `action: "scroll-to"`, see the
[scroll-to reference](../../docs/scroll-to.md) for parameters, visible bounds and errors.
Arbitrary page-script evaluation and interaction recording are not supported. Arbitrary page-script evaluation and interaction recording are not supported.
## Multi-session model ## Multi-session model
@@ -51,7 +51,12 @@ Use `browser_inspect` action `observe` as the primary semantic page view. It ret
text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may
also become stale after large DOM changes, so observe again before the next interaction. also become stale after large DOM changes, so observe again before the next interaction.
Use `browser_interact` for click, hover, focus, blur, fill, select, and key actions. Focus and blur Use `browser_interact` with `action=scroll-to` to bring an element into view. Its bounds are the
visible border-box portion in top-level viewport CSS pixels after ancestor clipping; partial
visibility is enough. Hidden or fully clipped targets fail. This does not test occlusion by other
elements. Use fresh element refs for iframe/shadow-root targets; selectors search the main document.
Use `browser_interact` for click, hover, scroll-to, focus, blur, fill, select, and key actions. Focus and blur
explicitly enter or leave UI states triggered by focus changes. An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are explicitly enter or leave UI states triggered by focus changes. An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are
labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref. labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref.
Do not click the trigger itself unless the user wants the trigger's action. Do not click the trigger itself unless the user wants the trigger's action.
@@ -167,12 +167,13 @@ const BROWSER_TOOL_SPECS: BrowserToolSpec[] = [
{ {
name: "browser_interact", name: "browser_interact",
description: description:
"Interact with an element in the active Agent Window tab. Actions: click, hover, focus, blur, fill, select, " + "Interact with an element in the active Agent Window tab. Actions: click, hover, scroll-to, focus, blur, fill, select, " +
"press. click/hover/focus/blur/fill/select require target; fill also requires value; select requires " + "press. click/hover/scroll-to/focus/blur/fill/select require target; fill also requires value; select requires " +
"values; press requires key and may optionally focus target first.", "values; press requires key and may optionally focus target first.",
actions: { actions: {
click: "interact.click", click: "interact.click",
hover: "interact.hover", hover: "interact.hover",
"scroll-to": "interact.scroll-to",
focus: "interact.focus", focus: "interact.focus",
blur: "interact.blur", blur: "interact.blur",
fill: "interact.fill", fill: "interact.fill",
@@ -13,7 +13,7 @@ import type { ToolDeps } from "./tools";
const MODIFIERS = ["alt", "ctrl", "meta", "shift"] as const; const MODIFIERS = ["alt", "ctrl", "meta", "shift"] as const;
/** Add hover, focus, blur and select without bypassing session ownership or observation. */ /** Add interaction primitives without bypassing session ownership or observation. */
export function registerPhaseOneInteractionTools( export function registerPhaseOneInteractionTools(
deps: ToolDeps, deps: ToolDeps,
register: ToolRegistrar, register: ToolRegistrar,
@@ -100,6 +100,89 @@ export function registerPhaseOneInteractionTools(
}), }),
); );
register(
defineTool({
name: "interact.scroll-to",
description:
"Scroll an element and its frame owners into view. Returns the visible portion's " +
"bounds in top-level viewport CSS pixels; fails if no visible area remains. " +
"Use a fresh ref for iframe targets; selectors search the main document.",
parameters: {
target: {
type: "string",
required: true,
description: "Element ref (@e3 / e3) or main-document CSS selector to scroll into view.",
},
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
timeoutMs: TIMEOUT_MS_PARAM,
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
x: { type: "number", required: true },
y: { type: "number", required: true },
width: { type: "number", required: true },
height: { type: "number", required: true },
},
},
render: (_args, value) => [
{
type: "text",
text:
`[session ${value.session}] scrolled into view on tab ${value.tabId}: ` +
`(${value.x}, ${value.y}, ${value.width}, ${value.height})`,
},
],
},
async execute(args, exec) {
requireNonEmpty(args.target, "target");
requirePositive(args.timeoutMs, "timeoutMs");
const sessionId = registry.resolve(args.session, "browser_interact(action=scroll-to)");
const cmdArgs = ["scroll-to", "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`);
appendTarget(cmdArgs, args.target);
const reply = (await runtime.run(
exec,
cmdArgs,
"scroll-to",
sessionId,
runnerTimeout(deps, args.timeoutMs),
)) as {
tab_id: number;
x: number;
y: number;
width: number;
height: number;
};
return {
session: sessionId,
tabId: reply.tab_id,
x: reply.x,
y: reply.y,
width: reply.width,
height: reply.height,
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"scroll-to",
args.target,
"--session",
args.session ?? "(current)",
]),
description: "Scroll an element into view",
}),
presentResult: runtime.presentTerminalResult,
}),
);
for (const action of ["focus", "blur"] as const) { for (const action of ["focus", "blur"] as const) {
register( register(
defineTool({ defineTool({
@@ -102,6 +102,7 @@ const ACTION_ROUTES: Record<string, readonly [string, string]> = {
"inspect.network": ["browser_inspect", "network"], "inspect.network": ["browser_inspect", "network"],
"interact.click": ["browser_interact", "click"], "interact.click": ["browser_interact", "click"],
"interact.hover": ["browser_interact", "hover"], "interact.hover": ["browser_interact", "hover"],
"interact.scroll-to": ["browser_interact", "scroll-to"],
"interact.focus": ["browser_interact", "focus"], "interact.focus": ["browser_interact", "focus"],
"interact.blur": ["browser_interact", "blur"], "interact.blur": ["browser_interact", "blur"],
"interact.fill": ["browser_interact", "fill"], "interact.fill": ["browser_interact", "fill"],
@@ -202,7 +203,7 @@ const EXPECTED_ACTIONS = {
browser_session: ["start", "stop", "list"], browser_session: ["start", "stop", "list"],
browser_page: ["navigate", "back", "forward", "reload", "wait"], browser_page: ["navigate", "back", "forward", "reload", "wait"],
browser_inspect: ["observe", "snapshot", "html", "screenshot", "console", "network"], browser_inspect: ["observe", "snapshot", "html", "screenshot", "console", "network"],
browser_interact: ["click", "hover", "focus", "blur", "fill", "select", "press"], browser_interact: ["click", "hover", "scroll-to", "focus", "blur", "fill", "select", "press"],
browser_tabs: ["list", "create", "select", "close", "borrow", "return"], browser_tabs: ["list", "create", "select", "close", "borrow", "return"],
browser_assist: ["resize", "emulate", "request-help"], browser_assist: ["resize", "emulate", "request-help"],
} as const; } as const;
@@ -540,6 +541,7 @@ describe("interaction tools", () => {
it.each([ it.each([
"focus", "focus",
"blur", "blur",
"scroll-to",
] as const)("interact.%s validates arguments and session ownership before running", async (action) => { ] as const)("interact.%s validates arguments and session ownership before running", async (action) => {
const { tools, calls } = setup({ "session start": START_REPLY("s1") }); const { tools, calls } = setup({ "session start": START_REPLY("s1") });
await startSession(tools); await startSession(tools);
@@ -555,6 +557,30 @@ describe("interaction tools", () => {
expect(calls).toHaveLength(1); expect(calls).toHaveLength(1);
}); });
it.each(["@e3", "#target"])("interact.scroll-to maps %s, bounds and timeout", async (target) => {
const { tools, calls } = setup({
"session start": START_REPLY("s1"),
"scroll-to": { tab_id: 7, x: 10, y: 20, width: 100, height: 80 },
});
await startSession(tools);
expect(
await tools
.get("interact.scroll-to")
?.execute({ target, tabId: 7, timeoutMs: 150_000 }, makeExec()),
).toEqual({ session: "s1", tabId: 7, x: 10, y: 20, width: 100, height: 80 });
expect(calls[1].args).toEqual([
"scroll-to",
"--session",
"s1",
"--tab-id",
"7",
"--timeout",
"150000ms",
target,
]);
expect(calls[1].options.timeoutMs).toBe(165_000);
});
it("interact.fill passes value and --no-clear", async () => { it("interact.fill passes value and --no-clear", async () => {
const { tools, calls } = setup(responses); const { tools, calls } = setup(responses);
await startSession(tools); await startSession(tools);
+9 -2
View File
@@ -55,10 +55,17 @@ Use this default loop:
```text ```text
bsk navigate <url> --session <id> bsk navigate <url> --session <id>
bsk observe --session <id> bsk observe --session <id>
bsk click|hover|focus|blur|fill|select|press ... --session <id> bsk click|hover|scroll-to|focus|blur|fill|select|press ... --session <id>
bsk observe --session <id> # after navigation or a meaningful DOM change bsk observe --session <id> # after navigation or a meaningful DOM change
``` ```
`bsk scroll-to <ref-or-selector> --session <id>` scrolls an element and its frame owners into view.
Use a fresh element ref for iframe/shadow-root targets; CSS selectors search the main document.
The result is the visible border-box portion's bounds in top-level viewport CSS pixels after
ancestor clipping. Partial visibility is enough; hidden or fully clipped targets fail with
`permission_denied` and `data.reason=element_not_visible`. This does not test occlusion by other elements.
For a specific tab or deadline: `bsk scroll-to @e3 --session <id> --tab-id 42 --timeout 5s`.
`bsk focus <ref>` explicitly focuses a target; `bsk blur <ref>` removes focus and reports whether `bsk focus <ref>` explicitly focuses a target; `bsk blur <ref>` removes focus and reports whether
it was focused. Use these for UI states triggered by focus changes. it was focused. Use these for UI states triggered by focus changes.
@@ -117,7 +124,7 @@ This list of names is complete. Never invent a command outside it; read
session start|stop|list browsers status doctor update logs session start|stop|list browsers status doctor update logs
navigate navigate-back navigate-forward reload wait-for-navigation wait-ms navigate navigate-back navigate-forward reload wait-for-navigation wait-ms
observe snapshot get-html screenshot console network observe snapshot get-html screenshot console network
click hover focus blur fill select press evaluate click hover scroll-to focus blur fill select press evaluate
tab list|create|close|select|borrow|return window resize emulate tab list|create|close|select|borrow|return window resize emulate
upload download request-help record start|stop upload download request-help record start|stop
``` ```