Compare commits

..
Author SHA1 Message Date
Zhang GH ba6a14de7c Merge pull request #127 from NianJiuZst/codex/scroll-to-element-primitive
feat(interaction): add scroll-to-element primitive
2026-09-10 11:04:06 +08:00
drakezhang fd0e9de61b docs: document scroll-to primitive contract 2026-09-10 10:15:57 +08:00
drakezhang 77e3f5e1f3 feat(dsh-plugin): expose scroll-to action 2026-09-10 00:05:52 +08:00
drakezhang 61c97c9cb5 fix(interaction): honor scroll cancellation and visible bounds 2026-09-10 00:05:48 +08:00
drakezhang 29355e041d merge: integrate main into scroll-to PR 2026-09-09 23:47:27 +08:00
Zhang GH cff1ea62da Merge pull request #126 from NianJiuZst/codex/focus-blur-primitive
feat(interaction): add explicit focus and blur primitives
2026-09-09 23:23:50 +08:00
drakezhang 20e8770006 feat(dsh-plugin): expose focus and blur actions 2026-09-09 22:47:47 +08:00
drakezhang ac9e7948bf fix(interaction): verify focus changes and honor cancellation 2026-09-09 22:47:42 +08:00
drakezhang aadc1376ff merge: integrate main into focus and blur PR 2026-09-09 22:33:48 +08:00
Zhang GH 0fe36bc0b1 Merge pull request #206 from Tencent/fix/oopif-scrollbar-geometry
fix(geometry): preserve OOPIF scale with occupied scrollbars
2026-09-09 17:45:21 +08:00
drakezhang a62a28d8cd fix(geometry): preserve OOPIF scale with occupied scrollbars 2026-09-09 16:58:25 +08:00
Hannah Liu 896c7944d8 Merge pull request #203 from Tencent/fix/capture-document-consistency
fix(observation): validate DOM identity across capture phases | 校验跨采集阶段的 DOM 身份一致性
2026-09-09 16:26:24 +08:00
Hannah Liu 7e5f3bae26 Merge pull request #200 from Tencent/fix/observation-document-consistency
refactor(observation):  unify cross-frame capture coordination and data ownership | 统一跨 frame 采集协调与数据归属
2026-09-09 16:25:52 +08:00
Hannah Liu af4c2fa229 Merge pull request #198 from Tencent/refactor/document-capture-facts
refactor(observation): decouple DOM processing from capture orchestration | 解耦 DOM 处理与采集流程
2026-09-09 16:24:42 +08:00
Hannah Liu ee10da6fe9 Merge pull request #195 from Tencent/fix/frame-geometry-contracts
fix(geometry): unify cross-frame coordinates and screenshot conversion | 统一跨 frame 坐标与截图转换
2026-09-09 16:24:06 +08:00
NianJiuZst 7a10be21cb feat(interaction): add scroll-to-element primitive 2026-08-21 20:56:24 +08:00
NianJiuZst 0055e90f91 feat(interaction): add explicit focus and blur primitives 2026-08-21 19:14:40 +08:00
56 changed files with 4013 additions and 104 deletions
+7
View File
@@ -5,6 +5,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/).
Starting from 0.2.0, CLI / Extension / DSH Plugin share the same version number.
## [Unreleased]
### Added
- [Scroll-to element primitive](docs/scroll-to.md) across CLI, Extension and DSH Plugin,
with ancestor-clipped visible bounds, iframe support and cooperative cancellation
## [0.2.1] - 2026-09-09
### Changed
+3
View File
@@ -207,6 +207,9 @@ through the [plugin](#deepseek-harness-plugin): the agent calls injected
## For Developers
The [scroll-to primitive reference](docs/scroll-to.md) covers its CLI, protocol
and plugin entry points, visible bounds and interruption behavior.
The repository is a Cargo + pnpm workspace:
- `crates/bsk-cli` — `bsk` CLI and local daemon
+2
View File
@@ -177,6 +177,8 @@ Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;
## 面向开发者
[scroll-to 原语说明](docs/scroll-to.md)介绍 CLI、协议和插件入口,以及可见区域、错误和中断语义。
本仓库是 Cargo + pnpm workspace:
- `crates/bsk-cli` — `bsk` CLI 与本地 daemon
@@ -599,6 +599,79 @@ describe("ToolDispatcher", () => {
expect(onSessionsChanged).toHaveBeenCalledTimes(2);
});
it.each([
"focus",
"blur",
"scroll_to",
] as const)("routes %s with hover cleanup and cooperative cancellation", async (action) => {
const tab = { id: 7, windowId: 4242, active: true };
vi.stubGlobal("chrome", {
tabs: {
get: vi.fn(async () => tab),
query: vi.fn(async () => [tab]),
sendMessage: vi.fn(async () => undefined),
},
});
const sessions = new SessionManager({
agentWindow: {
create: async () => 4242,
remove: async () => {},
ensureActiveTab: async () => 7,
},
});
const ctx = await sessions.start("aa11");
ctx.refStore.set("e1", 12, { tabId: 7 });
const { transport, sent, deliver } = fakeTransport();
const onBrowserControlResumed = vi.fn();
let resolveNode: ((value: object) => void) | undefined;
const cdp = {
send: vi.fn(async (_tabId: number, method: string) => {
if (method === "DOM.resolveNode")
return new Promise((resolve) => {
resolveNode = resolve;
});
if (method === "Runtime.callFunctionOn") return { result: { value: true } };
return {};
}),
} as unknown as TestDispatcherCdp;
const dispatcher = new ToolDispatcher({ transport, sessions, cdp, onBrowserControlResumed });
const hover = dispatcher as unknown as {
rememberHover: (sessionId: string, result: object) => void;
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
};
hover.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
await hover.setHoverBypass("aa11", 7, true);
dispatcher.start();
deliver(makeRequest(`tool.${action}`, { session_id: "aa11", ref: "e1" }));
await vi.waitFor(() => expect(resolveNode).toBeDefined());
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
deliver({ id: "cancel-focus", method: "cancel", params: { rpc_id: "r-1" } });
resolveNode!({ object: { objectId: "focus-target" } });
await vi.waitFor(() => expect(sent).toHaveLength(2));
expect(sent).toContainEqual({ id: "cancel-focus", result: { cancelled: true } });
expect(sent).toContainEqual({
id: "r-1",
error: expect.objectContaining({ code: "cancelled" }),
});
expect(cdp.send).not.toHaveBeenCalledWith(7, "DOM.focus", expect.anything());
expect(cdp.send).not.toHaveBeenCalledWith(7, "Runtime.callFunctionOn", expect.anything());
if (action === "scroll_to") {
expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObjectGroup", {
objectGroup: expect.any(String),
});
} else {
expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObject", {
objectId: "focus-target",
});
}
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
7,
expect.objectContaining({ enabled: false }),
);
expect(dispatcher.inflightAbortControllers.size).toBe(0);
dispatcher.stop();
});
it("invokes onBrowserControlResumed for browser-control tools but not passive reads", async () => {
vi.stubGlobal("chrome", {
tabs: {
@@ -436,6 +436,8 @@ describe("file transfer tools", () => {
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 210, 20, 210, 120, 10, 120]] };
}
if (method === "Runtime.evaluate")
return { result: { value: { width: 400, height: 300 } } };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 400, clientHeight: 300 } };
}
@@ -0,0 +1,166 @@
// @vitest-environment node
// Opt in with BSK_FOCUS_CHROME=/path/to/chrome; each test owns its browser/profile.
import { describe, expect, it } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import { handleBlur, handleFocus } from "../interaction";
import type { CdpRunner } from "../shared";
type Send = <T = Record<string, unknown>>(
method: string,
params?: object,
sessionId?: string,
) => Promise<T>;
async function withFocusBrowser(
run: (h: {
evaluate: (expression: string) => Promise<unknown>;
ref: () => Promise<void>;
focus: (signal?: AbortSignal) => ReturnType<typeof handleFocus>;
blur: (signal?: AbortSignal) => ReturnType<typeof handleBlur>;
calls: string[];
afterCommand: { current: (method: string) => void };
}) => Promise<void>,
) {
// Reuse the existing isolated Chrome launcher; no new browser dependency.
const { withChrome } = await import(
new URL(
"../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs",
import.meta.url,
).href
);
await withChrome(
{ executable: process.env.BSK_FOCUS_CHROME, deviceScale: 1, zoom: 1 },
async (send: Send) => {
const { targetId } = await send<{ targetId: string }>("Target.createTarget", {
url: "about:blank",
});
const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", {
targetId,
flatten: true,
});
await send("Page.bringToFront", {}, sessionId);
const evaluate = async (expression: string) => {
const reply = await send<{ result: { value: unknown }; exceptionDetails?: unknown }>(
"Runtime.evaluate",
{ expression, returnByValue: true },
sessionId,
);
expect(reply.exceptionDetails).toBeUndefined();
return reply.result.value;
};
await evaluate(
`document.body.innerHTML = '<input id="target"><input id="other">'; window.probe = document.querySelector('#target');`,
);
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("aa11");
const calls: string[] = [];
const afterCommand = { current: (_method: string) => {} };
const cdp: CdpRunner = {
send: async (_tabId, method, params) => {
calls.push(method);
const result = await send(method, params, sessionId);
afterCommand.current(method);
return result as never;
},
};
const deps = {
cdp,
tabsApi: {
get: async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab,
query: async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab],
},
};
await run({
evaluate,
calls,
afterCommand,
ref: async () => {
const { result } = await send<{ result: { objectId: string } }>(
"Runtime.evaluate",
{ expression: "window.probe" },
sessionId,
);
const { node } = await send<{ node: { backendNodeId: number } }>(
"DOM.describeNode",
{ objectId: result.objectId },
sessionId,
);
await send("Runtime.releaseObject", { objectId: result.objectId }, sessionId);
ctx.refStore.set("e1", node.backendNodeId, { tabId: 4 });
},
focus: (signal) =>
handleFocus(manager, { session_id: "aa11", ref: "e1" }, { ...deps, signal }),
blur: (signal) =>
handleBlur(manager, { session_id: "aa11", ref: "e1" }, { ...deps, signal }),
});
},
);
}
describe.skipIf(!process.env.BSK_FOCUS_CHROME)("real browser focus and blur", () => {
it.each(["light", "open", "closed"])("verifies focus and blur in %s DOM", async (mode) => {
await withFocusBrowser(async (h) => {
if (mode !== "light")
await h.evaluate(`(() => {
const host = document.createElement('div'); document.body.append(host);
const nested = document.createElement('div'); host.attachShadow({mode:'${mode}'}).append(nested);
const input = document.createElement('input'); nested.attachShadow({mode:'${mode}'}).append(input);
window.probe = input;
})()`);
await h.ref();
expect(await h.focus()).toMatchObject({ focused: true });
expect(await h.evaluate("window.probe.matches(':focus')")).toBe(true);
expect(await h.blur()).toMatchObject({ was_focused: true, focused: false });
expect(await h.evaluate("window.probe.matches(':focus')")).toBe(false);
expect(h.calls.filter((method) => method === "Runtime.releaseObject")).toHaveLength(2);
});
});
it("detects focus changes from event-handler microtasks", async () => {
await withFocusBrowser(async (h) => {
await h.ref();
await h.evaluate(
"window.probe.addEventListener('focus', () => queueMicrotask(() => document.querySelector('#other').focus()), {once:true})",
);
expect(await h.focus()).toMatchObject({ code: "cdp_failed" });
expect(await h.focus()).toMatchObject({ focused: true });
await h.evaluate(
"window.probe.addEventListener('blur', () => queueMicrotask(() => window.probe.focus()), {once:true})",
);
expect(await h.blur()).toMatchObject({ code: "cdp_failed" });
expect(await h.evaluate("window.probe.matches(':focus')")).toBe(true);
});
});
it("cancels before blur while still releasing the resolved object", async () => {
await withFocusBrowser(async (h) => {
await h.ref();
await h.focus();
const abort = new AbortController();
h.afterCommand.current = (method) => {
if (method === "DOM.resolveNode") abort.abort();
};
expect(await h.blur(abort.signal)).toMatchObject({ code: "cancelled" });
expect(await h.evaluate("window.probe.matches(':focus')")).toBe(true);
expect(h.calls.at(-1)).toBe("Runtime.releaseObject");
});
});
it("reports a page exception with its cause", async () => {
await withFocusBrowser(async (h) => {
await h.ref();
await h.evaluate("window.probe.blur = () => { throw new Error('page blur failed'); }");
expect(await h.blur()).toMatchObject({
code: "cdp_failed",
message: expect.stringContaining("page blur failed"),
});
expect(h.calls.at(-1)).toBe("Runtime.releaseObject");
});
});
});
@@ -0,0 +1,293 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import { SessionManager } from "@/session-manager/manager";
import { handleBlur, handleFocus } from "../interaction";
import type { CdpRunner } from "../shared";
interface ScriptParams {
functionDeclaration: string;
}
interface Call {
target: CdpTarget;
method: string;
}
async function setup(element?: HTMLElement, childSession = false) {
if (!element) {
document.body.innerHTML = '<input id="target"><input id="other">';
element = document.querySelector<HTMLInputElement>("#target")!;
}
const targetElement = element;
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("aa11");
ctx.refStore.set("e1", 12, {
tabId: 4,
...(childSession ? { frameId: "child", cdpSessionId: "child-session" } : {}),
});
const calls: Call[] = [];
const afterCommand = vi.fn<(call: Call) => void>();
const script = vi.fn(async (params: ScriptParams) => {
try {
const fn = new Function(`return (${params.functionDeclaration})`)();
return { result: { value: await fn.call(targetElement) } };
} catch (error) {
return { exceptionDetails: { text: "Uncaught", exception: { description: String(error) } } };
}
});
const focus = vi.fn(() => targetElement.focus());
const release = vi.fn(async () => ({}));
const send = async (target: CdpTarget, method: string, params?: object) => {
const call = { target, method };
calls.push(call);
try {
switch (method) {
case "DOM.getDocument":
return { root: { nodeId: 1 } };
case "DOM.querySelector":
return { nodeId: 2 };
case "DOM.describeNode":
return { node: { backendNodeId: 12 } };
case "DOM.scrollIntoViewIfNeeded":
return {};
case "DOM.resolveNode":
return { object: { objectId: "focus-target" } };
case "DOM.focus":
focus();
return {};
case "Runtime.callFunctionOn":
return await script(params as ScriptParams);
case "Runtime.releaseObject":
return await release();
default:
throw new Error(`unexpected ${method}`);
}
} finally {
afterCommand(call);
}
};
const cdp: CdpRunner = {
send: (tabId, method, params) => send({ tabId }, method, params) as never,
sendToTarget: send as CdpRunner["sendToTarget"],
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
...(childSession
? [
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 99,
target: { tabId: 4, sessionId: "child-session" },
},
]
: []),
],
})),
};
const tabsApi = {
get: vi.fn(async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
};
return {
element: targetElement,
calls,
afterCommand,
script,
focus,
release,
cdp,
tabsApi,
run: (
action: "focus" | "blur",
signal?: AbortSignal,
params: { ref?: string; selector?: string; tab_id?: number } = { ref: "e1" },
) =>
(action === "focus" ? handleFocus : handleBlur)(
manager,
{ session_id: "aa11", ...params },
{ cdp, tabsApi, signal },
),
};
}
afterEach(() => {
vi.restoreAllMocks();
document.body.innerHTML = "";
});
describe("focus and blur state verification", () => {
it.each(["open", "closed"] as const)("handles nested %s shadow roots", async (mode) => {
const host = document.createElement("div");
document.body.append(host);
const outer = host.attachShadow({ mode });
const innerHost = document.createElement("div");
outer.append(innerHost);
const inner = innerHost.attachShadow({ mode });
const element = document.createElement("input");
inner.append(element);
const h = await setup(element);
expect(await h.run("focus")).toMatchObject({ focused: true });
expect(inner.activeElement).toBe(element);
expect(await h.run("blur")).toMatchObject({ was_focused: true, focused: false });
expect(inner.activeElement).toBeNull();
expect(h.release).toHaveBeenCalledTimes(2);
});
it("keeps DOM focus usable when the document is in the background", async () => {
const h = await setup();
vi.spyOn(document, "hasFocus").mockReturnValue(false);
expect(await h.run("focus")).toMatchObject({ focused: true });
expect(await h.run("blur")).toMatchObject({ was_focused: true, focused: false });
});
it("blurring an unfocused target preserves the other element's focus", async () => {
const h = await setup();
const other = document.querySelector<HTMLInputElement>("#other")!;
other.focus();
expect(await h.run("blur")).toMatchObject({ was_focused: false, focused: false });
expect(document.activeElement).toBe(other);
});
it("detects focus redirected by a focus handler's microtask", async () => {
const h = await setup();
h.element.addEventListener("focus", () =>
queueMicrotask(() => document.querySelector<HTMLInputElement>("#other")!.focus()),
);
expect(await h.run("focus")).toMatchObject({
code: "cdp_failed",
message: "target element did not become focused",
});
expect(h.release).toHaveBeenCalledOnce();
});
it("does not report blur success when a microtask restores focus", async () => {
const h = await setup();
h.element.focus();
h.element.addEventListener("blur", () => queueMicrotask(() => h.element.focus()));
expect(await h.run("blur")).toMatchObject({
code: "cdp_failed",
message: "target element remained focused after blur()",
});
expect(document.activeElement).toBe(h.element);
expect(h.release).toHaveBeenCalledOnce();
});
it.each(["focus", "blur"] as const)("rejects a detached %s target", async (action) => {
const h = await setup();
h.element.remove();
expect(await h.run(action)).toMatchObject({ code: "cdp_failed" });
expect(h.release).toHaveBeenCalledOnce();
});
it("preserves a script exception's cause", async () => {
const h = await setup();
vi.spyOn(h.element, "blur").mockImplementation(() => {
throw new Error("page blur failed");
});
expect(await h.run("blur")).toMatchObject({
code: "cdp_failed",
message: expect.stringContaining("page blur failed"),
});
expect(h.release).toHaveBeenCalledOnce();
});
it.each(["focus", "blur"] as const)("rejects malformed %s script results", async (action) => {
const h = await setup();
h.script.mockResolvedValueOnce({ result: { value: null } });
expect(await h.run(action)).toMatchObject({
code: "cdp_failed",
message: expect.stringContaining("unexpected result"),
});
expect(h.release).toHaveBeenCalledOnce();
});
it.each([
"focus",
"blur",
] as const)("keeps %s, verification and cleanup in the OOPIF session", async (action) => {
const h = await setup(undefined, true);
if (action === "blur") h.element.focus();
expect(await h.run(action)).toMatchObject({ focused: action === "focus" });
const targetCalls = h.calls.filter((call) => call.method !== "DOM.scrollIntoViewIfNeeded");
expect(targetCalls.every((call) => call.target.sessionId === "child-session")).toBe(true);
expect(targetCalls.at(-1)?.method).toBe("Runtime.releaseObject");
});
it.each(["focus", "blur"] as const)("rejects %s on a user tab before CDP", async (action) => {
const h = await setup();
h.tabsApi.get.mockResolvedValue({ id: 9, windowId: 200, active: true } as chrome.tabs.Tab);
expect(await h.run(action, undefined, { ref: "e1", tab_id: 9 })).toMatchObject({
code: "permission_denied",
});
expect(h.calls).toEqual([]);
});
it("does not replace a successful result when navigation disposed the remote object", async () => {
const h = await setup();
h.release.mockRejectedValue(new Error("Cannot find context"));
expect(await h.run("focus")).toMatchObject({ focused: true });
});
});
describe("focus and blur cancellation", () => {
const boundaries = [
"DOM.getDocument",
"DOM.querySelector",
"DOM.describeNode",
"DOM.resolveNode",
"Runtime.callFunctionOn",
];
for (const action of ["focus", "blur"] as const) {
it.each(
action === "focus" ? [...boundaries, "DOM.scrollIntoViewIfNeeded", "DOM.focus"] : boundaries,
)(`${action} stops after cancellation during %s`, async (method) => {
const h = await setup();
const abort = new AbortController();
let cancelledAfter = 0;
h.afterCommand.mockImplementation((call) => {
if (call.method === method && !abort.signal.aborted) {
cancelledAfter = h.calls.length;
abort.abort();
}
});
expect(await h.run(action, abort.signal, { selector: "#target" })).toMatchObject({
code: "cancelled",
});
expect(abort.signal.aborted).toBe(true);
expect(
h.calls.slice(cancelledAfter).every((call) => call.method === "Runtime.releaseObject"),
).toBe(true);
if (h.calls.some((call) => call.method === "DOM.resolveNode"))
expect(h.release).toHaveBeenCalledOnce();
});
it(`${action} stops if cancellation arrives while resolving the tab`, async () => {
const h = await setup();
const abort = new AbortController();
h.tabsApi.query.mockImplementation(async () => {
abort.abort();
return [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab];
});
expect(await h.run(action, abort.signal)).toMatchObject({ code: "cancelled" });
expect(h.calls).toEqual([]);
});
}
it("does not scroll frame owners after cancellation in an OOPIF", async () => {
const h = await setup(undefined, true);
const abort = new AbortController();
h.afterCommand.mockImplementation((call) => {
if (call.method === "DOM.scrollIntoViewIfNeeded") abort.abort();
});
expect(await h.run("focus", abort.signal)).toMatchObject({ code: "cancelled" });
expect(h.calls.filter((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toHaveLength(1);
expect(h.focus).not.toHaveBeenCalled();
});
});
@@ -13,7 +13,138 @@ import {
import { GeometryContext } from "../geometry/frame-context";
import type { CdpRunner } from "../shared";
function scrollbarDriver(
visible = { width: 185, height: 89 },
frameSize: unknown = { width: 200, height: 100 },
) {
const send = vi.fn(async (target: { sessionId?: string }, method: string) => {
if (method === "Runtime.evaluate") return { result: { value: frameSize } };
if (method === "Page.getLayoutMetrics")
return {
cssLayoutViewport: {
clientWidth: target.sessionId ? visible.width : 800,
clientHeight: target.sessionId ? visible.height : 600,
},
};
if (method === "DOM.getBoxModel")
return {
model: {
content: target.sessionId
? [0, 0, 200, 0, 200, 100, 0, 100]
: [100, 100, 500, 100, 500, 300, 100, 300],
},
};
if (method === "DOM.getContentQuads")
return { quads: [[180, 80, 230, 80, 230, 120, 180, 120]] };
throw new Error(`unexpected ${method}`);
});
const cdp: CdpRunner = {
send: (tabId, method) => send({ tabId } as never, method) as never,
sendToTarget: send as CdpRunner["sendToTarget"],
getFrameGraph: async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 10,
target: { tabId: 4, sessionId: "child" },
},
{
frameId: "same-child",
parentFrameId: "child",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "child" },
},
],
}),
};
return { cdp, send };
}
describe("frame geometry projection", () => {
it.each([
{ width: 200, height: 100 },
{ width: 185, height: 100 },
{ width: 200, height: 89 },
{ width: 185, height: 89 },
])("preserves scale and clips scrollbar strips for visible viewport $width × $height", async (visible) => {
const { cdp, send } = scrollbarDriver(visible);
const geometry = await resolveNodeGeometry(cdp, 4, {
target: { tabId: 4, sessionId: "child" },
frameId: "child",
backendNodeId: 101,
});
expect(geometry).toMatchObject({
topBounds: {
x: 460,
y: 260,
width: (visible.width - 180) * 2,
height: (visible.height - 80) * 2,
},
actionPoint: { x: 460 + visible.width - 180, y: 260 + visible.height - 80 },
targetActionPoint: { x: (180 + visible.width) / 2, y: (80 + visible.height) / 2 },
});
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1);
expect(send.mock.calls.filter(([, method]) => method === "Page.getLayoutMetrics")).toHaveLength(
2,
);
});
it("shares the full viewport read across same-target frames, and refreshes it next operation", async () => {
const { cdp, send } = scrollbarDriver();
const context = new GeometryContext(cdp, 4);
await Promise.all([context.targetProjection("child"), context.targetProjection("same-child")]);
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(1);
await new GeometryContext(cdp, 4).targetProjection("child");
expect(send.mock.calls.filter(([, method]) => method === "Runtime.evaluate")).toHaveLength(2);
});
it.each([
null,
{},
{ width: 0, height: 100 },
{ width: 200, height: NaN },
])("rejects unavailable full viewport dimensions: %j", async (size) => {
const { cdp } = scrollbarDriver(undefined, size);
expect(await new GeometryContext(cdp, 4).targetProjection("child")).toBeNull();
});
it("clips an inner OOPIF against an intermediate OOPIF's occupied scrollbar strips", async () => {
const { cdp } = scrollbarDriver();
const outer = cdp.sendToTarget!;
cdp.sendToTarget = async (target, method, params) => {
if (method === "DOM.getBoxModel")
return { model: { content: [150, 60, 250, 60, 250, 120, 150, 120] } } as never;
if (target.sessionId !== "inner") return outer(target, method, params);
if (method === "Runtime.evaluate")
return { result: { value: { width: 100, height: 60 } } } as never;
if (method === "Page.getLayoutMetrics")
return { cssLayoutViewport: { clientWidth: 90, clientHeight: 50 } } as never;
if (method === "DOM.getContentQuads")
return { quads: [[0, 0, 100, 0, 100, 60, 0, 60]] } as never;
throw new Error(`unexpected ${method}`);
};
const graph = await cdp.getFrameGraph!(4);
graph.frames[2] = {
frameId: "inner",
parentFrameId: "child",
ownerBackendNodeId: 20,
target: { tabId: 4, sessionId: "inner" },
};
cdp.getFrameGraph = async () => graph;
expect(
await resolveNodeGeometry(cdp, 4, {
target: { tabId: 4, sessionId: "inner" },
frameId: "inner",
backendNodeId: 101,
}),
).toMatchObject({
topBounds: { x: 400, y: 220, width: 70, height: 58 },
});
});
it("keeps region bounds separate from polygon area", () => {
const region = [
rectPolygon({ x: 0, y: 0, w: 10, h: 10 }),
@@ -77,6 +208,8 @@ describe("frame geometry projection", () => {
const cdp: CdpRunner = {
send: send as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Runtime.evaluate")
return { result: { value: { width: 200, height: 100 } } };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
@@ -135,6 +268,8 @@ describe("frame geometry projection", () => {
throw new Error(`unexpected root command ${method}`);
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "Runtime.evaluate")
return { result: { value: { width: 200, height: 100 } } };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
@@ -196,6 +331,8 @@ describe("frame geometry projection", () => {
}) as CdpRunner["send"],
sendToTarget: vi.fn(async (_target, method) => {
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "Runtime.evaluate")
return { result: { value: { width: 200, height: 100 } } };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
@@ -433,6 +433,7 @@ describe("handleRequestHelp", () => {
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
}
if (method === "Runtime.evaluate") return { result: { value: { width: 200, height: 100 } } };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
@@ -2,8 +2,10 @@ import { describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import type { CdpRunner } from "@/tools/shared";
import {
handleBlur,
handleClick,
handleFill,
handleFocus,
handleHover,
handlePress,
handleSelect,
@@ -201,6 +203,7 @@ describe("handleClick", () => {
if (method === "DOM.getContentQuads") {
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
}
if (method === "Runtime.evaluate") return { result: { value: { width: 200, height: 100 } } };
if (method === "Page.getLayoutMetrics") {
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
}
@@ -220,6 +223,7 @@ describe("handleClick", () => {
{ sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" },
{ sessionId: "child-session", method: "DOM.getContentQuads" },
{ sessionId: "child-session", method: "Page.getLayoutMetrics" },
{ sessionId: "child-session", method: "Runtime.evaluate" },
]);
expect(fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength(3);
});
@@ -540,6 +544,161 @@ describe("handleHover", () => {
});
});
describe("handleFocus and handleBlur", () => {
it("focuses a ref and verifies the deep active element", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, { tabId: 4 });
const fake = makeFakeCdp({
"DOM.scrollIntoViewIfNeeded": () => ({}),
"DOM.focus": () => ({}),
"DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }),
"Runtime.callFunctionOn": () => ({ result: { value: true } }),
});
const res = await handleFocus(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res).toMatchObject({ tab_id: 4, used_ref: "e3", focused: true });
expect(fake.sent.map((call) => call.method)).toEqual([
"DOM.scrollIntoViewIfNeeded",
"DOM.resolveNode",
"DOM.focus",
"Runtime.callFunctionOn",
"Runtime.releaseObject",
]);
});
it("focuses an OOPIF ref in its CDP session", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, {
tabId: 4,
frameId: "child-frame",
cdpSessionId: "child-session",
});
const fake = makeFakeCdp({
"DOM.scrollIntoViewIfNeeded": () => ({}),
});
fake.cdp.getFrameGraph = vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child-frame",
parentFrameId: "main",
ownerBackendNodeId: 99,
target: { tabId: 4, sessionId: "child-session" },
},
],
}));
const targetCalls: Array<{ sessionId?: string; method: string }> = [];
fake.cdp.sendToTarget = vi.fn(async (target, method) => {
targetCalls.push({ sessionId: target.sessionId, method });
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "DOM.focus" || method === "Runtime.releaseObject") return {};
if (method === "DOM.resolveNode") return { object: { objectId: "focus-target" } };
if (method === "Runtime.callFunctionOn") return { result: { value: true } };
throw new Error(`unexpected child CDP call ${method}`);
}) as CdpRunner["sendToTarget"];
const res = await handleFocus(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.focused).toBe(true);
expect(targetCalls).toEqual([
{ sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" },
{ sessionId: "child-session", method: "DOM.resolveNode" },
{ sessionId: "child-session", method: "DOM.focus" },
{ sessionId: "child-session", method: "Runtime.callFunctionOn" },
{ sessionId: "child-session", method: "Runtime.releaseObject" },
]);
});
it("does not focus after cancellation during scrolling", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, { tabId: 4 });
const abort = new AbortController();
const fake = makeFakeCdp({
"DOM.scrollIntoViewIfNeeded": () => {
abort.abort();
return {};
},
});
const res = await handleFocus(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi, signal: abort.signal },
);
expect(res).toMatchObject({ code: "cancelled" });
expect(fake.sent.some((call) => call.method === "DOM.focus")).toBe(false);
});
it("blurs a ref and returns its previous focus state", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, { tabId: 4 });
const fake = makeFakeCdp({
"DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }),
"Runtime.callFunctionOn": vi
.fn()
.mockResolvedValueOnce({ result: { value: { ok: true, was_focused: true } } })
.mockResolvedValueOnce({ result: { value: false } }),
});
const res = await handleBlur(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res).toMatchObject({
tab_id: 4,
used_ref: "e3",
was_focused: true,
focused: false,
});
expect(fake.sent.map((call) => call.method)).toEqual([
"DOM.resolveNode",
"Runtime.callFunctionOn",
"Runtime.callFunctionOn",
"Runtime.releaseObject",
]);
});
it("rejects a target that does not implement blur", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, { tabId: 4 });
const fake = makeFakeCdp({
"DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }),
"Runtime.callFunctionOn": () => ({
result: { value: { ok: false, was_focused: false, focused: false } },
}),
});
const res = await handleBlur(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
expect(res).toMatchObject({ code: "invalid_params", message: /does not support blur/ });
});
});
function successfulFillScript(params: unknown) {
const script = params as { arguments?: Array<{ value: unknown }>; functionDeclaration: string };
const args = script.arguments ?? [];
@@ -3380,6 +3380,7 @@ describe("handleSnapshot", () => {
throw new Error(`unexpected root CDP method ${method}`);
});
const sendToTarget = vi.fn(async (_target, method: string) => {
if (method === "Runtime.evaluate") return { result: { value: { width: 400, height: 300 } } };
if (method === "Page.getLayoutMetrics" && ownerGeometry !== "unavailable")
return {
visualViewport: { clientWidth: 1000 },
@@ -0,0 +1,348 @@
// @vitest-environment node
// Opt in with BSK_SCROLL_CHROME=/path/to/chrome. Uses an isolated browser/profile.
import { describe, expect, it } from "vitest";
import type { CdpFrame, CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
import { SessionManager } from "@/session-manager/manager";
import type { ViewportRect } from "../geometry";
import { handleScrollTo } from "../scroll";
import type { CdpRunner } from "../shared";
type Send = <T = Record<string, unknown>>(
method: string,
params?: object,
sessionId?: string,
) => Promise<T>;
type Tree = { frame: { id: string; parentId?: string; name?: string }; childFrames?: Tree[] };
async function withBrowser(
run: (h: Awaited<ReturnType<typeof harness>>) => Promise<void>,
zoom = 1,
) {
const { withChrome } = await import(
new URL(
"../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs",
import.meta.url,
).href
);
await withChrome(
{ executable: process.env.BSK_SCROLL_CHROME, deviceScale: 1, zoom },
async (send: Send) => run(await harness(send)),
);
}
async function harness(send: Send) {
const { targetId } = await send<{ targetId: string }>("Target.createTarget", {
url: "about:blank",
});
const { sessionId: rootSession } = await send<{ sessionId: string }>("Target.attachToTarget", {
targetId,
flatten: true,
});
await send("Page.bringToFront", {}, rootSession);
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("aa11");
const calls: { method: string; sessionId?: string }[] = [];
const after = { command: (_method: string) => {} };
const sessionFor = (target: CdpTarget) => target.sessionId ?? rootSession;
const forward = async <T>(target: CdpTarget, method: string, params?: object): Promise<T> => {
calls.push({ method, sessionId: target.sessionId });
const result = await send<T>(method, params, sessionFor(target));
after.command(method);
return result;
};
const cdp: CdpRunner = {
send: (_tab, method, params) => forward({ tabId: 4 }, method, params),
sendToTarget: forward,
};
const evaluate = async <T>(expression: string, frame?: CdpFrame): Promise<T> => {
const session = frame ? sessionFor(frame.target) : rootSession;
const world = frame
? await send<{ executionContextId: number }>(
"Page.createIsolatedWorld",
{
frameId: frame.frameId,
worldName: "scroll-oracle",
},
session,
)
: undefined;
const reply = await send<{ result: { value: T }; exceptionDetails?: unknown }>(
"Runtime.evaluate",
{
expression,
contextId: world?.executionContextId,
returnByValue: true,
awaitPromise: true,
},
session,
);
expect(reply.exceptionDetails).toBeUndefined();
return reply.result.value;
};
const rememberRef = (frame: CdpFrame, backendNodeId: number) =>
ctx.refStore.set("e1", backendNodeId, {
tabId: 4,
frameId: frame.frameId,
cdpSessionId: frame.target.sessionId,
});
const ref = async (frame: CdpFrame, selector = "#probe") => {
const { executionContextId } = await send<{ executionContextId: number }>(
"Page.createIsolatedWorld",
{
frameId: frame.frameId,
worldName: "scroll-oracle",
},
sessionFor(frame.target),
);
const { result } = await send<{ result: { objectId: string } }>(
"Runtime.evaluate",
{
expression: `document.querySelector(${JSON.stringify(selector)})`,
contextId: executionContextId,
},
sessionFor(frame.target),
);
const { node } = await send<{ node: { backendNodeId: number } }>(
"DOM.describeNode",
{ objectId: result.objectId },
sessionFor(frame.target),
);
await send("Runtime.releaseObject", { objectId: result.objectId }, sessionFor(frame.target));
rememberRef(frame, node.backendNodeId);
};
const scroll = (target: { selector: string } | { ref: string }, signal?: AbortSignal) =>
handleScrollTo(
manager,
{ session_id: "aa11", ...target },
{
cdp,
signal,
tabsApi: {
get: async (id) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab,
query: async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab],
},
},
);
const loadFrames = async () => {
const { targetInfos } = await send<{ targetInfos: { targetId: string; type: string }[] }>(
"Target.getTargets",
);
const sessions = [rootSession];
for (const target of targetInfos.filter((t) => t.type === "iframe")) {
const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", {
targetId: target.targetId,
flatten: true,
});
sessions.push(sessionId);
}
const frames: (CdpFrame & { name: string })[] = [];
for (const session of sessions) {
const { frameTree } = await send<{ frameTree: Tree }>("Page.getFrameTree", {}, session);
const target = { tabId: 4, ...(session !== rootSession ? { sessionId: session } : {}) };
const visit = (tree: Tree, parentFrameId = tree.frame.parentId) => {
frames.push({ frameId: tree.frame.id, parentFrameId, target, name: tree.frame.name ?? "" });
for (const child of tree.childFrames ?? []) visit(child, tree.frame.id);
};
visit(frameTree);
}
for (const frame of frames) {
if (!frame.parentFrameId) continue;
const parent = frames.find((f) => f.frameId === frame.parentFrameId)!;
const owner = await send<{ backendNodeId: number }>(
"DOM.getFrameOwner",
{ frameId: frame.frameId },
sessionFor(parent.target),
);
frame.ownerBackendNodeId = owner.backendNodeId;
}
const graph: CdpFrameGraph = { rootFrameId: frames[0].frameId, frames };
cdp.getFrameGraph = async () => graph;
return frames;
};
return { send, rootSession, evaluate, ref, rememberRef, scroll, loadFrames, calls, after };
}
function intersection(a: ViewportRect, b: ViewportRect): ViewportRect | null {
const x = Math.max(a.x, b.x),
y = Math.max(a.y, b.y);
const right = Math.min(a.x + a.width, b.x + b.width),
bottom = Math.min(a.y + a.height, b.y + b.height);
return right > x && bottom > y ? { x, y, width: right - x, height: bottom - y } : null;
}
function expectRect(actual: unknown, expected: ViewportRect) {
expect(actual).not.toHaveProperty("code");
for (const key of ["x", "y", "width", "height"] as const)
expect(Math.abs((actual as ViewportRect)[key] - expected[key]), key).toBeLessThan(2);
}
describe.skipIf(!process.env.BSK_SCROLL_CHROME)("real browser scroll-to", () => {
it("scrolls ordinary elements and reports the clipped portion of nested scrollers", async () => {
await withBrowser(async (h) => {
for (const html of [
'<div style="height:2200px"></div><div id="target" style="width:100px;height:40px"></div>',
'<div style="height:200px"></div><div id="clip" style="height:80px;width:200px;overflow:auto"><div id="target" style="width:100px;height:400px"></div></div>',
'<div style="height:200px"></div><div id="clip" style="height:80px;width:200px;overflow:auto;transform:scale(1.25);transform-origin:0 0"><div id="target" style="width:100px;height:400px"></div></div>',
]) {
await h.evaluate(`document.body.innerHTML = ${JSON.stringify(html)}`);
const result = await h.scroll({ selector: "#target" });
const oracle = await h.evaluate<{ rect: ViewportRect; clip?: ViewportRect }>(`({
rect: document.querySelector('#target').getBoundingClientRect().toJSON(),
clip: document.querySelector('#clip')?.getBoundingClientRect().toJSON()
})`);
expectRect(result, oracle.clip ? intersection(oracle.rect, oracle.clip)! : oracle.rect);
}
});
});
it.each([
"visibility:hidden",
"opacity:0",
"display:none",
"content-visibility:hidden",
])("rejects %s targets", async (style) => {
await withBrowser(async (h) => {
await h.evaluate(
`document.body.innerHTML = '<div style="${style}"><button id="target">Target</button></div>'`,
);
expect(await h.scroll({ selector: "#target" })).toMatchObject({
code: "permission_denied",
data: { reason: "element_not_visible" },
});
});
});
it("rejects targets fully outside an overflow:clip ancestor", async () => {
await withBrowser(async (h) => {
await h.evaluate(
`document.body.innerHTML = '<div style="height:80px;overflow:clip"><div style="height:150px"></div><button id="target">Target</button></div>'`,
);
expect(await h.scroll({ selector: "#target" })).toMatchObject({ code: "permission_denied" });
});
});
it.each(["open", "closed"])("clips %s shadow-root refs", async (mode) => {
await withBrowser(async (h) => {
await h.evaluate(`(() => {
const host = document.createElement('div'); document.body.append(host);
const root = host.attachShadow({mode:'${mode}'});
root.innerHTML = '<div style="height:60px;overflow:auto"><button id="probe" style="height:200px;width:100px">Target</button></div>';
window.probe = root.querySelector('#probe');
})()`);
const frames = await h.loadFrames();
const { result } = await h.send<{ result: { objectId: string } }>(
"Runtime.evaluate",
{ expression: "window.probe" },
h.rootSession,
);
// Build the ref from the actual closed-root node, as an observation does.
const { node } = await h.send<{ node: { backendNodeId: number } }>(
"DOM.describeNode",
{ objectId: result.objectId },
h.rootSession,
);
await h.send("Runtime.releaseObject", { objectId: result.objectId }, h.rootSession);
h.rememberRef(frames[0], node.backendNodeId);
expect(await h.scroll({ ref: "e1" })).toMatchObject({ height: 60 });
});
});
it("cancels after scrolling and releases the measurement object without follow-up reads", async () => {
await withBrowser(async (h) => {
await h.evaluate("document.body.innerHTML = '<button id=target>Target</button>'");
const abort = new AbortController();
h.after.command = (method) => {
if (method === "DOM.resolveNode") abort.abort();
};
expect(await h.scroll({ selector: "#target" }, abort.signal)).toMatchObject({
code: "cancelled",
});
expect(h.calls.at(-1)!.method).toBe("Runtime.releaseObjectGroup");
expect(h.calls.some((call) => call.method === "Runtime.callFunctionOn")).toBe(false);
});
});
it.each([
{ fixture: "snapshot-coordinates", zoom: 1 },
{ fixture: "snapshot-coordinates", zoom: 1.25 },
{ fixture: "oopif-scrollbars", zoom: 1 },
{ fixture: "oopif-scrollbars", zoom: 0.8 },
])("scrolls nested frames: $fixture at zoom $zoom", async ({ fixture, zoom }) => {
const { createEvalServer } = await import(
new URL("../../../../../evals/browser/lib/server.mjs", import.meta.url).href
);
const server = createEvalServer();
const { baseUrl } = await server.start();
try {
await withBrowser(async (h) => {
await h.send("Page.navigate", { url: `${baseUrl}/${fixture}?run=scroll` }, h.rootSession);
await expect
.poll(
() =>
server
.snapshot("scroll")
.events.some(
(e: { type: string; data: { root?: boolean } }) =>
e.type === "geometry.ready" && e.data.root,
),
{ timeout: 10_000 },
)
.toBe(true);
const frames = await h.loadFrames();
expect(frames.length).toBe(fixture === "snapshot-coordinates" ? 5 : 3);
for (const frame of frames) {
await h.ref(frame);
const result = await h.scroll({ ref: "e1" });
// Independent DOM oracle: CSS border boxes plus this fixture's known
// axis-aligned iframe scales. Re-read after each scroll changes layout.
let rect = await h.evaluate<ViewportRect>(
"document.querySelector('#probe').getBoundingClientRect().toJSON()",
frame,
);
let current = frame;
while (true) {
const viewport = await h.evaluate<ViewportRect>(
"({ x:0, y:0, width:document.documentElement.clientWidth, height:document.documentElement.clientHeight })",
current,
);
rect = intersection(rect, viewport)!;
if (!current.parentFrameId) break;
const parent = frames.find((f) => f.frameId === current.parentFrameId)!;
const owner = await h.evaluate<{ x: number; y: number; scale: number }>(
`(() => {
const iframe = document.querySelector('iframe[name="${current.name}"]');
const r = iframe.getBoundingClientRect(); const s = getComputedStyle(iframe);
const scale = r.width / iframe.offsetWidth;
return { x:r.x + (iframe.clientLeft + parseFloat(s.paddingLeft))*scale,
y:r.y + (iframe.clientTop + parseFloat(s.paddingTop))*scale, scale };
})()`,
parent,
);
rect = {
x: owner.x + rect.x * owner.scale,
y: owner.y + rect.y * owner.scale,
width: rect.width * owner.scale,
height: rect.height * owner.scale,
};
current = parent;
}
expectRect(result, rect);
}
const child = frames.find((frame) => frame.parentFrameId === frames[0].frameId)!;
await h.ref(child);
await h.evaluate(
`document.querySelector('iframe[name="${child.name}"]').style.opacity = '0'`,
);
expect(await h.scroll({ ref: "e1" })).toMatchObject({ code: "permission_denied" });
}, zoom);
} finally {
await server.stop();
}
}, 30_000);
});
@@ -0,0 +1,310 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
import { SessionManager } from "@/session-manager/manager";
import type { JavaScriptDialogInfo, ScrollToParams } from "@/transport/types";
import type { ViewportRect } from "../geometry";
import { handleScrollTo } from "../scroll";
import type { CdpRunner } from "../shared";
interface Call {
target: CdpTarget;
method: string;
params?: Record<string, unknown>;
aborted: boolean;
}
async function fixture(frame: "top" | "same-target" | "oopif" = "top") {
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("aa11");
ctx.refStore.set("e3", 1234, {
tabId: 4,
...(frame !== "top" ? { frameId: "child" } : {}),
...(frame === "oopif" ? { cdpSessionId: "child-session" } : {}),
});
const abort = new AbortController();
const graph: CdpFrameGraph = {
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 99,
target: { tabId: 4, ...(frame === "oopif" ? { sessionId: "child-session" } : {}) },
},
],
};
const rectangles = new Map<number, ViewportRect | null>([
[1234, { x: 10, y: 20, width: 100, height: 40 }],
[99, { x: 204, y: 306, width: 400, height: 200 }],
]);
const calls: Call[] = [];
const hooks = {
after: (_call: Call) => {},
reply: (_call: Call): object | undefined => undefined,
};
const send = async <T>(target: CdpTarget, method: string, params?: object): Promise<T> => {
const call = {
target,
method,
params: params as Call["params"],
aborted: abort.signal.aborted,
};
calls.push(call);
const override = hooks.reply(call);
hooks.after(call);
if (override) return override as T;
const args = call.params ?? {};
const replies: Record<string, unknown> = {
"DOM.getDocument": { root: { nodeId: 1 } },
"DOM.querySelector": { nodeId: 2 },
"DOM.describeNode": { node: { backendNodeId: 1234 } },
"DOM.scrollIntoViewIfNeeded": {},
"DOM.resolveNode": { object: { objectId: String(args.backendNodeId) } },
"Runtime.callFunctionOn": {
result: {
value: String(args.functionDeclaration).includes("IntersectionObserver")
? rectangles.get(Number(args.objectId))
: { width: 200, height: 100 },
},
},
"Runtime.releaseObject": {},
"Runtime.releaseObjectGroup": {},
"Runtime.evaluate": { result: { value: { width: 200, height: 100 } } },
"Page.getLayoutMetrics": {
cssLayoutViewport: {
clientWidth: target.sessionId ? 185 : 1280,
clientHeight: target.sessionId ? 89 : 720,
},
},
"DOM.getBoxModel": { model: { content: [204, 306, 604, 306, 604, 506, 204, 506] } },
};
if (!(method in replies)) throw new Error(`unexpected CDP call ${method}`);
return replies[method] as T;
};
const cdp: CdpRunner = {
send: (tabId, method, params) => send({ tabId }, method, params),
sendToTarget: send,
getFrameGraph: vi.fn(async () => {
hooks.after({ target: { tabId: 4 }, method: "getFrameGraph", aborted: abort.signal.aborted });
return graph;
}),
trackSessionTab: vi.fn(),
};
const tabsApi = {
get: vi.fn(async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
};
const run = (params: Partial<ScrollToParams> = {}) =>
handleScrollTo(
manager,
{ session_id: "aa11", ref: "@e3", ...params },
{ cdp, tabsApi, signal: abort.signal },
);
return { ctx, cdp, graph, rectangles, calls, hooks, abort, tabsApi, run };
}
afterEach(() => vi.restoreAllMocks());
describe("handleScrollTo", () => {
it("scrolls a ref and returns renderer-clipped bounds", async () => {
const f = await fixture();
expect(await f.run()).toMatchObject({
tab_id: 4,
used_ref: "e3",
x: 10,
y: 20,
width: 100,
height: 40,
});
expect(f.calls[0]).toMatchObject({
method: "DOM.scrollIntoViewIfNeeded",
params: { backendNodeId: 1234 },
});
const group = f.calls.find((call) => call.method === "DOM.resolveNode")!.params!.objectGroup;
expect(f.calls.at(-1)).toMatchObject({
method: "Runtime.releaseObjectGroup",
params: { objectGroup: group },
});
});
it("resolves a selector and tracks the attached tab even when it is missing", async () => {
const f = await fixture();
expect(await f.run({ ref: undefined, selector: "#target" })).toMatchObject({
used_selector: "#target",
});
f.hooks.reply = ({ method }) => (method === "DOM.querySelector" ? { nodeId: 0 } : undefined);
expect(await f.run({ ref: undefined, selector: "#missing" })).toMatchObject({
code: "not_found",
data: { reason: "selector_not_found" },
});
expect(f.cdp.trackSessionTab).toHaveBeenCalledWith("aa11", 4);
});
it.each([
"same-target",
"oopif",
] as const)("projects %s refs and scrolls their owners", async (frame) => {
const f = await fixture(frame);
expect(await f.run()).toMatchObject({ x: 224, y: 346, width: 200, height: 80 });
expect(
f.calls
.filter((call) => call.method === "DOM.scrollIntoViewIfNeeded")
.map((call) => call.params!.backendNodeId),
).toEqual([99, 1234]);
expect(f.cdp.getFrameGraph).toHaveBeenCalledTimes(1);
if (frame === "oopif")
expect(f.calls.find((call) => call.params?.backendNodeId === 1234)!.target.sessionId).toBe(
"child-session",
);
});
it("clips OOPIF bounds to occupied scrollbars without changing the frame scale", async () => {
const f = await fixture("oopif");
f.rectangles.set(1234, { x: 180, y: 80, width: 5, height: 9 });
expect(await f.run()).toMatchObject({ x: 564, y: 466, width: 10, height: 18 });
});
it("clips against the iframe owner's ordinary DOM ancestors", async () => {
const f = await fixture("oopif");
f.rectangles.set(99, { x: 204, y: 306, width: 400, height: 80 });
expect(await f.run()).toMatchObject({ x: 224, y: 346, width: 200, height: 40 });
});
it.each([1234, 99])("rejects a hidden or fully clipped node %i", async (node) => {
const f = await fixture("oopif");
f.rectangles.set(node, null);
expect(await f.run()).toMatchObject({
code: "permission_denied",
data: { reason: "element_not_visible" },
});
expect(f.calls.at(-1)!.method).toBe("Runtime.releaseObjectGroup");
});
it.each([
{ ref: undefined },
{ selector: "#target" },
{ timeout_ms: 0 },
])("rejects invalid params %j", async (params) => {
const f = await fixture();
expect(await f.run(params)).toMatchObject({ code: "invalid_params" });
expect(f.calls).toEqual([]);
});
it.each(["stale", "other-tab"])("rejects %s refs before scrolling", async (mode) => {
const f = await fixture();
if (mode === "stale") f.ctx.refStore.clear();
else f.ctx.refStore.set("e3", 1234, { tabId: 8 });
expect(await f.run()).toMatchObject({ code: "not_found", data: { reason: "ref_not_found" } });
expect(f.calls).toEqual([]);
});
it("enforces the Agent Window boundary", async () => {
const f = await fixture();
f.tabsApi.query.mockResolvedValue([{ id: 4, windowId: 200, active: true } as chrome.tabs.Tab]);
f.tabsApi.get.mockResolvedValue({ id: 4, windowId: 200, active: true } as chrome.tabs.Tab);
expect(await f.run({ tab_id: 4 })).toMatchObject({ code: "permission_denied" });
expect(f.calls).toEqual([]);
});
it.each([
"getFrameGraph",
"DOM.scrollIntoViewIfNeeded",
"DOM.resolveNode",
"Runtime.callFunctionOn",
"Page.getLayoutMetrics",
"Runtime.evaluate",
"DOM.getBoxModel",
])("honors cancellation during %s and only cleans up afterwards", async (method) => {
const f = await fixture("oopif");
f.hooks.after = (call) => {
if (call.method === method) f.abort.abort();
};
expect(await f.run()).toMatchObject({ code: "cancelled" });
expect(
f.calls.filter((call) => call.aborted && call.method !== "Runtime.releaseObjectGroup"),
).toEqual([]);
});
it.each([
"DOM.getDocument",
"DOM.querySelector",
"DOM.describeNode",
])("cancels selector resolution during %s", async (method) => {
const f = await fixture();
f.hooks.after = (call) => {
if (call.method === method) f.abort.abort();
};
expect(await f.run({ ref: undefined, selector: "#target" })).toMatchObject({
code: "cancelled",
});
expect(f.calls.filter((call) => call.aborted)).toEqual([]);
});
it("does not start after early cancellation", async () => {
const f = await fixture();
f.abort.abort();
expect(await f.run()).toMatchObject({ code: "cancelled" });
expect(f.calls).toEqual([]);
expect(f.tabsApi.query).not.toHaveBeenCalled();
});
it("does not issue fallback scrolling after cancellation", async () => {
const f = await fixture();
f.hooks.reply = ({ method }) => {
if (method === "DOM.scrollIntoViewIfNeeded") {
f.abort.abort();
throw new Error("scroll failed");
}
return undefined;
};
expect(await f.run()).toMatchObject({ code: "cancelled" });
expect(f.calls.map((call) => call.method)).toEqual(["DOM.scrollIntoViewIfNeeded"]);
});
it("stops target scrolling after a deadline expires while scrolling its owner", async () => {
const f = await fixture("oopif");
const now = vi.spyOn(Date, "now").mockReturnValue(100);
f.hooks.after = ({ method }) => {
if (method === "DOM.scrollIntoViewIfNeeded") now.mockReturnValue(201);
};
expect(await f.run({ timeout_ms: 100 })).toMatchObject({ code: "timeout" });
expect(f.calls.filter((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toHaveLength(1);
});
it("reports measurement exceptions and releases objects even if cleanup fails", async () => {
const f = await fixture();
f.hooks.reply = ({ method }) => {
if (method === "Runtime.callFunctionOn")
return { exceptionDetails: { text: "measurement failed" } };
if (method === "Runtime.releaseObjectGroup") throw new Error("target closed");
return undefined;
};
expect(await f.run()).toMatchObject({ code: "cdp_failed", message: "measurement failed" });
expect(f.calls.at(-1)!.method).toBe("Runtime.releaseObjectGroup");
});
it("attaches dialogs observed during scrolling", async () => {
const f = await fixture();
f.cdp.dialogCursor = () => 5;
const dialogs: JavaScriptDialogInfo[] = [
{
sequence: 6,
tab_id: 4,
type: "alert",
message: "scrolled",
handled: "accepted",
url: "https://example.test",
},
];
f.cdp.dialogsSince = vi.fn(() => dialogs);
expect(await f.run()).toHaveProperty("dialogs", dialogs);
expect(f.cdp.dialogsSince).toHaveBeenCalledWith(4, 5);
});
});
@@ -1,6 +1,7 @@
// @vitest-environment node
import { describe, expect, it } from "vitest";
import type { CdpFrame, CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
import { resolveNodeGeometry } from "../frame-geometry";
import type { CdpRunner } from "../shared";
import { captureObservationFacts } from "../vom/capture-coordinator";
@@ -11,12 +12,28 @@ type Send = <T = Record<string, unknown>>(
) => Promise<T>;
type Tree = { frame: { id: string; parentId?: string; name?: string }; childFrames?: Tree[] };
type Rect = { x: number; y: number; w: number; h: number };
type Oracle = { probe: Rect; owners: Record<string, { x: number; y: number; scale: number }> };
type Oracle = {
probes: Record<string, Rect>;
viewport: { width: number; height: number };
owners: Record<string, { x: number; y: number; scale: number }>;
};
function clip(rect: Rect, viewport: Oracle["viewport"]): Rect | null {
const x = Math.max(0, rect.x),
y = Math.max(0, rect.y);
const right = Math.min(viewport.width, rect.x + rect.w);
const bottom = Math.min(viewport.height, rect.y + rect.h);
return right > x && bottom > y ? { x, y, w: right - x, h: bottom - y } : null;
}
// The independent oracle uses DOM border boxes and this fixture's axis-aligned
// iframe transforms. No production snapshot conversion/projection builds expectations.
const oracleExpression = `(() => {
const box = document.querySelector('#probe').getBoundingClientRect();
const probes = {};
for (const node of document.querySelectorAll('[data-geometry-probe]')) {
const box = node.getBoundingClientRect();
probes[node.id] = { x: box.x, y: box.y, w: box.width, h: box.height };
}
const owners = {};
for (const frame of document.querySelectorAll('iframe')) {
const rect = frame.getBoundingClientRect();
@@ -28,17 +45,31 @@ const oracleExpression = `(() => {
scale,
};
}
return { probe: { x: box.x, y: box.y, w: box.width, h: box.height }, owners };
return { probes, viewport: { width: document.documentElement.clientWidth, height: document.documentElement.clientHeight }, owners };
})()`;
describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate contract", () => {
it.each([
{ deviceScale: 1, zoom: 1 },
{ deviceScale: 0.8, zoom: 1 },
{ deviceScale: 1, zoom: 1.25 },
{ deviceScale: 2, zoom: 1 },
{ deviceScale: 2, zoom: 0.8 },
])("matches DOM geometry at device scale $deviceScale and zoom $zoom", async (configuration) => {
it.each(
[
{ deviceScale: 1, zoom: 1 },
{ deviceScale: 0.8, zoom: 1 },
{ deviceScale: 1, zoom: 1.25 },
{ deviceScale: 2, zoom: 1 },
{ deviceScale: 2, zoom: 0.8 },
]
.flatMap((configuration) => [
{ ...configuration, fixture: "snapshot-coordinates", scrollbars: "none" },
{ ...configuration, fixture: "oopif-scrollbars", scrollbars: "both" },
])
.concat(
["vertical", "horizontal", "none"].map((scrollbars) => ({
deviceScale: 1,
zoom: 1,
fixture: "oopif-scrollbars",
scrollbars,
})),
),
)("$fixture: device scale $deviceScale, zoom $zoom, scrollbars $scrollbars", async (configuration) => {
const evalRoot = new URL("../../../../../evals/browser/", import.meta.url);
const { createEvalServer } = await import(new URL("lib/server.mjs", evalRoot).href);
const { withChrome } = await import(
@@ -59,7 +90,9 @@ describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate c
);
await send(
"Page.navigate",
{ url: `${baseUrl}/snapshot-coordinates?run=coordinates` },
{
url: `${baseUrl}/${configuration.fixture}?run=coordinates&scrollbars=${configuration.scrollbars}`,
},
rootSession,
);
await expect
@@ -86,7 +119,7 @@ describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate c
});
sessions.push(sessionId);
}
expect(sessions.length).toBe(2); // The cross-site fixture must actually be an OOPIF.
expect(sessions.length).toBe(configuration.fixture === "oopif-scrollbars" ? 3 : 2);
const frames: CdpFrame[] = [];
const names = new Map<string, string>();
const sessionFor = (target: CdpTarget) => target.sessionId ?? rootSession;
@@ -107,7 +140,7 @@ describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate c
};
visit(frameTree);
}
expect(frames).toHaveLength(5);
expect(frames).toHaveLength(configuration.fixture === "oopif-scrollbars" ? 3 : 5);
for (const frame of frames) {
if (!frame.parentFrameId) continue;
const parent = frames.find((item) => item.frameId === frame.parentFrameId)!;
@@ -127,6 +160,11 @@ describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate c
},
sendToTarget: (target, method, params) => {
calls.push(`${target.sessionId ?? target.tabId}:${method}`);
if (
method === "Runtime.evaluate" &&
(params as { expression?: string })?.expression?.includes("window.innerWidth")
)
calls.push(`${target.sessionId ?? target.tabId}:viewport-size`);
return send(method, params, sessionFor(target));
},
getFrameGraph: async () => graph,
@@ -179,47 +217,130 @@ describe.skipIf(!process.env.BSK_GEOMETRY_CHROME)("real DOMSnapshot coordinate c
);
oracles.set(frame.frameId, reply.result.value);
}
const childViewport = await send<{
result: {
value: { width: number; height: number; clientWidth: number; clientHeight: number };
};
}>(
"Runtime.evaluate",
{
expression:
"({ width: innerWidth, height: innerHeight, clientWidth: document.documentElement.clientWidth, clientHeight: document.documentElement.clientHeight })",
returnByValue: true,
},
sessions[1],
);
for (const dimension of ["Width", "Height"] as const) {
const occupied =
childViewport.result.value[dimension === "Width" ? "width" : "height"] -
childViewport.result.value[`client${dimension}`];
if (
configuration.scrollbars === "both" ||
configuration.scrollbars === (dimension === "Width" ? "vertical" : "horizontal")
)
expect(occupied).toBeGreaterThan(0);
else expect(occupied).toBe(0);
}
const facts = await captureObservationFacts(cdp, 1);
expect(facts.issues).toEqual([]);
for (const frame of frames) {
const node = facts.documents
.find((doc) => doc.frame.frameId === frame.frameId)
?.domNodes.find((node) => node.attrs.id === "probe");
expect(node, `missing probe in ${names.get(frame.frameId) || "root"}`).toBeDefined();
const local = oracles.get(frame.frameId)!.probe;
const top = { ...local };
let current = frame;
while (current.parentFrameId) {
const owner = oracles.get(current.parentFrameId)!.owners[names.get(current.frameId)!];
top.x = owner.x + top.x * owner.scale;
top.y = owner.y + top.y * owner.scale;
top.w *= owner.scale;
top.h *= owner.scale;
current = frames.find((item) => item.frameId === current.parentFrameId)!;
}
expect(
node!.rect,
JSON.stringify({
frame: names.get(frame.frameId),
local,
top,
viewport: facts.viewport,
}),
).not.toBeNull();
for (const key of ["x", "y", "w", "h"] as const) {
expect(
Math.abs(node!.localRect![key] - local[key]),
`${names.get(frame.frameId)} local ${key}`,
).toBeLessThan(2);
expect(
Math.abs(node!.rect![key] - top[key]),
`${names.get(frame.frameId)} top ${key}`,
).toBeLessThan(2);
}
}
expect(calls.filter((call) => call.endsWith(":Page.getLayoutMetrics"))).toHaveLength(
sessions.length,
);
// Identity verification also uses Runtime.evaluate; count the full viewport reads separately.
const viewportReadsBeforeLive = calls.filter((call) => call.endsWith(":viewport-size"));
expect(new Set(viewportReadsBeforeLive).size).toBe(viewportReadsBeforeLive.length);
expect(viewportReadsBeforeLive.length).toBeLessThanOrEqual(sessions.length - 1);
for (const frame of frames) {
for (const [id, local] of Object.entries(oracles.get(frame.frameId)!.probes)) {
const node = facts.documents
.find((doc) => doc.frame.frameId === frame.frameId)
?.domNodes.find((node) => node.attrs.id === id);
expect(node, `missing ${id} in ${names.get(frame.frameId) || "root"}`).toBeDefined();
let top = clip(local, oracles.get(frame.frameId)!.viewport);
let current = frame;
while (top && current.parentFrameId) {
const parentOracle = oracles.get(current.parentFrameId)!;
const owner = parentOracle.owners[names.get(current.frameId)!];
top = clip(
{
x: owner.x + top.x * owner.scale,
y: owner.y + top.y * owner.scale,
w: top.w * owner.scale,
h: top.h * owner.scale,
},
parentOracle.viewport,
);
current = frames.find((item) => item.frameId === current.parentFrameId)!;
}
for (const key of ["x", "y", "w", "h"] as const)
expect(
Math.abs(node!.localRect![key] - local[key]),
`${names.get(frame.frameId)} ${id} local ${key}`,
).toBeLessThan(2);
if (!top) {
expect(node!.rect, `${id} must be clipped out`).toBeNull();
if (configuration.fixture === "oopif-scrollbars")
expect(
await resolveNodeGeometry(cdp, 1, {
target: frame.target,
frameId: frame.frameId,
backendNodeId: node!.backendNodeId,
}),
).toMatchObject({ code: "permission_denied" });
continue;
}
expect(node!.rect).not.toBeNull();
for (const key of ["x", "y", "w", "h"] as const)
expect(
Math.abs(node!.rect![key] - top[key]),
`${names.get(frame.frameId)} ${id} top ${key}`,
).toBeLessThan(2);
if (configuration.fixture !== "oopif-scrollbars") continue;
const live = await resolveNodeGeometry(cdp, 1, {
target: frame.target,
frameId: frame.frameId,
backendNodeId: node!.backendNodeId,
});
if ("code" in live) throw new Error(live.message);
for (const [key, liveKey] of [
["x", "x"],
["y", "y"],
["w", "width"],
["h", "height"],
] as const)
expect(
Math.abs(live.topBounds[liveKey] - top[key]),
`${id} live ${key}`,
).toBeLessThan(2);
const before = server.snapshot("coordinates").events.length;
for (const type of ["mousePressed", "mouseReleased"])
await send(
"Input.dispatchMouseEvent",
{ type, ...live.actionPoint, button: "left", clickCount: 1 },
rootSession,
);
await expect
.poll(
() =>
server
.snapshot("coordinates")
.events.slice(before)
.some(
(event: { type: string; path: string; data: { probe?: string } }) =>
event.type === "geometry.clicked" &&
event.data.probe === id &&
event.path ===
(frame.parentFrameId
? names.get(frame.frameId) === "nested"
? "/oopif-scrollbars/nested"
: "/oopif-scrollbars/frame"
: "/oopif-scrollbars"),
),
{ timeout: 3000 },
)
.toBe(true);
}
}
const metrics = await send<{
cssVisualViewport: { zoom: number };
visualViewport: { clientWidth: number };
+49 -1
View File
@@ -2,12 +2,14 @@ import { OVERLAY_AUTOMATION_BYPASS } from "@/lib/overlay-bridge";
import type { SessionManager } from "@/session-manager/manager";
import type { Transport } from "@/transport/transport";
import type {
BlurParams,
ClickParams,
ConsoleParams,
DownloadParams,
EmulateParams,
EvaluateParams,
FillParams,
FocusParams,
GetHtmlParams,
HoverParams,
HoverResult,
@@ -27,6 +29,7 @@ import type {
ResponseFrame,
RpcError,
ScreenshotParams,
ScrollToParams,
SelectParams,
SnapshotParams,
UploadParams,
@@ -39,7 +42,15 @@ import { type EmulateCdpRunner, handleEmulate } from "./emulate";
import { classifyCdpError } from "./errors";
import { handleEvaluate } from "./evaluate";
import { handleRequestHelp } from "./human-loop";
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction";
import {
handleBlur,
handleClick,
handleFill,
handleFocus,
handleHover,
handlePress,
handleSelect,
} from "./interaction";
import {
handleNavigate,
handleNavigateBack,
@@ -61,6 +72,7 @@ import {
handleRecordStop,
type RecordRuntimeDeps,
} from "./record";
import { handleScrollTo } from "./scroll";
import {
handleSessionStart,
handleSessionStop,
@@ -515,6 +527,39 @@ export class ToolDispatcher {
);
return this.rememberHover((req.params as HoverParams).session_id, result);
}
case "tool.scroll_to":
return this.withHoverReleaseForRequest(
req.params as ScrollToParams,
() =>
handleScrollTo(
this.sessions,
req.params as ScrollToParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
),
signal,
);
case "tool.focus":
return this.withHoverReleaseForRequest(
req.params as FocusParams,
() =>
handleFocus(
this.sessions,
req.params as FocusParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
),
signal,
);
case "tool.blur":
return this.withHoverReleaseForRequest(
req.params as BlurParams,
() =>
handleBlur(
this.sessions,
req.params as BlurParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
),
signal,
);
case "tool.fill":
return this.withHoverReleaseForRequest(
req.params as FillParams,
@@ -785,6 +830,9 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
case "tool.reload":
case "tool.click":
case "tool.hover":
case "tool.scroll_to":
case "tool.focus":
case "tool.blur":
case "tool.fill":
case "tool.press":
case "tool.select":
+2 -1
View File
@@ -161,7 +161,8 @@ export async function resolveNodeGeometry(
return geometryError(`could not resolve frame geometry for ${address.frameId}`);
topVisibleRegions = projectRegionToViewport(localRegion, projection);
if (address.target.sessionId) {
const localViewport = projection.edges[0]?.sourceViewport ?? projection.topViewport;
const localViewport = await context.viewport(address.target);
if (!localViewport) return geometryError("could not resolve target viewport geometry");
const localVisibleRegions = localRegion
.map((polygon) =>
clipPolygon(
@@ -9,8 +9,10 @@ import {
type Polygon,
type ProjectiveEdge,
parseCdpQuad,
projectPolygon,
type Quad,
type Size,
viewportPolygon,
} from "../geometry";
import { type CdpRunner, sendToCdpTarget } from "../shared";
import type { CoordinateOwner, CssViewport, SnapshotProjectionResult } from "./coordinate-types";
@@ -67,6 +69,7 @@ export function cssViewport(metrics: LayoutMetrics): CssViewport {
/** One read-only measurement phase. Discard after scrolling or any later operation. */
export class GeometryContext {
private readonly metrics = new Map<string, Promise<LayoutMetrics>>();
private readonly frameViewports = new Map<string, Promise<Size | null>>();
private readonly owners = new Map<string, Promise<Quad | null>>();
private readonly projections = new Map<string, Promise<GeometryProjection | null>>();
private readonly snapshotOwners = new Map<string, Promise<{ quad: Quad; size: Size } | null>>();
@@ -157,6 +160,35 @@ export class GeometryContext {
: null;
}
/** The full OOPIF viewport maps to its owner's content quad. Layout metrics
* exclude occupied scrollbars and remain the clipping boundary, not the scale. */
private frameViewport(target: CdpTarget): Promise<Size | null> {
const key = cdpTargetKey(target);
let promise = this.frameViewports.get(key);
if (!promise) {
promise = this.request<{ result?: { value?: Size }; exceptionDetails?: unknown }>(
target,
"Runtime.evaluate",
{
expression: "({ width: window.innerWidth, height: window.innerHeight })",
returnByValue: true,
},
).then(({ result, exceptionDetails }) => {
const size = result?.value;
return !exceptionDetails &&
size &&
Number.isFinite(size.width) &&
Number.isFinite(size.height) &&
size.width > 0 &&
size.height > 0
? size
: null;
});
this.frameViewports.set(key, promise);
}
return promise;
}
ownerContent(target: CdpTarget, backendNodeId: number): Promise<Quad | null> {
const key = `${cdpTargetKey(target)}:${backendNodeId}`;
let promise = this.owners.get(key);
@@ -342,12 +374,19 @@ export class GeometryContext {
const parent = frames.get(root.parentFrameId);
if (!parent || root.ownerBackendNodeId === undefined) return null;
const destinationQuad = await this.ownerContent(parent.target, root.ownerBackendNodeId);
const source = await this.viewport(root.target);
if (!destinationQuad) return null;
const source = await this.frameViewport(root.target);
const visible = await this.viewport(root.target);
const parentRoot = this.targetRoot(frames, parent);
if (!destinationQuad || !source || !parentRoot) return null;
if (!source || !visible || !parentRoot) return null;
const destinationClips = await this.clips(frames, parent, parentRoot);
if (!destinationClips) return null;
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
const edge = { sourceViewport: source, destinationQuad, destinationClips };
// Keep scrollbar strips clipped at every target boundary, including
// intermediate OOPIFs. Project the clip with the same full-viewport scale.
if (source.width !== visible.width || source.height !== visible.height)
destinationClips.push(projectPolygon(viewportPolygon(visible), edge));
edges.push(edge);
root = parentRoot;
}
const topViewport = await this.viewport(root.target);
+206 -2
View File
@@ -1,5 +1,4 @@
// DOM interaction tools — `tool.click`, `tool.fill`, `tool.press`, and
// `tool.select`.
// DOM interaction tools — click, hover, focus/blur, fill, press, and select.
//
// All interaction tools:
// 1. Resolve target tab (sandbox: must be inside Agent Window).
@@ -14,10 +13,14 @@ import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import type { SessionContext, SessionManager } from "@/session-manager/manager";
import type {
BlurParams,
BlurResult,
ClickParams,
ClickResult,
FillParams,
FillResult,
FocusParams,
FocusResult,
HoverParams,
HoverResult,
KeyModifier,
@@ -227,6 +230,207 @@ export async function resolveBackendNode(
}
}
// Check the target's own root, then its hosts: closed shadow roots are not
// reachable via host.shadowRoot. DOM focus remains meaningful in background tabs.
const FOCUS_CHECK = `function() {
if (!this.isConnected) return false;
let element = this;
while (element) {
const root = element.getRootNode();
if (root.activeElement !== element) return false;
element = root.host;
}
return true;
}`;
const BLUR_TARGET = `function() {
if (!this.isConnected) throw new Error('blur target is detached');
const wasFocused = (${FOCUS_CHECK}).call(this);
if (typeof this.blur !== 'function') return { ok: false, was_focused: wasFocused };
this.blur();
return { ok: true, was_focused: wasFocused };
}`;
interface FocusScriptReply<T> {
result?: { value?: T };
exceptionDetails?: { text?: string; exception?: { description?: string } };
}
function checkFocusAbort(signal: AbortSignal | undefined): void {
if (signal?.aborted) throw new DOMException("interaction aborted", "AbortError");
}
// Only focus/blur use this view. Guard every CDP boundary, including commands
// inside selector resolution and frame scrolling, without changing other tools.
function focusCdp(cdp: CdpRunner, signal: AbortSignal | undefined): CdpRunner {
const send = <T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T> => {
checkFocusAbort(signal);
return cdpRunnerForTarget(cdp, target).send<T>(target.tabId, method, params);
};
return {
send: (tabId, method, params) => send({ tabId }, method, params),
sendToTarget: send,
trackSessionTab: cdp.trackSessionTab?.bind(cdp),
getFrameGraph: cdp.getFrameGraph
? (tabId) => {
checkFocusAbort(signal);
return cdp.getFrameGraph!(tabId);
}
: undefined,
};
}
// ---------------------------------------------------------------------------
// tool.focus / tool.blur
// ---------------------------------------------------------------------------
export function handleFocus(
manager: SessionManager,
params: FocusParams,
deps: InteractionDeps = getDefaultDeps(),
): Promise<FocusResult | RpcError> {
return changeFocus(manager, params, deps, "focus");
}
export function handleBlur(
manager: SessionManager,
params: BlurParams,
deps: InteractionDeps = getDefaultDeps(),
): Promise<BlurResult | RpcError> {
return changeFocus(manager, params, deps, "blur");
}
function changeFocus(
manager: SessionManager,
params: FocusParams,
deps: InteractionDeps,
action: "focus",
): Promise<FocusResult | RpcError>;
function changeFocus(
manager: SessionManager,
params: BlurParams,
deps: InteractionDeps,
action: "blur",
): Promise<BlurResult | RpcError>;
async function changeFocus(
manager: SessionManager,
params: FocusParams | BlurParams,
deps: InteractionDeps,
action: "focus" | "blur",
): Promise<FocusResult | BlurResult | RpcError> {
const ctx = lookupSession(manager, params, action);
if (isRpcError(ctx)) return ctx;
const cdp = focusCdp(deps.cdp, deps.signal);
let objectId: string | undefined;
let cleanupCdp: CdpRunner | undefined;
let tabId: number | undefined;
try {
checkFocusAbort(deps.signal);
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
checkFocusAbort(deps.signal);
if (isRpcError(target)) return target;
const denied = enforceAgentWindow(ctx, target, action);
if (denied) return denied;
tabId = target.tabId;
const dialogCursor = markDialogCursor(deps.cdp, tabId);
const node = await resolveBackendNode(cdp, ctx, target, params, action);
checkFocusAbort(deps.signal);
if (isRpcError(node)) return node;
const nodeCdp = cdpRunnerForTarget(cdp, node.cdpTarget);
cleanupCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
deps.cdp.trackSessionTab?.(ctx.sessionId, tabId);
if (action === "focus") {
const scrollErr = await scrollElementAndFramesIntoView(
cdp,
tabId,
node.cdpTarget,
node.backendNodeId,
node.frameId,
);
checkFocusAbort(deps.signal);
if (scrollErr) return scrollErr;
}
const resolved = await backendNodeToObject(nodeCdp, tabId, node.backendNodeId);
// Keep the handle before checking cancellation so finally can release it.
if (!isRpcError(resolved)) objectId = resolved;
checkFocusAbort(deps.signal);
if (isRpcError(resolved)) return resolved;
const runScript = async <T>(functionDeclaration: string): Promise<T | undefined> => {
const reply = await nodeCdp.send<FocusScriptReply<T>>(
target.tabId,
"Runtime.callFunctionOn",
{
objectId,
functionDeclaration,
returnByValue: true,
},
);
checkFocusAbort(deps.signal);
if (reply.exceptionDetails) {
const details = reply.exceptionDetails;
throw new Error(
`${action} script failed: ${details.exception?.description ?? details.text ?? "unknown exception"}`,
);
}
return reply.result?.value;
};
let wasFocused: boolean | undefined;
if (action === "focus") {
await nodeCdp.send(tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
checkFocusAbort(deps.signal);
} else {
const mutation = await runScript<{ ok: boolean; was_focused: boolean }>(BLUR_TARGET);
if (typeof mutation?.ok !== "boolean" || typeof mutation.was_focused !== "boolean") {
throw new Error("blur script returned an unexpected result");
}
if (!mutation.ok) {
return { code: "invalid_params", message: "target element does not support blur()" };
}
wasFocused = mutation.was_focused;
}
// Use a separate call so microtasks from focus/blur handlers finish before
// we report success. In particular, a blur handler can restore focus.
const focused = await runScript<boolean>(FOCUS_CHECK);
if (typeof focused !== "boolean") {
throw new Error(`${action} verification returned an unexpected result`);
}
if (focused !== (action === "focus")) {
throw new Error(
action === "focus"
? "target element did not become focused"
: "target element remained focused after blur()",
);
}
return attachDialogs(deps.cdp, tabId, dialogCursor, {
tab_id: tabId,
used_ref: node.usedRef,
used_selector: node.usedSelector,
focused,
...(action === "blur" ? { was_focused: wasFocused! } : {}),
});
} catch (err) {
return (
throwIfAborted(deps.signal) ?? {
code: "cdp_failed",
message: err instanceof Error ? err.message : String(err),
}
);
} finally {
if (objectId !== undefined && cleanupCdp && tabId !== undefined) {
// Cleanup must also run after cancellation. Navigation may have already
// disposed the object, which must not replace the operation's result.
try {
await cleanupCdp.send(tabId, "Runtime.releaseObject", { objectId });
} catch {
// The target or execution context may no longer exist.
}
}
}
}
export async function resolveActionTarget(
cdp: CdpRunner,
ctx: SessionContext,
@@ -0,0 +1,142 @@
import { cdpTargetKey } from "@/browser-driver/frame-graph";
import type { NodeAddress } from "./frame-geometry";
import {
clipPolygon,
projectRegionToViewport,
type Region,
rectPolygon,
regionBounds,
type ViewportRect,
} from "./geometry";
import { GeometryContext } from "./geometry/frame-context";
import { type CdpRunner, sendToCdpTarget } from "./shared";
// Ask the renderer to apply containing-block clips (including shadow DOM),
// instead of treating layout quads as visible pixels. Each document is measured
// locally; the existing frame projections supply the top-viewport coordinates.
// IntersectionObserver reports a bounding rectangle, not an occlusion/hit test.
const VISIBLE_RECT = `function(timeoutMs) {
const element = this;
const visible = () => element.isConnected && element.checkVisibility({
checkOpacity: true, checkVisibilityCSS: true, contentVisibilityAuto: true
});
if (typeof element.checkVisibility !== 'function' || !visible()) return null;
return new Promise((resolve, reject) => {
const observer = new IntersectionObserver(([entry]) => {
clearTimeout(timer);
observer.disconnect();
const rect = entry.intersectionRect;
resolve(visible() && entry.isIntersecting && rect.width > 0 && rect.height > 0
? { x: rect.x, y: rect.y, width: rect.width, height: rect.height }
: null);
}, { root: element.ownerDocument });
const timer = setTimeout(() => {
observer.disconnect();
reject(new Error('scroll-to visibility measurement timed out'));
}, timeoutMs);
observer.observe(element);
});
}`;
async function visibleRect(
cdp: CdpRunner,
address: NodeAddress,
deadline: number,
): Promise<ViewportRect | null> {
const resolved = await sendToCdpTarget<{ object?: { objectId?: string } }>(
cdp,
address.target,
"DOM.resolveNode",
{ backendNodeId: address.backendNodeId },
);
if (!resolved.object?.objectId) throw new Error("scroll-to could not resolve the target element");
const reply = await sendToCdpTarget<{
result?: { value?: ViewportRect | null };
exceptionDetails?: { text?: string; exception?: { description?: string } };
}>(cdp, address.target, "Runtime.callFunctionOn", {
objectId: resolved.object.objectId,
functionDeclaration: VISIBLE_RECT,
// Bound this read even if a background document stops rendering. Its
// observer always disconnects; cancellation cannot leave a live observer.
arguments: [{ value: Math.max(1, Math.min(1_000, deadline - Date.now())) }],
awaitPromise: true,
returnByValue: true,
});
if (reply.exceptionDetails) {
const details = reply.exceptionDetails;
throw new Error(details.exception?.description ?? details.text ?? "visibility script failed");
}
const rect = reply.result?.value;
if (rect === null) return null;
if (
!rect ||
![rect.x, rect.y, rect.width, rect.height].every(Number.isFinite) ||
rect.width <= 0 ||
rect.height <= 0
) {
throw new Error("scroll-to visibility measurement returned invalid bounds");
}
return rect;
}
async function projectRect(
context: GeometryContext,
address: NodeAddress,
rect: ViewportRect,
): Promise<Region> {
let region: Region = [rectPolygon({ x: rect.x, y: rect.y, w: rect.width, h: rect.height })];
if (!address.frameId) return region;
const frame = await context.frame(address.frameId);
if (!frame || cdpTargetKey(frame.target) !== cdpTargetKey(address.target)) {
throw new Error("scroll-to frame no longer belongs to its target");
}
const parent = frame.parentFrameId ? await context.frame(frame.parentFrameId) : undefined;
if (parent && cdpTargetKey(parent.target) === cdpTargetKey(frame.target)) {
// DOM rectangles are document-local. Same-target iframe quads are already
// target-local, so cross this one boundary before the target projection.
const viewport = await context.viewport(frame.target);
if (frame.ownerBackendNodeId === undefined || !viewport) {
throw new Error("scroll-to could not resolve the frame viewport");
}
const local = await context.snapshotProjection(address, frame.ownerBackendNodeId, [], viewport);
if (local.status !== "available") throw new Error("scroll-to could not project the frame");
region = projectRegionToViewport(region, local.projection.geometry);
}
const projection = await context.targetProjection(frame.frameId);
if (!projection) throw new Error("scroll-to could not project the target viewport");
return projectRegionToViewport(region, projection);
}
/** Only scroll-to needs renderer-clipped bounds; other interaction geometry is unchanged. */
export async function scrollVisibleBounds(
cdp: CdpRunner,
tabId: number,
address: NodeAddress,
deadline: number,
): Promise<ViewportRect | null> {
const context = new GeometryContext(cdp, tabId);
const rect = await visibleRect(cdp, address, deadline);
if (!rect) return null;
let region = await projectRect(context, address, rect);
if (address.frameId) {
const ancestry = await context.ancestry(address.frameId);
if (!ancestry) throw new Error("scroll-to could not resolve the frame ancestry");
for (const child of ancestry) {
const parent = child.parentFrameId ? await context.frame(child.parentFrameId) : undefined;
if (!parent || child.ownerBackendNodeId === undefined) {
throw new Error("scroll-to could not resolve the frame owner");
}
const owner = {
target: parent.target,
frameId: parent.frameId,
backendNodeId: child.ownerBackendNodeId,
};
// A visible child document can still have a hidden or clipped iframe.
const ownerRect = await visibleRect(cdp, owner, deadline);
if (!ownerRect) return null;
const clips = await projectRect(context, owner, ownerRect);
region = region.flatMap((polygon) => clips.map((clip) => clipPolygon(polygon, clip)));
}
}
return regionBounds(region);
}
+137
View File
@@ -0,0 +1,137 @@
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
import { type CdpTarget, cdpTargetKey } from "@/browser-driver/frame-graph";
import type { SessionManager } from "@/session-manager/manager";
import type { RpcError, ScrollToParams, ScrollToResult } from "@/transport/types";
import { attachDialogs, markDialogCursor } from "./dialogs";
import { cdpError, rpcError } from "./errors";
import { scrollElementAndFramesIntoView } from "./frame-geometry";
import { resolveBackendNode } from "./interaction";
import { scrollVisibleBounds } from "./scroll-visibility";
import {
type CdpRunner,
type ChromeTabsApi,
chromeTabsApi,
enforceAgentWindow,
isRpcError,
lookupSession,
resolveTargetTab,
sendToCdpTarget,
} from "./shared";
export interface ScrollToDeps {
cdp: CdpRunner;
tabsApi: ChromeTabsApi;
signal?: AbortSignal;
}
let defaultDeps: { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } | null = null;
function getDefaultDeps(): { cdp: ChromiumCdp; tabsApi: ChromeTabsApi } {
if (!defaultDeps) defaultDeps = { cdp: new ChromiumCdp(), tabsApi: chromeTabsApi };
return defaultDeps;
}
export async function handleScrollTo(
manager: SessionManager,
params: ScrollToParams,
deps: ScrollToDeps = getDefaultDeps(),
): Promise<ScrollToResult | RpcError> {
const ctxOrErr = lookupSession(manager, params, "scroll-to");
if (isRpcError(ctxOrErr)) return ctxOrErr;
const ctx = ctxOrErr;
const timeout = params.timeout_ms ?? 30_000;
if (!Number.isInteger(timeout) || timeout <= 0) {
return { code: "invalid_params", message: "timeout_ms must be a positive integer" };
}
const deadline = Date.now() + timeout;
const checkActive = () => {
if (deps.signal?.aborted) throw new DOMException("scroll-to aborted", "AbortError");
if (Date.now() >= deadline) throw new DOMException("scroll-to timed out", "TimeoutError");
};
const objectGroup = `bsk-scroll-${crypto.randomUUID()}`;
const objectTargets = new Map<string, CdpTarget>();
const send = async <T>(target: CdpTarget, method: string, args?: object): Promise<T> => {
checkActive();
// Also own objects allocated by the shared scroll fallback. Record the
// target before awaiting so cancellation cannot skip their cleanup.
if (method === "DOM.resolveNode") objectTargets.set(cdpTargetKey(target), target);
const result = await sendToCdpTarget<T>(
deps.cdp,
target,
method,
method === "DOM.resolveNode" ? { ...args, objectGroup } : args,
);
checkActive();
return result;
};
let graph: ReturnType<NonNullable<CdpRunner["getFrameGraph"]>> | undefined;
const cdp: CdpRunner = {
send: (tabId, method, args) => send({ tabId }, method, args),
sendToTarget: send,
trackSessionTab: deps.cdp.trackSessionTab?.bind(deps.cdp),
getFrameGraph: deps.cdp.getFrameGraph
? async (tabId) => {
checkActive();
graph ??= deps.cdp.getFrameGraph!(tabId);
const result = await graph;
checkActive();
return result;
}
: undefined,
};
try {
checkActive();
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
checkActive();
if (isRpcError(target)) return target;
const denied = enforceAgentWindow(ctx, target, "scroll-to");
if (denied) return denied;
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
const node = await resolveBackendNode(cdp, ctx, target, params, "scroll-to");
checkActive();
if (isRpcError(node)) return node;
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
const scrollError = await scrollElementAndFramesIntoView(
cdp,
target.tabId,
node.cdpTarget,
node.backendNodeId,
node.frameId,
);
checkActive();
if (scrollError) return scrollError;
const bounds = await scrollVisibleBounds(
cdp,
target.tabId,
{
target: node.cdpTarget,
backendNodeId: node.backendNodeId,
frameId: node.frameId,
},
deadline,
);
checkActive();
if (!bounds)
return rpcError(
"permission_denied",
"element_not_visible",
"scroll-to target has no visible area",
);
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
tab_id: target.tabId,
used_ref: node.usedRef,
used_selector: node.usedSelector,
...bounds,
});
} catch (error) {
if (deps.signal?.aborted) return { code: "cancelled", message: "scroll-to aborted" };
if (Date.now() >= deadline) return { code: "timeout", message: "scroll-to timed out" };
return cdpError(error);
} finally {
// Cleanup bypasses the cancellation guard and cannot replace the result.
for (const target of objectTargets.values()) {
await sendToCdpTarget(deps.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }).catch(
() => {},
);
}
}
}
@@ -655,6 +655,7 @@ function childSnapshot(frameId: string, backendNodeId: number) {
describe("OOPIF capture", () => {
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
const sendToTarget = vi.fn(async (target, method) => {
if (method === "Runtime.evaluate") return { result: { value: { width: 300, height: 200 } } };
if (method === "Page.getLayoutMetrics") {
return {
visualViewport: { clientWidth: 1000 },
+53
View File
@@ -470,6 +470,59 @@ export interface HoverResult {
dialogs?: JavaScriptDialogInfo[];
}
export interface ScrollToParams {
session_id: string;
ref?: string;
selector?: string;
tab_id?: number;
timeout_ms?: number;
}
export interface ScrollToResult {
tab_id: number;
used_ref?: string;
used_selector?: string;
/** Clipped border-box bounds in top-level viewport CSS pixels; not an occlusion test. */
x: number;
y: number;
width: number;
height: number;
dialogs?: JavaScriptDialogInfo[];
}
export interface FocusParams {
session_id: string;
ref?: string;
selector?: string;
tab_id?: number;
timeout_ms?: number;
}
export interface FocusResult {
tab_id: number;
used_ref?: string;
used_selector?: string;
focused: boolean;
dialogs?: JavaScriptDialogInfo[];
}
export interface BlurParams {
session_id: string;
ref?: string;
selector?: string;
tab_id?: number;
timeout_ms?: number;
}
export interface BlurResult {
tab_id: number;
used_ref?: string;
used_selector?: string;
was_focused: boolean;
focused: boolean;
dialogs?: JavaScriptDialogInfo[];
}
export interface FillParams {
session_id: string;
value: string;
+12 -2
View File
@@ -55,10 +55,20 @@ Use this default loop:
```text
bsk navigate <url> --session <id>
bsk observe --session <id>
bsk click|hover|fill|select|press ... --session <id>
bsk click|hover|scroll-to|focus|blur|fill|select|press ... --session <id>
bsk observe --session <id> # after navigation or a meaningful DOM change
```
`bsk scroll-to <ref-or-selector> --session <id>` scrolls an element and its frame owners into view.
Use a fresh element ref for iframe/shadow-root targets; CSS selectors search the main document.
The result is the visible border-box portion's bounds in top-level viewport CSS pixels after
ancestor clipping. Partial visibility is enough; hidden or fully clipped targets fail with
`permission_denied` and `data.reason=element_not_visible`. This does not test occlusion by other elements.
For a specific tab or deadline: `bsk scroll-to @e3 --session <id> --tab-id 42 --timeout 5s`.
`bsk focus <ref>` explicitly focuses a target; `bsk blur <ref>` removes focus and reports whether
it was focused. Use these for UI states triggered by focus changes.
Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also
make them stale. Observe again before the next interaction.
@@ -114,7 +124,7 @@ This list of names is complete. Never invent a command outside it; read
session start|stop|list browsers status doctor update logs
navigate navigate-back navigate-forward reload wait-for-navigation wait-ms
observe snapshot get-html screenshot console network
click hover fill select press evaluate
click hover scroll-to focus blur fill select press evaluate
tab list|create|close|select|borrow|return window resize emulate
upload download request-help record start|stop
```
+123 -3
View File
@@ -1,4 +1,5 @@
//! `bsk click` / `bsk fill` / `bsk press` (M7 interaction tools).
//! `bsk click` / `bsk focus` / `bsk blur` / `bsk fill` / `bsk press`
//! interaction tools.
//!
//! The `<target>` positional accepts either a snapshot ref (`@e3`,
//! `e3`) or a CSS selector. Use `--ref` / `--selector` explicitly when
@@ -10,8 +11,9 @@ use std::time::Duration;
use anyhow::Context;
use bsk_protocol::Method;
use bsk_protocol::tools::{
ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier,
MouseButton, PressParams, PressResult, SelectParams, SelectResult,
BlurParams, BlurResult, ClickParams, ClickResult, FillParams, FillResult, FocusParams,
FocusResult, HoverParams, HoverResult, KeyModifier, MouseButton, PressParams, PressResult,
SelectParams, SelectResult,
};
use clap::{Args, ValueEnum};
@@ -274,6 +276,124 @@ pub fn dispatch_hover(args: HoverArgs, format: Format) -> Result<(), CliError> {
Ok(())
}
// ---------------------------------------------------------------------------
// bsk focus / bsk blur
// ---------------------------------------------------------------------------
#[derive(Debug, Clone, Args)]
pub struct FocusArgs {
/// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used.
pub target: Option<String>,
#[arg(long = "ref")]
pub ref_: Option<String>,
#[arg(long = "selector")]
pub selector: Option<String>,
#[arg(long)]
pub session: String,
#[arg(long = "tab-id")]
pub tab_id: Option<i64>,
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
pub timeout: u32,
}
pub fn dispatch_focus(args: FocusArgs, format: Format) -> Result<(), CliError> {
let info = ensure_daemon().context("ensure daemon is running")?;
let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?;
let params = FocusParams {
session_id: args.session,
ref_,
selector,
tab_id: args.tab_id,
timeout_ms: Some(args.timeout),
};
let reply: FocusResult = call(
info.sock_path,
Method::ToolFocus,
params,
"focus-1",
args.timeout,
)?;
match format {
Format::Json => println!(
"{}",
serde_json::to_string_pretty(&reply)
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
),
Format::Human => {
let target =
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref());
println!(
"focus ok tab={} target={target} focused={}",
reply.tab_id, reply.focused
);
print_dialog_summaries(&reply.dialogs);
}
}
Ok(())
}
#[derive(Debug, Clone, Args)]
pub struct BlurArgs {
/// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used.
pub target: Option<String>,
#[arg(long = "ref")]
pub ref_: Option<String>,
#[arg(long = "selector")]
pub selector: Option<String>,
#[arg(long)]
pub session: String,
#[arg(long = "tab-id")]
pub tab_id: Option<i64>,
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
pub timeout: u32,
}
pub fn dispatch_blur(args: BlurArgs, format: Format) -> Result<(), CliError> {
let info = ensure_daemon().context("ensure daemon is running")?;
let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?;
let params = BlurParams {
session_id: args.session,
ref_,
selector,
tab_id: args.tab_id,
timeout_ms: Some(args.timeout),
};
let reply: BlurResult = call(
info.sock_path,
Method::ToolBlur,
params,
"blur-1",
args.timeout,
)?;
match format {
Format::Json => println!(
"{}",
serde_json::to_string_pretty(&reply)
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
),
Format::Human => {
let target =
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref());
println!(
"blur ok tab={} target={target} was_focused={} focused={}",
reply.tab_id, reply.was_focused, reply.focused
);
print_dialog_summaries(&reply.dialogs);
}
}
Ok(())
}
// ---------------------------------------------------------------------------
// bsk fill
// ---------------------------------------------------------------------------
+15 -1
View File
@@ -28,6 +28,7 @@ pub mod record_recovery;
pub mod record_state;
pub mod render_error;
pub mod screenshot;
pub mod scroll;
pub mod session;
pub mod snapshot;
pub mod status;
@@ -47,12 +48,15 @@ use crate::cli::evaluate::EvaluateArgs;
use crate::cli::get_html::GetHtmlArgs;
use crate::cli::human_loop::RequestHelpArgs;
use crate::cli::install_skill::InstallSkillArgs;
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs};
use crate::cli::interaction::{
BlurArgs, ClickArgs, FillArgs, FocusArgs, HoverArgs, PressArgs, SelectArgs,
};
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
use crate::cli::network::NetworkArgs;
use crate::cli::observe::ObserveArgs;
use crate::cli::record::RecordCmd;
use crate::cli::screenshot::ScreenshotArgs;
use crate::cli::scroll::ScrollToArgs;
use crate::cli::session::SessionCmd;
use crate::cli::snapshot::SnapshotArgs;
use crate::cli::tab::TabCmd;
@@ -173,6 +177,16 @@ pub enum Command {
/// Hover a snapshot ref or CSS selector.
Hover(HoverArgs),
/// Scroll a snapshot ref or CSS selector into the visible viewport.
#[command(name = "scroll-to")]
ScrollTo(ScrollToArgs),
/// Focus a snapshot ref or CSS selector.
Focus(FocusArgs),
/// Remove focus from a snapshot ref or CSS selector.
Blur(BlurArgs),
/// Fill an input / textarea / contenteditable.
Fill(FillArgs),
+100
View File
@@ -0,0 +1,100 @@
//! `bsk scroll-to` — bring a target element into the visible viewport.
use std::time::Duration;
use anyhow::Context;
use bsk_protocol::Method;
use bsk_protocol::tools::{ScrollToParams, ScrollToResult};
use clap::Args;
use crate::cli::dialogs::print_dialog_summaries;
use crate::cli::ensure_daemon::ensure_daemon;
use crate::cli::error::{CliError, Format};
use crate::cli::interaction::split_target;
use crate::cli::navigate::parse_timeout_ms;
#[derive(Debug, Clone, Args)]
pub struct ScrollToArgs {
/// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used.
pub target: Option<String>,
#[arg(long = "ref")]
pub ref_: Option<String>,
#[arg(long = "selector")]
pub selector: Option<String>,
#[arg(long)]
pub session: String,
#[arg(long = "tab-id")]
pub tab_id: Option<i64>,
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
pub timeout: u32,
}
pub fn dispatch(args: ScrollToArgs, format: Format) -> Result<(), CliError> {
let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?;
let info = ensure_daemon().context("ensure daemon is running")?;
let params = ScrollToParams {
session_id: args.session,
ref_,
selector,
tab_id: args.tab_id,
timeout_ms: Some(args.timeout),
};
let reply: ScrollToResult = crate::cli::business_rpc::call(
info.sock_path,
"scroll-to-1",
Method::ToolScrollTo,
Some(params),
ipc_timeout(args.timeout),
)?;
match format {
Format::Json => println!(
"{}",
serde_json::to_string_pretty(&reply)
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
),
Format::Human => {
let target =
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref());
println!(
"scroll-to ok tab={} target={target} bounds=({}, {}, {}, {})",
reply.tab_id, reply.x, reply.y, reply.width, reply.height
);
print_dialog_summaries(&reply.dialogs);
}
}
Ok(())
}
fn format_used_target(used_ref: Option<&str>, used_selector: Option<&str>) -> String {
used_ref
.map(|r| format!("@{r}"))
.or_else(|| used_selector.map(str::to_string))
.unwrap_or_else(|| "?".into())
}
fn ipc_timeout(timeout_ms: u32) -> Duration {
Duration::from_millis(u64::from(timeout_ms))
.checked_add(Duration::from_secs(15))
.unwrap_or(Duration::from_secs(u64::from(timeout_ms / 1_000) + 15))
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn split_target_detects_refs_and_selectors() {
let (ref_, selector) = split_target(Some("@e3".into()), None, None).unwrap();
assert_eq!(ref_.as_deref(), Some("@e3"));
assert!(selector.is_none());
let (ref_, selector) = split_target(Some("#target".into()), None, None).unwrap();
assert!(ref_.is_none());
assert_eq!(selector.as_deref(), Some("#target"));
}
}
+3
View File
@@ -253,6 +253,9 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
| Method::ToolReload
| Method::ToolClick
| Method::ToolHover
| Method::ToolScrollTo
| Method::ToolFocus
| Method::ToolBlur
| Method::ToolFill
| Method::ToolPress
| Method::ToolSelect
+3
View File
@@ -94,6 +94,9 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
Command::Reload(args) => cli::navigate::dispatch_reload(args, format),
Command::Click(args) => cli::interaction::dispatch_click(args, format),
Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
Command::ScrollTo(args) => cli::scroll::dispatch(args, format),
Command::Focus(args) => cli::interaction::dispatch_focus(args, format),
Command::Blur(args) => cli::interaction::dispatch_blur(args, format),
Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
Command::Press(args) => cli::interaction::dispatch_press(args, format),
Command::Select(args) => cli::interaction::dispatch_select(args, format),
+59
View File
@@ -255,6 +255,65 @@ fn parses_hover_with_settle() {
assert_eq!(args.settle, 300);
}
#[test]
fn parses_scroll_to_target() {
let cli = parse(&["bsk", "scroll-to", "@e2", "--session", "s1"]);
let Command::ScrollTo(args) = cli.command else {
panic!("expected scroll-to command");
};
assert_eq!(args.target.as_deref(), Some("@e2"));
}
#[test]
fn parses_scroll_to_explicit_target_tab_and_timeout() {
for flag in ["--ref", "--selector"] {
let cli = parse(&[
"bsk",
"scroll-to",
flag,
"e3",
"--session",
"s1",
"--tab-id",
"42",
"--timeout",
"5s",
]);
let Command::ScrollTo(args) = cli.command else {
panic!("expected scroll-to command");
};
assert_eq!(args.tab_id, Some(42));
assert_eq!(args.timeout, 5_000);
assert_eq!(
if flag == "--ref" {
args.ref_
} else {
args.selector
}
.as_deref(),
Some("e3")
);
}
}
#[test]
fn parses_focus_target() {
let cli = parse(&["bsk", "focus", "@e2", "--session", "s1"]);
let Command::Focus(args) = cli.command else {
panic!("expected focus command");
};
assert_eq!(args.target.as_deref(), Some("@e2"));
}
#[test]
fn parses_blur_selector() {
let cli = parse(&["bsk", "blur", "--selector", "#search", "--session", "s1"]);
let Command::Blur(args) = cli.command else {
panic!("expected blur command");
};
assert_eq!(args.selector.as_deref(), Some("#search"));
}
#[test]
fn rejects_zero_click_count() {
assert!(
+23 -13
View File
@@ -293,6 +293,15 @@ async fn session_user_interrupt_event_cancels_inflight_with_user_aborted() {
/// catch a tool dispatched arbitrarily later.
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle() {
assert_idle_interrupt_rejects(Method::ToolClick).await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
async fn user_interrupt_rejects_scroll_to_before_forwarding() {
assert_idle_interrupt_rejects(Method::ToolScrollTo).await;
}
async fn assert_idle_interrupt_rejects(method: Method) {
let (handle, sock) = spawn_daemon().await;
let mut ws = connect_ext(handle.ws_addr()).await;
let _ = handshake_as_ext(&mut ws).await;
@@ -301,10 +310,11 @@ async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle()
// Fake extension: answer tool.session_start, observe other
// requests but never reply. We expect the daemon to reject the
// tool call BEFORE it ever forwards a tool.click frame.
// tool call BEFORE it ever forwards a browser-input frame.
let ws_sink_for_responder = Arc::clone(&ws_sink);
let observed_click_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let observed_click_count_clone = Arc::clone(&observed_click_count);
let observed_call_count = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let observed_call_count_clone = Arc::clone(&observed_call_count);
let forwarded_method = method.clone();
let responder = tokio::spawn(async move {
let mut ws_stream = ws_stream;
while let Some(Ok(msg)) = ws_stream.next().await {
@@ -329,8 +339,8 @@ async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle()
g.send(Message::Text(serde_json::to_string(&reply).unwrap()))
.await
.unwrap();
} else if req.method == Method::ToolClick {
observed_click_count_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
} else if req.method == forwarded_method {
observed_call_count_clone.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
// Deliberately do NOT reply — daemon should reject before
// ever forwarding to us.
let _ = req;
@@ -377,12 +387,12 @@ async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle()
let state = handle.state();
wait_for_session_interrupt_pending(&state, &start.session_id).await;
// The CLI now issues a tool.click (mutating). The daemon must
// The CLI now issues a browser-input (mutating). The daemon must
// reject it WITHOUT forwarding to the extension.
let click_outcome = ipc
let outcome = ipc
.call::<_, serde_json::Value>(
"click-after-interrupt",
Method::ToolClick,
"input-after-interrupt",
method,
Some(json!({
"session_id": start.session_id,
"ref": "fake-ref-1",
@@ -391,7 +401,7 @@ async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle()
)
.await
.unwrap();
let err = click_outcome.expect_err("tool.click must be rejected");
let err = outcome.expect_err("browser-input must be rejected");
assert_eq!(
err.code,
ErrorCode::UserAborted,
@@ -399,11 +409,11 @@ async fn user_interrupt_rejects_next_mutating_tool_call_when_session_was_idle()
err
);
// Crucially: the extension never saw a tool.click frame.
// Crucially: the extension never saw a browser-input frame.
assert_eq!(
observed_click_count.load(std::sync::atomic::Ordering::SeqCst),
observed_call_count.load(std::sync::atomic::Ordering::SeqCst),
0,
"tool.click must NOT have been forwarded to the extension"
"browser-input must NOT have been forwarded to the extension"
);
responder.abort();
+138 -6
View File
@@ -1,5 +1,4 @@
//! M7 integration: drive the 7 new tool RPCs (navigate /
//! navigate_back / navigate_forward / reload / click / fill / press)
//! M7 integration: drive interaction and navigation tool RPCs
//! through the IPC + per-session queue + fake extension and assert
//! the wire shapes line up. The extension stub mirrors each method's
//! `*Result` so we exercise the daemon's serialise → forward →
@@ -13,11 +12,13 @@ use bsk::daemon::{self, DaemonConfig};
use bsk::ipc_client::IpcClient;
use bsk_protocol::system::{HandshakeParams, HandshakeResult};
use bsk_protocol::tools::{
ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, NavigateBackParams,
NavigateBackResult, NavigateForwardParams, NavigateForwardResult, NavigateParams,
NavigateResult, PressParams, PressResult, ReloadParams, ReloadResult, SelectParams,
SelectResult, SessionStartParams, SessionStartResult, WaitUntil,
BlurParams, BlurResult, ClickParams, ClickResult, FillParams, FillResult, FocusParams,
FocusResult, KeyModifier, MouseButton, NavigateBackParams, NavigateBackResult,
NavigateForwardParams, NavigateForwardResult, NavigateParams, NavigateResult, PressParams,
PressResult, ReloadParams, ReloadResult, SelectParams, SelectResult, SessionStartParams,
SessionStartResult, WaitUntil,
};
use bsk_protocol::tools::{ScrollToParams, ScrollToResult};
use bsk_protocol::{
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
};
@@ -382,6 +383,137 @@ async fn click_round_trips_ref_and_modifiers() {
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn focus_round_trips_verified_state() {
let (handle, sock) = spawn_daemon().await;
let mut ws = connect_ext(handle.ws_addr()).await;
let _ = do_handshake(&mut ws).await;
run_extension(ws, |req| {
assert_eq!(req.method, Method::ToolFocus);
let params = req.params.clone().unwrap();
let p: FocusParams = serde_json::from_value(params).unwrap();
assert_eq!(p.ref_.as_deref(), Some("@e3"));
ResponseBody::Ok(
serde_json::to_value(FocusResult {
tab_id: 9,
used_ref: Some("e3".into()),
used_selector: None,
focused: true,
dialogs: vec![],
})
.unwrap(),
)
});
let session_id = ipc_session_start(&sock).await;
let result: FocusResult = ipc_tool_call(
&sock,
Method::ToolFocus,
FocusParams {
session_id,
ref_: Some("@e3".into()),
selector: None,
tab_id: None,
timeout_ms: Some(5_000),
},
)
.await
.expect("focus ok");
assert!(result.focused);
assert_eq!(result.used_ref.as_deref(), Some("e3"));
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn blur_round_trips_previous_focus_state() {
let (handle, sock) = spawn_daemon().await;
let mut ws = connect_ext(handle.ws_addr()).await;
let _ = do_handshake(&mut ws).await;
run_extension(ws, |req| {
assert_eq!(req.method, Method::ToolBlur);
let params = req.params.clone().unwrap();
let p: BlurParams = serde_json::from_value(params).unwrap();
assert_eq!(p.selector.as_deref(), Some("#search"));
ResponseBody::Ok(
serde_json::to_value(BlurResult {
tab_id: 9,
used_ref: None,
used_selector: Some("#search".into()),
was_focused: true,
focused: false,
dialogs: vec![],
})
.unwrap(),
)
});
let session_id = ipc_session_start(&sock).await;
let result: BlurResult = ipc_tool_call(
&sock,
Method::ToolBlur,
BlurParams {
session_id,
ref_: None,
selector: Some("#search".into()),
tab_id: None,
timeout_ms: Some(5_000),
},
)
.await
.expect("blur ok");
assert!(result.was_focused);
assert!(!result.focused);
assert_eq!(result.used_selector.as_deref(), Some("#search"));
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn scroll_to_round_trips_visible_bounds() {
let (handle, sock) = spawn_daemon().await;
let mut ws = connect_ext(handle.ws_addr()).await;
let _ = do_handshake(&mut ws).await;
run_extension(ws, |req| {
assert_eq!(req.method, Method::ToolScrollTo);
let params = req.params.clone().unwrap();
let parsed: ScrollToParams = serde_json::from_value(params).unwrap();
assert_eq!(parsed.ref_.as_deref(), Some("@e3"));
ResponseBody::Ok(
serde_json::to_value(ScrollToResult {
tab_id: 9,
used_ref: Some("e3".into()),
used_selector: None,
x: 10.0,
y: 20.0,
width: 100.0,
height: 40.0,
dialogs: vec![],
})
.unwrap(),
)
});
let session_id = ipc_session_start(&sock).await;
let result: ScrollToResult = ipc_tool_call(
&sock,
Method::ToolScrollTo,
ScrollToParams {
session_id,
ref_: Some("@e3".into()),
selector: None,
tab_id: None,
timeout_ms: Some(5_000),
},
)
.await
.expect("scroll-to ok");
assert_eq!(result.used_ref.as_deref(), Some("e3"));
assert_eq!(result.width, 100.0);
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn fill_round_trips_clear_before_default() {
let (handle, sock) = spawn_daemon().await;
@@ -0,0 +1,42 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "BlurParams",
"type": "object",
"required": [
"session_id"
],
"properties": {
"ref": {
"description": "Optional `@e<N>` ref allocated by the last observation. Mutually exclusive with `selector`.",
"type": [
"string",
"null"
]
},
"selector": {
"type": [
"string",
"null"
]
},
"session_id": {
"type": "string"
},
"tab_id": {
"description": "Target tab. Defaults to the Agent Window's active tab.",
"type": [
"integer",
"null"
],
"format": "int64"
},
"timeout_ms": {
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 1.0
}
}
}
@@ -0,0 +1,112 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "BlurResult",
"type": "object",
"required": [
"focused",
"tab_id",
"was_focused"
],
"properties": {
"dialogs": {
"type": "array",
"items": {
"$ref": "#/definitions/JavaScriptDialogInfo"
}
},
"focused": {
"description": "Always false for a successful blur; included for machine verification.",
"type": "boolean"
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"used_ref": {
"type": [
"string",
"null"
]
},
"used_selector": {
"type": [
"string",
"null"
]
},
"was_focused": {
"description": "Whether the target held focus before the call.",
"type": "boolean"
}
},
"definitions": {
"JavaScriptDialogHandledAction": {
"description": "How the extension resolved the dialog so CDP could continue.",
"type": "string",
"enum": [
"accepted",
"dismissed"
]
},
"JavaScriptDialogInfo": {
"description": "One observed + handled JavaScript dialog during a tool call.",
"type": "object",
"required": [
"handled",
"message",
"sequence",
"tab_id",
"type"
],
"properties": {
"default_prompt": {
"type": [
"string",
"null"
]
},
"handled": {
"$ref": "#/definitions/JavaScriptDialogHandledAction"
},
"has_browser_handler": {
"type": [
"boolean",
"null"
]
},
"message": {
"type": "string"
},
"sequence": {
"description": "Monotonic per-tab sequence for ordering within a session.",
"type": "integer",
"format": "uint64",
"minimum": 0.0
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"type": {
"$ref": "#/definitions/JavaScriptDialogType"
},
"url": {
"type": [
"string",
"null"
]
}
}
},
"JavaScriptDialogType": {
"description": "Native JS dialog kind reported by CDP.",
"type": "string",
"enum": [
"alert",
"confirm",
"prompt",
"beforeunload"
]
}
}
}
@@ -0,0 +1,42 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "FocusParams",
"type": "object",
"required": [
"session_id"
],
"properties": {
"ref": {
"description": "Optional `@e<N>` ref allocated by the last observation. Mutually exclusive with `selector`.",
"type": [
"string",
"null"
]
},
"selector": {
"type": [
"string",
"null"
]
},
"session_id": {
"type": "string"
},
"tab_id": {
"description": "Target tab. Defaults to the Agent Window's active tab.",
"type": [
"integer",
"null"
],
"format": "int64"
},
"timeout_ms": {
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 1.0
}
}
}
@@ -0,0 +1,107 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "FocusResult",
"type": "object",
"required": [
"focused",
"tab_id"
],
"properties": {
"dialogs": {
"type": "array",
"items": {
"$ref": "#/definitions/JavaScriptDialogInfo"
}
},
"focused": {
"description": "Whether the target holds DOM focus in its document or shadow root after the call.",
"type": "boolean"
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"used_ref": {
"type": [
"string",
"null"
]
},
"used_selector": {
"type": [
"string",
"null"
]
}
},
"definitions": {
"JavaScriptDialogHandledAction": {
"description": "How the extension resolved the dialog so CDP could continue.",
"type": "string",
"enum": [
"accepted",
"dismissed"
]
},
"JavaScriptDialogInfo": {
"description": "One observed + handled JavaScript dialog during a tool call.",
"type": "object",
"required": [
"handled",
"message",
"sequence",
"tab_id",
"type"
],
"properties": {
"default_prompt": {
"type": [
"string",
"null"
]
},
"handled": {
"$ref": "#/definitions/JavaScriptDialogHandledAction"
},
"has_browser_handler": {
"type": [
"boolean",
"null"
]
},
"message": {
"type": "string"
},
"sequence": {
"description": "Monotonic per-tab sequence for ordering within a session.",
"type": "integer",
"format": "uint64",
"minimum": 0.0
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"type": {
"$ref": "#/definitions/JavaScriptDialogType"
},
"url": {
"type": [
"string",
"null"
]
}
}
},
"JavaScriptDialogType": {
"description": "Native JS dialog kind reported by CDP.",
"type": "string",
"enum": [
"alert",
"confirm",
"prompt",
"beforeunload"
]
}
}
}
@@ -0,0 +1,42 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "ScrollToParams",
"type": "object",
"required": [
"session_id"
],
"properties": {
"ref": {
"description": "Optional `@e<N>` ref allocated by the last observation. Mutually exclusive with `selector`.",
"type": [
"string",
"null"
]
},
"selector": {
"type": [
"string",
"null"
]
},
"session_id": {
"type": "string"
},
"tab_id": {
"description": "Target tab. Defaults to the Agent Window's active tab.",
"type": [
"integer",
"null"
],
"format": "int64"
},
"timeout_ms": {
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 1.0
}
}
}
@@ -0,0 +1,123 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "ScrollToResult",
"type": "object",
"required": [
"height",
"tab_id",
"width",
"x",
"y"
],
"properties": {
"dialogs": {
"type": "array",
"items": {
"$ref": "#/definitions/JavaScriptDialogInfo"
}
},
"height": {
"type": "number",
"format": "double"
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"used_ref": {
"type": [
"string",
"null"
]
},
"used_selector": {
"type": [
"string",
"null"
]
},
"width": {
"type": "number",
"format": "double"
},
"x": {
"description": "Bounding rectangle of the element's visible border-box portion, in top-level viewport CSS pixels, after ancestor and viewport clipping. Partial visibility is sufficient. This is not an occlusion or hit test.",
"type": "number",
"format": "double"
},
"y": {
"type": "number",
"format": "double"
}
},
"definitions": {
"JavaScriptDialogHandledAction": {
"description": "How the extension resolved the dialog so CDP could continue.",
"type": "string",
"enum": [
"accepted",
"dismissed"
]
},
"JavaScriptDialogInfo": {
"description": "One observed + handled JavaScript dialog during a tool call.",
"type": "object",
"required": [
"handled",
"message",
"sequence",
"tab_id",
"type"
],
"properties": {
"default_prompt": {
"type": [
"string",
"null"
]
},
"handled": {
"$ref": "#/definitions/JavaScriptDialogHandledAction"
},
"has_browser_handler": {
"type": [
"boolean",
"null"
]
},
"message": {
"type": "string"
},
"sequence": {
"description": "Monotonic per-tab sequence for ordering within a session.",
"type": "integer",
"format": "uint64",
"minimum": 0.0
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"type": {
"$ref": "#/definitions/JavaScriptDialogType"
},
"url": {
"type": [
"string",
"null"
]
}
}
},
"JavaScriptDialogType": {
"description": "Native JS dialog kind reported by CDP.",
"type": "string",
"enum": [
"alert",
"confirm",
"prompt",
"beforeunload"
]
}
}
}
@@ -79,6 +79,12 @@ fn main() {
dump!(ClickResult, "tool_click_result");
dump!(HoverParams, "tool_hover_params");
dump!(HoverResult, "tool_hover_result");
dump!(ScrollToParams, "tool_scroll_to_params");
dump!(ScrollToResult, "tool_scroll_to_result");
dump!(FocusParams, "tool_focus_params");
dump!(FocusResult, "tool_focus_result");
dump!(BlurParams, "tool_blur_params");
dump!(BlurResult, "tool_blur_result");
dump!(FillParams, "tool_fill_params");
dump!(FillResult, "tool_fill_result");
dump!(PressParams, "tool_press_params");
+18
View File
@@ -72,6 +72,12 @@ pub enum Method {
ToolClick,
#[serde(rename = "tool.hover")]
ToolHover,
#[serde(rename = "tool.scroll_to")]
ToolScrollTo,
#[serde(rename = "tool.focus")]
ToolFocus,
#[serde(rename = "tool.blur")]
ToolBlur,
#[serde(rename = "tool.fill")]
ToolFill,
#[serde(rename = "tool.press")]
@@ -168,6 +174,9 @@ impl Method {
| Method::ToolNavigateForward
| Method::ToolReload
| Method::ToolClick
| Method::ToolScrollTo
| Method::ToolFocus
| Method::ToolBlur
| Method::ToolFill
| Method::ToolPress
| Method::ToolSelect
@@ -305,6 +314,9 @@ mod tests {
assert!(Method::ToolNavigateForward.is_mutating());
assert!(Method::ToolReload.is_mutating());
assert!(Method::ToolClick.is_mutating());
assert!(Method::ToolScrollTo.is_mutating());
assert!(Method::ToolFocus.is_mutating());
assert!(Method::ToolBlur.is_mutating());
assert!(Method::ToolFill.is_mutating());
assert!(Method::ToolPress.is_mutating());
assert!(Method::ToolSelect.is_mutating());
@@ -349,6 +361,9 @@ mod tests {
assert_eq!(Method::ToolHover.effect(), MethodEffect::TransientInput);
assert_eq!(Method::ToolObserve.effect(), MethodEffect::TransientInput);
assert_eq!(Method::ToolClick.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::ToolScrollTo.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::ToolFocus.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::ToolBlur.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::Cancel.effect(), MethodEffect::ControlPlane);
}
@@ -358,6 +373,9 @@ mod tests {
assert!(Method::ToolHover.requires_interrupt_gate());
assert!(Method::ToolObserve.requires_interrupt_gate());
assert!(Method::ToolClick.requires_interrupt_gate());
assert!(Method::ToolScrollTo.requires_interrupt_gate());
assert!(Method::ToolFocus.requires_interrupt_gate());
assert!(Method::ToolBlur.requires_interrupt_gate());
assert!(!Method::Cancel.requires_interrupt_gate());
}
}
+109 -2
View File
@@ -1,5 +1,5 @@
//! DOM interaction tools (`tool.click`, `tool.fill`, `tool.press`,
//! `tool.select`).
//! DOM interaction tools (`tool.click`, `tool.hover`, `tool.focus`,
//! `tool.blur`, `tool.fill`, `tool.press`, `tool.select`).
//!
//! Element-targeted tools accept either a snapshot `ref` (`@e<N>` form,
//! normalised against the session's RefStore) **or** a CSS selector
@@ -131,6 +131,82 @@ pub struct HoverResult {
pub dialogs: Vec<JavaScriptDialogInfo>,
}
// ---------------------------------------------------------------------------
// focus / blur
// ---------------------------------------------------------------------------
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct FocusParams {
pub session_id: String,
/// Optional `@e<N>` ref allocated by the last observation.
/// Mutually exclusive with `selector`.
#[serde(
rename = "ref",
alias = "ref_",
default,
skip_serializing_if = "Option::is_none"
)]
pub ref_: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selector: Option<String>,
/// Target tab. Defaults to the Agent Window's active tab.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tab_id: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 1))]
pub timeout_ms: Option<u32>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct FocusResult {
pub tab_id: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_ref: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_selector: Option<String>,
/// Whether the target holds DOM focus in its document or shadow root after the call.
pub focused: bool,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dialogs: Vec<JavaScriptDialogInfo>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct BlurParams {
pub session_id: String,
/// Optional `@e<N>` ref allocated by the last observation.
/// Mutually exclusive with `selector`.
#[serde(
rename = "ref",
alias = "ref_",
default,
skip_serializing_if = "Option::is_none"
)]
pub ref_: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selector: Option<String>,
/// Target tab. Defaults to the Agent Window's active tab.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tab_id: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 1))]
pub timeout_ms: Option<u32>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct BlurResult {
pub tab_id: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_ref: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_selector: Option<String>,
/// Whether the target held focus before the call.
pub was_focused: bool,
/// Always false for a successful blur; included for machine verification.
pub focused: bool,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dialogs: Vec<JavaScriptDialogInfo>,
}
// ---------------------------------------------------------------------------
// fill
// ---------------------------------------------------------------------------
@@ -325,6 +401,37 @@ mod tests {
assert_eq!(round, p);
}
#[test]
fn focus_params_serialise_ref_field_name() {
let p = FocusParams {
session_id: "abcd".into(),
ref_: Some("@e4".into()),
selector: None,
tab_id: Some(42),
timeout_ms: Some(5_000),
};
let v = serde_json::to_value(&p).unwrap();
assert_eq!(v.get("ref").and_then(|v| v.as_str()), Some("@e4"));
assert!(v.get("ref_").is_none());
let round: FocusParams = serde_json::from_value(v).unwrap();
assert_eq!(round, p);
}
#[test]
fn blur_result_round_trips_focus_state() {
let result = BlurResult {
tab_id: 42,
used_ref: Some("e4".into()),
used_selector: None,
was_focused: true,
focused: false,
dialogs: vec![],
};
let value = serde_json::to_value(&result).unwrap();
let round: BlurResult = serde_json::from_value(value).unwrap();
assert_eq!(round, result);
}
#[test]
fn modifiers_render_as_lowercase_strings() {
let v = serde_json::to_value(KeyModifier::Ctrl).unwrap();
+2
View File
@@ -14,6 +14,7 @@ mod record_common;
mod record_v2;
mod record_v3;
pub mod script;
pub mod scroll;
pub mod session;
pub mod tabs;
pub mod waits;
@@ -30,6 +31,7 @@ pub use network::*;
pub use observation::*;
pub use record::*;
pub use script::*;
pub use scroll::*;
pub use session::*;
pub use tabs::*;
pub use waits::*;
+67
View File
@@ -0,0 +1,67 @@
//! Element scrolling primitive (`tool.scroll_to`).
use schemars::JsonSchema;
use serde::{Deserialize, Serialize};
use super::JavaScriptDialogInfo;
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct ScrollToParams {
pub session_id: String,
/// Optional `@e<N>` ref allocated by the last observation.
/// Mutually exclusive with `selector`.
#[serde(
rename = "ref",
alias = "ref_",
default,
skip_serializing_if = "Option::is_none"
)]
pub ref_: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selector: Option<String>,
/// Target tab. Defaults to the Agent Window's active tab.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tab_id: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 1))]
pub timeout_ms: Option<u32>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct ScrollToResult {
pub tab_id: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_ref: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_selector: Option<String>,
/// Bounding rectangle of the element's visible border-box portion, in
/// top-level viewport CSS pixels, after ancestor and viewport clipping.
/// Partial visibility is sufficient. This is not an occlusion or hit test.
pub x: f64,
pub y: f64,
pub width: f64,
pub height: f64,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dialogs: Vec<JavaScriptDialogInfo>,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn params_serialise_ref_field_name() {
let params = ScrollToParams {
session_id: "abcd".into(),
ref_: Some("@e3".into()),
selector: None,
tab_id: Some(42),
timeout_ms: Some(5_000),
};
let value = serde_json::to_value(&params).unwrap();
assert_eq!(value.get("ref").and_then(|v| v.as_str()), Some("@e3"));
assert!(value.get("ref_").is_none());
let round: ScrollToParams = serde_json::from_value(value).unwrap();
assert_eq!(round, params);
}
}
+5
View File
@@ -84,6 +84,11 @@ Shared Rust types + JSON Schema generation. TypeScript mirrors frame shapes in
4. Extension dispatcher validates sandbox rules, invokes CDP via `BrowserDriver`.
5. Response travels CLI ← daemon ← extension; CLI prints result and exits.
The [scroll-to primitive reference](scroll-to.md) documents `tool.scroll_to`,
including its CLI/plugin mappings, visible-bounds contract and cancellation
behavior. It follows the same routing path and is classified as a browser
mutation for session queueing and user-interruption gating.
## Session and sandbox model
- **Session** = opaque ID (4 lowercase letters in v0.1) + dedicated **Agent Window**
+131
View File
@@ -0,0 +1,131 @@
# Scroll-to primitive
`scroll-to` brings an existing element and its containing frames into view, then
returns the visible portion's bounds. Use it to reveal a target before inspecting
the page or taking a screenshot. It accepts an element target rather than a pixel
distance or scroll direction.
| Entry point | Name |
| --- | --- |
| CLI | `bsk scroll-to` |
| Wire protocol | `tool.scroll_to` |
| DeepSeek Harness | `browser_interact` with `action: "scroll-to"` |
## CLI
Use a session you started and a fresh ref from an observation. The ids below are
examples; substitute the session, tab and ref returned by your own calls.
```sh
bsk scroll-to @e3 --session abcd --tab-id 42 --timeout 5s --json
bsk scroll-to --selector '#details' --session abcd
```
Supply exactly one target: a positional ref or selector, `--ref`, or `--selector`.
Refs accept `@e3` and `e3`. CSS selectors search only the main document; use refs
for elements in iframes (including out-of-process iframes) or shadow roots.
Refs belong to one session and tab and may become stale after page changes.
`--session` is required. `--tab-id` defaults to the Agent Window's active tab.
`--timeout` defaults to `30s` and must be positive; durations such as `5000ms` and
`5s` are accepted. The command operates on Agent Window tabs, including user tabs
explicitly borrowed into that window.
Example JSON output:
```json
{
"tab_id": 42,
"used_ref": "e3",
"x": 10,
"y": 20,
"width": 100,
"height": 80
}
```
`used_ref` or `used_selector` identifies the resolved target. The optional
`dialogs` array reports JavaScript dialogs handled during the call.
## Result contract
`x` and `y` are the upper-left corner, and `width` and `height` are the size of
the visible border-box portion's bounding rectangle. All four values use
**top-level viewport CSS pixels**, after scrolling and clipping by ancestor
containers and frame viewports. They are not document or screenshot pixel
coordinates.
Partial visibility is sufficient: a 400px-high element clipped by an 80px-high
scroll container can succeed with `height: 80`. Hidden or fully clipped targets
fail. The rectangle is not an occlusion or hit test: another element can cover
the target, and the rectangle's center is not a guaranteed click point. Layout
can change after the result; use fresh refs for subsequent interactions.
## Protocol and plugin
The CLI sends a normal request through the daemon to the extension:
```json
{
"id": "scroll-1",
"method": "tool.scroll_to",
"params": {
"session_id": "abcd",
"ref": "@e3",
"tab_id": 42,
"timeout_ms": 5000
}
}
```
Protocol callers must supply exactly one nonempty `ref` or `selector`.
`tab_id` is optional; `timeout_ms` is a positive integer and defaults to 30000.
These target constraints are enforced by the handler. See the generated
[parameter schema](../crates/bsk-protocol/schema/tool_scroll_to_params.json) and
[result schema](../crates/bsk-protocol/schema/tool_scroll_to_result.json) for field
types. The wire response wraps the result above in `{ "id": "scroll-1", "result": ... }`.
The plugin exposes the same action with its usual camelCase parameters:
```json
{
"action": "scroll-to",
"session": "abcd",
"target": "@e3",
"tabId": 42,
"timeoutMs": 5000
}
```
Call this through `browser_interact` using a plugin-owned session. Its result
contains `session`, `tabId`, `x`, `y`, `width` and `height` with the same bounds
semantics. Omitted `session` uses the plugin's current session.
## Errors and interruption
The wire response uses the normal `error.code`, `error.message` and optional
`error.data.reason` fields. Common cases are:
| Code | `data.reason` | Meaning |
| --- | --- | --- |
| `invalid_params` | — | Missing/conflicting target or invalid timeout/tab id |
| `not_found` | `ref_not_found` | Ref is unknown, expired or belongs to another tab |
| `not_found` | `selector_not_found` | Main-document selector matched no element |
| `permission_denied` | `agent_window_scope` | Target tab has not been borrowed into the Agent Window |
| `permission_denied` | `element_not_visible` | No visible area remains after scrolling |
| `cancelled` | — | The call was cancelled |
| `timeout` | — | The action deadline expired |
| `cdp_failed` | varies | Browser command, frame geometry or visibility measurement failed |
The protocol classifies scroll-to as a browser mutation, so it uses the existing
per-session queue and user-interruption gate. The extension releases any retained
hover state before dispatch. Cancellation and deadline checks surround each CDP
step, including frame lookup, scrolling and measurement. Once detected, they
prevent further scrolling and a successful response; remote-object cleanup still
runs. Already performed scrolling is not rolled back, so inspect the current
page before retrying an interrupted call.
The implementation is scoped to scroll-to: it reuses frame scrolling and
coordinate projection, with a separate visibility measurement that accounts for
ancestor clipping and hidden frame owners. Shared click, hover and screenshot
geometry is unchanged.
@@ -0,0 +1,41 @@
# OOPIF occupied-scrollbar regression
An iframe's content quad includes its child viewport's scrollbar space, while
`Page.getLayoutMetrics().cssLayoutViewport` excludes that space. Mapping the latter
onto the whole quad stretches positions and sizes. The full target-local viewport
must determine scale; the visible viewport must still clip content at every OOPIF
boundary and constrain target-local action points.
This fixture alternates loopback hostnames across two nested OOPIFs. It includes
scrolling, borders, padding, scaled iframe owners, a partially clipped button and
a fully clipped button. Custom scrollbars occupy different widths and heights.
The `scrollbars` query parameter accepts `both` (default), `vertical`, `horizontal`
or `none`.
Run the numerical regression with Node 22+ and a local Chrome executable:
```sh
BSK_GEOMETRY_CHROME=/path/to/chrome pnpm --filter @browser-skill/extension exec vitest run \
src/tools/__tests__/snapshot-coordinates.browser.test.ts
```
The existing browser runner is shared with the snapshot-unit regression. The test:
- Verifies actual OOPIF targets and occupied scrollbar dimensions.
- Compares snapshot and live geometry with independent DOM rectangles and clipping.
- Dispatches real root-target clicks and verifies the intended frame/button received them.
- Rejects fully clipped controls and checks snapshot measurement reuse per target.
- Covers five device-scale/browser-zoom combinations plus single-axis and no-scrollbar cases.
The runner creates and removes isolated browser profiles. It is opt-in and skipped
in normal unit runs when `BSK_GEOMETRY_CHROME` is unset.
Run the CLI fixture smoke with a connected test extension:
```sh
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --case oopif-scrollbars --bsk ./target/debug/bsk
```
The smoke assertions verify both nested frames have occupied scrollbars, that the
marker is observed and that the session is closed. Numeric geometry and actual
click assertions are exercised by the browser test above, not by smoke alone.
@@ -0,0 +1,39 @@
{
"$schema": "../../../schemas/case.schema.json",
"schemaVersion": 1,
"id": "oopif-scrollbars",
"title": "Nested OOPIFs with occupied scrollbars and clipped controls",
"suite": "regression",
"tags": ["geometry", "iframe", "scroll"],
"fixture": { "startPath": "/oopif-scrollbars" },
"prompts": {
"en": "Open {url}, observe the page and its frames, report the OOPIF-SCROLLBARS marker, then close the session.",
"zh-CN": "打开 {url},观察页面及其 iframe,报告 OOPIF-SCROLLBARS 标记,然后关闭会话。"
},
"coverage": ["session.start", "session.stop", "page.navigate", "inspect.observe"],
"assertions": {
"site": [
{
"label": "nested frame fixture settled",
"type": "geometry.ready",
"where": { "data.root": true },
"minCount": 1
},
{
"label": "both child frames have occupied scrollbars",
"type": "geometry.scrollbars",
"where": { "data.vertical": true, "data.horizontal": true },
"minCount": 2
}
],
"response": [{ "label": "observed the fixture marker", "includes": "OOPIF-SCROLLBARS" }],
"adapter": [{ "label": "browser session was closed", "key": "sessionStopped" }]
},
"smoke": {
"steps": [
{ "action": "navigate" },
{ "action": "wait-site-event", "type": "geometry.ready", "where": { "root": true } },
{ "action": "observe" }
]
}
}
@@ -0,0 +1,56 @@
import { page, withRun } from "../../../lib/fixtures.mjs";
export default {
id: "oopif-scrollbars",
routes: ["/oopif-scrollbars", "/oopif-scrollbars/frame", "/oopif-scrollbars/nested"],
render({ pathname, runId, query }) {
const root = pathname === "/oopif-scrollbars";
const nested = pathname.endsWith("/nested");
const mode = query.get("scrollbars") ?? "both";
return page({
title: "OOPIF scrollbar geometry regression",
body: `
<style>
main { margin: 0; width: auto; }
body { width: 2000px; height: 2400px; }
html { overflow-anchor: none; }
${!root ? `html { overflow-x: ${mode === "vertical" || mode === "none" ? "hidden" : "scroll"}; overflow-y: ${mode === "horizontal" || mode === "none" ? "hidden" : "scroll"}; } html::-webkit-scrollbar { width: 17px; height: 11px; }` : ""}
button { position: absolute; padding: 0; border: 0; border-radius: 0; }
#probe { left: ${root ? 180 : nested ? 30 : 100}px; top: ${root ? 360 : nested ? 40 : 160}px; width: 60px; height: 20px; }
#edge { position: fixed; right: -20px; bottom: -15px; width: 80px; height: 60px; }
#outside { position: fixed; left: calc(100% + 2px); top: calc(100% + 2px); width: 10px; height: 10px; }
iframe { position: absolute; box-sizing: content-box; width: 420px; height: 320px;
left: 340px; top: 400px; border: 6px solid black; padding: 8px; transform: scale(1.2); transform-origin: 0 0; }
#nested { left: 150px; top: 210px; width: 200px; height: 120px; border-width: 4px; padding: 6px; transform: scale(0.8); }
</style>
<button id="probe" data-geometry-probe>OOPIF-SCROLLBARS</button>
${!root ? '<button id="edge" data-geometry-probe>EDGE</button><button id="outside" data-geometry-probe>OUTSIDE</button>' : ""}
${!nested ? `<iframe id="${root ? "cross" : "nested"}" name="${root ? "cross" : "nested"}" title="${root ? "outer" : "nested"} cross-site frame"></iframe>` : ""}
`,
script: `
history.scrollRestoration = "manual";
const child = document.querySelector("iframe");
if (child) {
const url = new URL(${JSON.stringify(withRun(root ? "/oopif-scrollbars/frame" : "/oopif-scrollbars/nested", runId, { scrollbars: mode }))}, location.href);
url.hostname = location.hostname === "127.0.0.1" ? "localhost" : "127.0.0.1";
child.src = url.href;
}
document.addEventListener("click", event => {
const probe = event.target.closest("[data-geometry-probe]");
if (probe) browserEval.send("geometry.clicked", { probe: probe.id });
});
window.addEventListener("load", () => {
scrollTo(${root ? "80, 240" : nested ? "10, 20" : "40, 100"});
requestAnimationFrame(() => requestAnimationFrame(() => {
document.documentElement.dataset.geometryReady = "true";
if (!${root}) browserEval.send("geometry.scrollbars", {
vertical: innerWidth > document.documentElement.clientWidth,
horizontal: innerHeight > document.documentElement.clientHeight,
});
browserEval.send("geometry.ready", { root: ${root} });
}));
});
`,
});
},
};
@@ -33,12 +33,9 @@ BSK_AUTO_UPDATE=off pnpm eval:browser smoke --case snapshot-coordinates --bsk ./
Smoke alone does **not** certify coordinate accuracy; CLI observations do not expose raw boxes.
Use the geometric test above for the regression's numeric assertions.
## Separate existing boundary
## Related scrollbar regression
The OOPIF root uses non-occupying scrollbars in the unit-conversion regression. Root and
same-process frames retain normal scrollbars. Append `&classic-scrollbars` to the fixture URL
to reproduce the separate existing OOPIF projection issue: `targetProjection()` maps the CSS
layout viewport (which excludes occupying scrollbars) onto the entire owner content quad.
This inflates the projected coordinates when scrollbars occupy space. Main already has that
mapping; fixing it also changes shared live geometry and is outside this snapshot-unit fix.
The test does not relax its numeric tolerance to absorb that error.
The unit-conversion fixture keeps non-occupying OOPIF scrollbars by default. The same
browser test also runs the dedicated [OOPIF scrollbar fixture](../oopif-scrollbars/README.md),
which covers occupied scrollbars, two OOPIF boundaries, clipping and real clicks.
Append `&classic-scrollbars` to this fixture's URL to enable its normal OOPIF scrollbars.
+5 -4
View File
@@ -41,22 +41,23 @@ test("case manifests are discovered, ordered, and grouped into suites", () => {
"diagnostics",
"mobile-emulation",
"generated-form",
"oopif-scrollbars",
"snapshot-coordinates",
],
);
assert.deepEqual(repositorySummary(cases, fixtureRegistry).suites, {
core: 6,
matrix: 1,
regression: 1,
regression: 2,
});
});
test("repository validation links every case to a fixture and valid workflow evidence", () => {
assert.deepEqual(validateRepositoryCases(cases, fixtureRegistry), []);
const summary = repositorySummary(cases, fixtureRegistry);
assert.equal(summary.cases, 8);
assert.equal(summary.fixtureModules, 9);
assert.equal(summary.fixtureRoutes, 15);
assert.equal(summary.cases, 9);
assert.equal(summary.fixtureModules, 10);
assert.equal(summary.fixtureRoutes, 18);
});
test("manifest validation rejects unknown operations and incomplete workflow steps", () => {
+4 -1
View File
@@ -54,10 +54,13 @@ the `bsk` CLI and browser extension separately when a release requires it.
| `browser_session` | `start`, `stop`, `list` | Manage plugin-owned Agent Window sessions. |
| `browser_page` | `navigate`, `back`, `forward`, `reload`, `wait` | Navigate the active tab and wait for page lifecycle events. |
| `browser_inspect` | `observe`, `snapshot`, `html`, `screenshot`, `console`, `network` | Read semantic or diagnostic page state and capture screenshots. |
| `browser_interact` | `click`, `hover`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. |
| `browser_interact` | `click`, `hover`, `scroll-to`, `focus`, `blur`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. |
| `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. |
| `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. |
For `browser_interact` with `action: "scroll-to"`, see the
[scroll-to reference](../../docs/scroll-to.md) for parameters, visible bounds and errors.
Arbitrary page-script evaluation and interaction recording are not supported.
## Multi-session model
@@ -51,8 +51,13 @@ Use `browser_inspect` action `observe` as the primary semantic page view. It ret
text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may
also become stale after large DOM changes, so observe again before the next interaction.
Use `browser_interact` for click, hover, fill, select, and key actions. An observation marks a
hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are
Use `browser_interact` with `action=scroll-to` to bring an element into view. Its bounds are the
visible border-box portion in top-level viewport CSS pixels after ancestor clipping; partial
visibility is enough. Hidden or fully clipped targets fail. This does not test occlusion by other
elements. Use fresh element refs for iframe/shadow-root targets; selectors search the main document.
Use `browser_interact` for click, hover, scroll-to, focus, blur, fill, select, and key actions. Focus and blur
explicitly enter or leave UI states triggered by focus changes. An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are
labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref.
Do not click the trigger itself unless the user wants the trigger's action.
@@ -167,12 +167,15 @@ const BROWSER_TOOL_SPECS: BrowserToolSpec[] = [
{
name: "browser_interact",
description:
"Interact with an element in the active Agent Window tab. Actions: click, hover, fill, select, " +
"press. click/hover/fill/select require target; fill also requires value; select requires " +
"Interact with an element in the active Agent Window tab. Actions: click, hover, scroll-to, focus, blur, fill, select, " +
"press. click/hover/scroll-to/focus/blur/fill/select require target; fill also requires value; select requires " +
"values; press requires key and may optionally focus target first.",
actions: {
click: "interact.click",
hover: "interact.hover",
"scroll-to": "interact.scroll-to",
focus: "interact.focus",
blur: "interact.blur",
fill: "interact.fill",
select: "interact.select",
press: "interact.press",
@@ -562,6 +562,10 @@ export function actionForLabel(label: string): string {
return "clicking";
case "hover":
return "hovering";
case "focus":
return "focusing";
case "blur":
return "blurring";
case "fill":
return "filling";
case "select":
@@ -13,7 +13,7 @@ import type { ToolDeps } from "./tools";
const MODIFIERS = ["alt", "ctrl", "meta", "shift"] as const;
/** Add hover and select without bypassing session ownership or observation. */
/** Add interaction primitives without bypassing session ownership or observation. */
export function registerPhaseOneInteractionTools(
deps: ToolDeps,
register: ToolRegistrar,
@@ -100,6 +100,171 @@ export function registerPhaseOneInteractionTools(
}),
);
register(
defineTool({
name: "interact.scroll-to",
description:
"Scroll an element and its frame owners into view. Returns the visible portion's " +
"bounds in top-level viewport CSS pixels; fails if no visible area remains. " +
"Use a fresh ref for iframe targets; selectors search the main document.",
parameters: {
target: {
type: "string",
required: true,
description: "Element ref (@e3 / e3) or main-document CSS selector to scroll into view.",
},
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
timeoutMs: TIMEOUT_MS_PARAM,
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
x: { type: "number", required: true },
y: { type: "number", required: true },
width: { type: "number", required: true },
height: { type: "number", required: true },
},
},
render: (_args, value) => [
{
type: "text",
text:
`[session ${value.session}] scrolled into view on tab ${value.tabId}: ` +
`(${value.x}, ${value.y}, ${value.width}, ${value.height})`,
},
],
},
async execute(args, exec) {
requireNonEmpty(args.target, "target");
requirePositive(args.timeoutMs, "timeoutMs");
const sessionId = registry.resolve(args.session, "browser_interact(action=scroll-to)");
const cmdArgs = ["scroll-to", "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`);
appendTarget(cmdArgs, args.target);
const reply = (await runtime.run(
exec,
cmdArgs,
"scroll-to",
sessionId,
runnerTimeout(deps, args.timeoutMs),
)) as {
tab_id: number;
x: number;
y: number;
width: number;
height: number;
};
return {
session: sessionId,
tabId: reply.tab_id,
x: reply.x,
y: reply.y,
width: reply.width,
height: reply.height,
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"scroll-to",
args.target,
"--session",
args.session ?? "(current)",
]),
description: "Scroll an element into view",
}),
presentResult: runtime.presentTerminalResult,
}),
);
for (const action of ["focus", "blur"] as const) {
register(
defineTool({
name: `interact.${action}`,
description:
action === "focus"
? "Focus an element and verify its DOM focus state."
: "Remove focus from an element and report whether it was focused.",
parameters: {
target: {
type: "string",
required: true,
description: "Snapshot ref (@e3 / e3) or CSS selector of the element.",
},
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
timeoutMs: TIMEOUT_MS_PARAM,
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
focused: { type: "boolean", required: true },
wasFocused: {
type: "boolean",
description: "Whether the target was focused before blur.",
},
},
},
render: (_args, value) => [
{
type: "text",
text:
`[session ${value.session}] ${action} on tab ${value.tabId}: focused=${value.focused}` +
(value.wasFocused === undefined ? "" : `, wasFocused=${value.wasFocused}`),
},
],
},
async execute(args, exec) {
requireNonEmpty(args.target, "target");
requirePositive(args.timeoutMs, "timeoutMs");
const sessionId = registry.resolve(args.session, `browser_interact(action=${action})`);
const cmdArgs = [action, "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`);
appendTarget(cmdArgs, args.target);
const reply = (await runtime.run(
exec,
cmdArgs,
action,
sessionId,
runnerTimeout(deps, args.timeoutMs),
)) as {
tab_id: number;
focused: boolean;
was_focused?: boolean;
};
return {
session: sessionId,
tabId: reply.tab_id,
focused: reply.focused,
...(action === "blur" ? { wasFocused: reply.was_focused } : {}),
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
action,
args.target,
"--session",
args.session ?? "(current)",
]),
description: action === "focus" ? "Focus an element" : "Remove focus from an element",
}),
presentResult: runtime.presentTerminalResult,
}),
);
}
register(
defineTool({
name: "interact.select",
@@ -102,6 +102,9 @@ const ACTION_ROUTES: Record<string, readonly [string, string]> = {
"inspect.network": ["browser_inspect", "network"],
"interact.click": ["browser_interact", "click"],
"interact.hover": ["browser_interact", "hover"],
"interact.scroll-to": ["browser_interact", "scroll-to"],
"interact.focus": ["browser_interact", "focus"],
"interact.blur": ["browser_interact", "blur"],
"interact.fill": ["browser_interact", "fill"],
"interact.select": ["browser_interact", "select"],
"interact.press": ["browser_interact", "press"],
@@ -200,7 +203,7 @@ const EXPECTED_ACTIONS = {
browser_session: ["start", "stop", "list"],
browser_page: ["navigate", "back", "forward", "reload", "wait"],
browser_inspect: ["observe", "snapshot", "html", "screenshot", "console", "network"],
browser_interact: ["click", "hover", "fill", "select", "press"],
browser_interact: ["click", "hover", "scroll-to", "focus", "blur", "fill", "select", "press"],
browser_tabs: ["list", "create", "select", "close", "borrow", "return"],
browser_assist: ["resize", "emulate", "request-help"],
} as const;
@@ -499,6 +502,85 @@ describe("interaction tools", () => {
]);
});
it.each([
"focus",
"blur",
] as const)("interact.%s maps focus state, target, tab and timeout", async (action) => {
const { tools, calls } = setup({
"session start": START_REPLY("s1"),
[action]: {
tab_id: 7,
focused: action === "focus",
...(action === "blur" ? { was_focused: true } : {}),
},
});
await startSession(tools);
const target = action === "focus" ? "@e3" : "#field";
const value = await tools
.get(`interact.${action}`)
?.execute({ target, tabId: 7, timeoutMs: 150_000 }, makeExec());
expect(value).toEqual({
session: "s1",
tabId: 7,
focused: action === "focus",
...(action === "blur" ? { wasFocused: true } : {}),
});
expect(calls[1].args).toEqual([
action,
"--session",
"s1",
"--tab-id",
"7",
"--timeout",
"150000ms",
target,
]);
expect(calls[1].options.timeoutMs).toBe(165_000);
});
it.each([
"focus",
"blur",
"scroll-to",
] as const)("interact.%s validates arguments and session ownership before running", async (action) => {
const { tools, calls } = setup({ "session start": START_REPLY("s1") });
await startSession(tools);
const tool = tools.get(`interact.${action}`)!;
await expect(tool.execute({}, makeExec())).rejects.toThrow(/invalid arguments/i);
await expect(tool.execute({ target: " " }, makeExec())).rejects.toThrow(/non-empty/i);
await expect(tool.execute({ target: "e1", timeoutMs: 0 }, makeExec())).rejects.toThrow(
/greater than zero/i,
);
await expect(tool.execute({ target: "e1", session: "foreign" }, makeExec())).rejects.toThrow(
/session/i,
);
expect(calls).toHaveLength(1);
});
it.each(["@e3", "#target"])("interact.scroll-to maps %s, bounds and timeout", async (target) => {
const { tools, calls } = setup({
"session start": START_REPLY("s1"),
"scroll-to": { tab_id: 7, x: 10, y: 20, width: 100, height: 80 },
});
await startSession(tools);
expect(
await tools
.get("interact.scroll-to")
?.execute({ target, tabId: 7, timeoutMs: 150_000 }, makeExec()),
).toEqual({ session: "s1", tabId: 7, x: 10, y: 20, width: 100, height: 80 });
expect(calls[1].args).toEqual([
"scroll-to",
"--session",
"s1",
"--tab-id",
"7",
"--timeout",
"150000ms",
target,
]);
expect(calls[1].options.timeoutMs).toBe(165_000);
});
it("interact.fill passes value and --no-clear", async () => {
const { tools, calls } = setup(responses);
await startSession(tools);
+12 -2
View File
@@ -55,10 +55,20 @@ Use this default loop:
```text
bsk navigate <url> --session <id>
bsk observe --session <id>
bsk click|hover|fill|select|press ... --session <id>
bsk click|hover|scroll-to|focus|blur|fill|select|press ... --session <id>
bsk observe --session <id> # after navigation or a meaningful DOM change
```
`bsk scroll-to <ref-or-selector> --session <id>` scrolls an element and its frame owners into view.
Use a fresh element ref for iframe/shadow-root targets; CSS selectors search the main document.
The result is the visible border-box portion's bounds in top-level viewport CSS pixels after
ancestor clipping. Partial visibility is enough; hidden or fully clipped targets fail with
`permission_denied` and `data.reason=element_not_visible`. This does not test occlusion by other elements.
For a specific tab or deadline: `bsk scroll-to @e3 --session <id> --tab-id 42 --timeout 5s`.
`bsk focus <ref>` explicitly focuses a target; `bsk blur <ref>` removes focus and reports whether
it was focused. Use these for UI states triggered by focus changes.
Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also
make them stale. Observe again before the next interaction.
@@ -114,7 +124,7 @@ This list of names is complete. Never invent a command outside it; read
session start|stop|list browsers status doctor update logs
navigate navigate-back navigate-forward reload wait-for-navigation wait-ms
observe snapshot get-html screenshot console network
click hover fill select press evaluate
click hover scroll-to focus blur fill select press evaluate
tab list|create|close|select|borrow|return window resize emulate
upload download request-help record start|stop
```