Merge pull request #126 from NianJiuZst/codex/focus-blur-primitive

feat(interaction): add explicit focus and blur primitives
This commit is contained in:
Zhang GH
2026-09-09 23:23:50 +08:00
committed by GitHub
28 changed files with 1789 additions and 26 deletions
@@ -599,6 +599,70 @@ describe("ToolDispatcher", () => {
expect(onSessionsChanged).toHaveBeenCalledTimes(2); expect(onSessionsChanged).toHaveBeenCalledTimes(2);
}); });
it.each([
"focus",
"blur",
] as const)("routes %s with hover cleanup and cooperative cancellation", async (action) => {
const tab = { id: 7, windowId: 4242, active: true };
vi.stubGlobal("chrome", {
tabs: {
get: vi.fn(async () => tab),
query: vi.fn(async () => [tab]),
sendMessage: vi.fn(async () => undefined),
},
});
const sessions = new SessionManager({
agentWindow: {
create: async () => 4242,
remove: async () => {},
ensureActiveTab: async () => 7,
},
});
const ctx = await sessions.start("aa11");
ctx.refStore.set("e1", 12, { tabId: 7 });
const { transport, sent, deliver } = fakeTransport();
const onBrowserControlResumed = vi.fn();
let resolveNode: ((value: object) => void) | undefined;
const cdp = {
send: vi.fn(async (_tabId: number, method: string) => {
if (method === "DOM.resolveNode")
return new Promise((resolve) => {
resolveNode = resolve;
});
if (method === "Runtime.callFunctionOn") return { result: { value: true } };
return {};
}),
} as unknown as TestDispatcherCdp;
const dispatcher = new ToolDispatcher({ transport, sessions, cdp, onBrowserControlResumed });
const hover = dispatcher as unknown as {
rememberHover: (sessionId: string, result: object) => void;
setHoverBypass: (sessionId: string, tabId: number, enabled: boolean) => Promise<void>;
};
hover.rememberHover("aa11", { tab_id: 7, x: 10, y: 20 });
await hover.setHoverBypass("aa11", 7, true);
dispatcher.start();
deliver(makeRequest(`tool.${action}`, { session_id: "aa11", ref: "e1" }));
await vi.waitFor(() => expect(resolveNode).toBeDefined());
expect(onBrowserControlResumed).toHaveBeenCalledWith("aa11");
deliver({ id: "cancel-focus", method: "cancel", params: { rpc_id: "r-1" } });
resolveNode!({ object: { objectId: "focus-target" } });
await vi.waitFor(() => expect(sent).toHaveLength(2));
expect(sent).toContainEqual({ id: "cancel-focus", result: { cancelled: true } });
expect(sent).toContainEqual({
id: "r-1",
error: expect.objectContaining({ code: "cancelled" }),
});
expect(cdp.send).not.toHaveBeenCalledWith(7, "DOM.focus", expect.anything());
expect(cdp.send).not.toHaveBeenCalledWith(7, "Runtime.callFunctionOn", expect.anything());
expect(cdp.send).toHaveBeenCalledWith(7, "Runtime.releaseObject", { objectId: "focus-target" });
expect(chrome.tabs.sendMessage).toHaveBeenCalledWith(
7,
expect.objectContaining({ enabled: false }),
);
expect(dispatcher.inflightAbortControllers.size).toBe(0);
dispatcher.stop();
});
it("invokes onBrowserControlResumed for browser-control tools but not passive reads", async () => { it("invokes onBrowserControlResumed for browser-control tools but not passive reads", async () => {
vi.stubGlobal("chrome", { vi.stubGlobal("chrome", {
tabs: { tabs: {
@@ -0,0 +1,166 @@
// @vitest-environment node
// Opt in with BSK_FOCUS_CHROME=/path/to/chrome; each test owns its browser/profile.
import { describe, expect, it } from "vitest";
import { SessionManager } from "@/session-manager/manager";
import { handleBlur, handleFocus } from "../interaction";
import type { CdpRunner } from "../shared";
type Send = <T = Record<string, unknown>>(
method: string,
params?: object,
sessionId?: string,
) => Promise<T>;
async function withFocusBrowser(
run: (h: {
evaluate: (expression: string) => Promise<unknown>;
ref: () => Promise<void>;
focus: (signal?: AbortSignal) => ReturnType<typeof handleFocus>;
blur: (signal?: AbortSignal) => ReturnType<typeof handleBlur>;
calls: string[];
afterCommand: { current: (method: string) => void };
}) => Promise<void>,
) {
// Reuse the existing isolated Chrome launcher; no new browser dependency.
const { withChrome } = await import(
new URL(
"../../../../../evals/browser/cases/regression/snapshot-coordinates/chrome.mjs",
import.meta.url,
).href
);
await withChrome(
{ executable: process.env.BSK_FOCUS_CHROME, deviceScale: 1, zoom: 1 },
async (send: Send) => {
const { targetId } = await send<{ targetId: string }>("Target.createTarget", {
url: "about:blank",
});
const { sessionId } = await send<{ sessionId: string }>("Target.attachToTarget", {
targetId,
flatten: true,
});
await send("Page.bringToFront", {}, sessionId);
const evaluate = async (expression: string) => {
const reply = await send<{ result: { value: unknown }; exceptionDetails?: unknown }>(
"Runtime.evaluate",
{ expression, returnByValue: true },
sessionId,
);
expect(reply.exceptionDetails).toBeUndefined();
return reply.result.value;
};
await evaluate(
`document.body.innerHTML = '<input id="target"><input id="other">'; window.probe = document.querySelector('#target');`,
);
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("aa11");
const calls: string[] = [];
const afterCommand = { current: (_method: string) => {} };
const cdp: CdpRunner = {
send: async (_tabId, method, params) => {
calls.push(method);
const result = await send(method, params, sessionId);
afterCommand.current(method);
return result as never;
},
};
const deps = {
cdp,
tabsApi: {
get: async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab,
query: async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab],
},
};
await run({
evaluate,
calls,
afterCommand,
ref: async () => {
const { result } = await send<{ result: { objectId: string } }>(
"Runtime.evaluate",
{ expression: "window.probe" },
sessionId,
);
const { node } = await send<{ node: { backendNodeId: number } }>(
"DOM.describeNode",
{ objectId: result.objectId },
sessionId,
);
await send("Runtime.releaseObject", { objectId: result.objectId }, sessionId);
ctx.refStore.set("e1", node.backendNodeId, { tabId: 4 });
},
focus: (signal) =>
handleFocus(manager, { session_id: "aa11", ref: "e1" }, { ...deps, signal }),
blur: (signal) =>
handleBlur(manager, { session_id: "aa11", ref: "e1" }, { ...deps, signal }),
});
},
);
}
describe.skipIf(!process.env.BSK_FOCUS_CHROME)("real browser focus and blur", () => {
it.each(["light", "open", "closed"])("verifies focus and blur in %s DOM", async (mode) => {
await withFocusBrowser(async (h) => {
if (mode !== "light")
await h.evaluate(`(() => {
const host = document.createElement('div'); document.body.append(host);
const nested = document.createElement('div'); host.attachShadow({mode:'${mode}'}).append(nested);
const input = document.createElement('input'); nested.attachShadow({mode:'${mode}'}).append(input);
window.probe = input;
})()`);
await h.ref();
expect(await h.focus()).toMatchObject({ focused: true });
expect(await h.evaluate("window.probe.matches(':focus')")).toBe(true);
expect(await h.blur()).toMatchObject({ was_focused: true, focused: false });
expect(await h.evaluate("window.probe.matches(':focus')")).toBe(false);
expect(h.calls.filter((method) => method === "Runtime.releaseObject")).toHaveLength(2);
});
});
it("detects focus changes from event-handler microtasks", async () => {
await withFocusBrowser(async (h) => {
await h.ref();
await h.evaluate(
"window.probe.addEventListener('focus', () => queueMicrotask(() => document.querySelector('#other').focus()), {once:true})",
);
expect(await h.focus()).toMatchObject({ code: "cdp_failed" });
expect(await h.focus()).toMatchObject({ focused: true });
await h.evaluate(
"window.probe.addEventListener('blur', () => queueMicrotask(() => window.probe.focus()), {once:true})",
);
expect(await h.blur()).toMatchObject({ code: "cdp_failed" });
expect(await h.evaluate("window.probe.matches(':focus')")).toBe(true);
});
});
it("cancels before blur while still releasing the resolved object", async () => {
await withFocusBrowser(async (h) => {
await h.ref();
await h.focus();
const abort = new AbortController();
h.afterCommand.current = (method) => {
if (method === "DOM.resolveNode") abort.abort();
};
expect(await h.blur(abort.signal)).toMatchObject({ code: "cancelled" });
expect(await h.evaluate("window.probe.matches(':focus')")).toBe(true);
expect(h.calls.at(-1)).toBe("Runtime.releaseObject");
});
});
it("reports a page exception with its cause", async () => {
await withFocusBrowser(async (h) => {
await h.ref();
await h.evaluate("window.probe.blur = () => { throw new Error('page blur failed'); }");
expect(await h.blur()).toMatchObject({
code: "cdp_failed",
message: expect.stringContaining("page blur failed"),
});
expect(h.calls.at(-1)).toBe("Runtime.releaseObject");
});
});
});
@@ -0,0 +1,293 @@
import { afterEach, describe, expect, it, vi } from "vitest";
import type { CdpTarget } from "@/browser-driver/frame-graph";
import { SessionManager } from "@/session-manager/manager";
import { handleBlur, handleFocus } from "../interaction";
import type { CdpRunner } from "../shared";
interface ScriptParams {
functionDeclaration: string;
}
interface Call {
target: CdpTarget;
method: string;
}
async function setup(element?: HTMLElement, childSession = false) {
if (!element) {
document.body.innerHTML = '<input id="target"><input id="other">';
element = document.querySelector<HTMLInputElement>("#target")!;
}
const targetElement = element;
const manager = new SessionManager({
agentWindow: {
create: async () => 100,
remove: async () => {},
ensureActiveTab: async () => 4,
},
});
const ctx = await manager.start("aa11");
ctx.refStore.set("e1", 12, {
tabId: 4,
...(childSession ? { frameId: "child", cdpSessionId: "child-session" } : {}),
});
const calls: Call[] = [];
const afterCommand = vi.fn<(call: Call) => void>();
const script = vi.fn(async (params: ScriptParams) => {
try {
const fn = new Function(`return (${params.functionDeclaration})`)();
return { result: { value: await fn.call(targetElement) } };
} catch (error) {
return { exceptionDetails: { text: "Uncaught", exception: { description: String(error) } } };
}
});
const focus = vi.fn(() => targetElement.focus());
const release = vi.fn(async () => ({}));
const send = async (target: CdpTarget, method: string, params?: object) => {
const call = { target, method };
calls.push(call);
try {
switch (method) {
case "DOM.getDocument":
return { root: { nodeId: 1 } };
case "DOM.querySelector":
return { nodeId: 2 };
case "DOM.describeNode":
return { node: { backendNodeId: 12 } };
case "DOM.scrollIntoViewIfNeeded":
return {};
case "DOM.resolveNode":
return { object: { objectId: "focus-target" } };
case "DOM.focus":
focus();
return {};
case "Runtime.callFunctionOn":
return await script(params as ScriptParams);
case "Runtime.releaseObject":
return await release();
default:
throw new Error(`unexpected ${method}`);
}
} finally {
afterCommand(call);
}
};
const cdp: CdpRunner = {
send: (tabId, method, params) => send({ tabId }, method, params) as never,
sendToTarget: send as CdpRunner["sendToTarget"],
getFrameGraph: vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
...(childSession
? [
{
frameId: "child",
parentFrameId: "main",
ownerBackendNodeId: 99,
target: { tabId: 4, sessionId: "child-session" },
},
]
: []),
],
})),
};
const tabsApi = {
get: vi.fn(async (id: number) => ({ id, windowId: 100, active: true }) as chrome.tabs.Tab),
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
};
return {
element: targetElement,
calls,
afterCommand,
script,
focus,
release,
cdp,
tabsApi,
run: (
action: "focus" | "blur",
signal?: AbortSignal,
params: { ref?: string; selector?: string; tab_id?: number } = { ref: "e1" },
) =>
(action === "focus" ? handleFocus : handleBlur)(
manager,
{ session_id: "aa11", ...params },
{ cdp, tabsApi, signal },
),
};
}
afterEach(() => {
vi.restoreAllMocks();
document.body.innerHTML = "";
});
describe("focus and blur state verification", () => {
it.each(["open", "closed"] as const)("handles nested %s shadow roots", async (mode) => {
const host = document.createElement("div");
document.body.append(host);
const outer = host.attachShadow({ mode });
const innerHost = document.createElement("div");
outer.append(innerHost);
const inner = innerHost.attachShadow({ mode });
const element = document.createElement("input");
inner.append(element);
const h = await setup(element);
expect(await h.run("focus")).toMatchObject({ focused: true });
expect(inner.activeElement).toBe(element);
expect(await h.run("blur")).toMatchObject({ was_focused: true, focused: false });
expect(inner.activeElement).toBeNull();
expect(h.release).toHaveBeenCalledTimes(2);
});
it("keeps DOM focus usable when the document is in the background", async () => {
const h = await setup();
vi.spyOn(document, "hasFocus").mockReturnValue(false);
expect(await h.run("focus")).toMatchObject({ focused: true });
expect(await h.run("blur")).toMatchObject({ was_focused: true, focused: false });
});
it("blurring an unfocused target preserves the other element's focus", async () => {
const h = await setup();
const other = document.querySelector<HTMLInputElement>("#other")!;
other.focus();
expect(await h.run("blur")).toMatchObject({ was_focused: false, focused: false });
expect(document.activeElement).toBe(other);
});
it("detects focus redirected by a focus handler's microtask", async () => {
const h = await setup();
h.element.addEventListener("focus", () =>
queueMicrotask(() => document.querySelector<HTMLInputElement>("#other")!.focus()),
);
expect(await h.run("focus")).toMatchObject({
code: "cdp_failed",
message: "target element did not become focused",
});
expect(h.release).toHaveBeenCalledOnce();
});
it("does not report blur success when a microtask restores focus", async () => {
const h = await setup();
h.element.focus();
h.element.addEventListener("blur", () => queueMicrotask(() => h.element.focus()));
expect(await h.run("blur")).toMatchObject({
code: "cdp_failed",
message: "target element remained focused after blur()",
});
expect(document.activeElement).toBe(h.element);
expect(h.release).toHaveBeenCalledOnce();
});
it.each(["focus", "blur"] as const)("rejects a detached %s target", async (action) => {
const h = await setup();
h.element.remove();
expect(await h.run(action)).toMatchObject({ code: "cdp_failed" });
expect(h.release).toHaveBeenCalledOnce();
});
it("preserves a script exception's cause", async () => {
const h = await setup();
vi.spyOn(h.element, "blur").mockImplementation(() => {
throw new Error("page blur failed");
});
expect(await h.run("blur")).toMatchObject({
code: "cdp_failed",
message: expect.stringContaining("page blur failed"),
});
expect(h.release).toHaveBeenCalledOnce();
});
it.each(["focus", "blur"] as const)("rejects malformed %s script results", async (action) => {
const h = await setup();
h.script.mockResolvedValueOnce({ result: { value: null } });
expect(await h.run(action)).toMatchObject({
code: "cdp_failed",
message: expect.stringContaining("unexpected result"),
});
expect(h.release).toHaveBeenCalledOnce();
});
it.each([
"focus",
"blur",
] as const)("keeps %s, verification and cleanup in the OOPIF session", async (action) => {
const h = await setup(undefined, true);
if (action === "blur") h.element.focus();
expect(await h.run(action)).toMatchObject({ focused: action === "focus" });
const targetCalls = h.calls.filter((call) => call.method !== "DOM.scrollIntoViewIfNeeded");
expect(targetCalls.every((call) => call.target.sessionId === "child-session")).toBe(true);
expect(targetCalls.at(-1)?.method).toBe("Runtime.releaseObject");
});
it.each(["focus", "blur"] as const)("rejects %s on a user tab before CDP", async (action) => {
const h = await setup();
h.tabsApi.get.mockResolvedValue({ id: 9, windowId: 200, active: true } as chrome.tabs.Tab);
expect(await h.run(action, undefined, { ref: "e1", tab_id: 9 })).toMatchObject({
code: "permission_denied",
});
expect(h.calls).toEqual([]);
});
it("does not replace a successful result when navigation disposed the remote object", async () => {
const h = await setup();
h.release.mockRejectedValue(new Error("Cannot find context"));
expect(await h.run("focus")).toMatchObject({ focused: true });
});
});
describe("focus and blur cancellation", () => {
const boundaries = [
"DOM.getDocument",
"DOM.querySelector",
"DOM.describeNode",
"DOM.resolveNode",
"Runtime.callFunctionOn",
];
for (const action of ["focus", "blur"] as const) {
it.each(
action === "focus" ? [...boundaries, "DOM.scrollIntoViewIfNeeded", "DOM.focus"] : boundaries,
)(`${action} stops after cancellation during %s`, async (method) => {
const h = await setup();
const abort = new AbortController();
let cancelledAfter = 0;
h.afterCommand.mockImplementation((call) => {
if (call.method === method && !abort.signal.aborted) {
cancelledAfter = h.calls.length;
abort.abort();
}
});
expect(await h.run(action, abort.signal, { selector: "#target" })).toMatchObject({
code: "cancelled",
});
expect(abort.signal.aborted).toBe(true);
expect(
h.calls.slice(cancelledAfter).every((call) => call.method === "Runtime.releaseObject"),
).toBe(true);
if (h.calls.some((call) => call.method === "DOM.resolveNode"))
expect(h.release).toHaveBeenCalledOnce();
});
it(`${action} stops if cancellation arrives while resolving the tab`, async () => {
const h = await setup();
const abort = new AbortController();
h.tabsApi.query.mockImplementation(async () => {
abort.abort();
return [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab];
});
expect(await h.run(action, abort.signal)).toMatchObject({ code: "cancelled" });
expect(h.calls).toEqual([]);
});
}
it("does not scroll frame owners after cancellation in an OOPIF", async () => {
const h = await setup(undefined, true);
const abort = new AbortController();
h.afterCommand.mockImplementation((call) => {
if (call.method === "DOM.scrollIntoViewIfNeeded") abort.abort();
});
expect(await h.run("focus", abort.signal)).toMatchObject({ code: "cancelled" });
expect(h.calls.filter((call) => call.method === "DOM.scrollIntoViewIfNeeded")).toHaveLength(1);
expect(h.focus).not.toHaveBeenCalled();
});
});
@@ -2,8 +2,10 @@ import { describe, expect, it, vi } from "vitest";
import { SessionManager } from "@/session-manager/manager"; import { SessionManager } from "@/session-manager/manager";
import type { CdpRunner } from "@/tools/shared"; import type { CdpRunner } from "@/tools/shared";
import { import {
handleBlur,
handleClick, handleClick,
handleFill, handleFill,
handleFocus,
handleHover, handleHover,
handlePress, handlePress,
handleSelect, handleSelect,
@@ -542,6 +544,161 @@ describe("handleHover", () => {
}); });
}); });
describe("handleFocus and handleBlur", () => {
it("focuses a ref and verifies the deep active element", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, { tabId: 4 });
const fake = makeFakeCdp({
"DOM.scrollIntoViewIfNeeded": () => ({}),
"DOM.focus": () => ({}),
"DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }),
"Runtime.callFunctionOn": () => ({ result: { value: true } }),
});
const res = await handleFocus(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res).toMatchObject({ tab_id: 4, used_ref: "e3", focused: true });
expect(fake.sent.map((call) => call.method)).toEqual([
"DOM.scrollIntoViewIfNeeded",
"DOM.resolveNode",
"DOM.focus",
"Runtime.callFunctionOn",
"Runtime.releaseObject",
]);
});
it("focuses an OOPIF ref in its CDP session", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, {
tabId: 4,
frameId: "child-frame",
cdpSessionId: "child-session",
});
const fake = makeFakeCdp({
"DOM.scrollIntoViewIfNeeded": () => ({}),
});
fake.cdp.getFrameGraph = vi.fn(async () => ({
rootFrameId: "main",
frames: [
{ frameId: "main", target: { tabId: 4 } },
{
frameId: "child-frame",
parentFrameId: "main",
ownerBackendNodeId: 99,
target: { tabId: 4, sessionId: "child-session" },
},
],
}));
const targetCalls: Array<{ sessionId?: string; method: string }> = [];
fake.cdp.sendToTarget = vi.fn(async (target, method) => {
targetCalls.push({ sessionId: target.sessionId, method });
if (method === "DOM.scrollIntoViewIfNeeded") return {};
if (method === "DOM.focus" || method === "Runtime.releaseObject") return {};
if (method === "DOM.resolveNode") return { object: { objectId: "focus-target" } };
if (method === "Runtime.callFunctionOn") return { result: { value: true } };
throw new Error(`unexpected child CDP call ${method}`);
}) as CdpRunner["sendToTarget"];
const res = await handleFocus(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res.focused).toBe(true);
expect(targetCalls).toEqual([
{ sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" },
{ sessionId: "child-session", method: "DOM.resolveNode" },
{ sessionId: "child-session", method: "DOM.focus" },
{ sessionId: "child-session", method: "Runtime.callFunctionOn" },
{ sessionId: "child-session", method: "Runtime.releaseObject" },
]);
});
it("does not focus after cancellation during scrolling", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, { tabId: 4 });
const abort = new AbortController();
const fake = makeFakeCdp({
"DOM.scrollIntoViewIfNeeded": () => {
abort.abort();
return {};
},
});
const res = await handleFocus(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi, signal: abort.signal },
);
expect(res).toMatchObject({ code: "cancelled" });
expect(fake.sent.some((call) => call.method === "DOM.focus")).toBe(false);
});
it("blurs a ref and returns its previous focus state", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, { tabId: 4 });
const fake = makeFakeCdp({
"DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }),
"Runtime.callFunctionOn": vi
.fn()
.mockResolvedValueOnce({ result: { value: { ok: true, was_focused: true } } })
.mockResolvedValueOnce({ result: { value: false } }),
});
const res = await handleBlur(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
expect(res).toMatchObject({
tab_id: 4,
used_ref: "e3",
was_focused: true,
focused: false,
});
expect(fake.sent.map((call) => call.method)).toEqual([
"DOM.resolveNode",
"Runtime.callFunctionOn",
"Runtime.callFunctionOn",
"Runtime.releaseObject",
]);
});
it("rejects a target that does not implement blur", async () => {
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
const ctx = await sm.start("aa11");
ctx.refStore.set("e3", 1234, { tabId: 4 });
const fake = makeFakeCdp({
"DOM.resolveNode": () => ({ object: { objectId: "focus-target" } }),
"Runtime.callFunctionOn": () => ({
result: { value: { ok: false, was_focused: false, focused: false } },
}),
});
const res = await handleBlur(
sm,
{ session_id: "aa11", ref: "@e3" },
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
);
expect(res).toMatchObject({ code: "invalid_params", message: /does not support blur/ });
});
});
function successfulFillScript(params: unknown) { function successfulFillScript(params: unknown) {
const script = params as { arguments?: Array<{ value: unknown }>; functionDeclaration: string }; const script = params as { arguments?: Array<{ value: unknown }>; functionDeclaration: string };
const args = script.arguments ?? []; const args = script.arguments ?? [];
+35 -1
View File
@@ -2,12 +2,14 @@ import { OVERLAY_AUTOMATION_BYPASS } from "@/lib/overlay-bridge";
import type { SessionManager } from "@/session-manager/manager"; import type { SessionManager } from "@/session-manager/manager";
import type { Transport } from "@/transport/transport"; import type { Transport } from "@/transport/transport";
import type { import type {
BlurParams,
ClickParams, ClickParams,
ConsoleParams, ConsoleParams,
DownloadParams, DownloadParams,
EmulateParams, EmulateParams,
EvaluateParams, EvaluateParams,
FillParams, FillParams,
FocusParams,
GetHtmlParams, GetHtmlParams,
HoverParams, HoverParams,
HoverResult, HoverResult,
@@ -39,7 +41,15 @@ import { type EmulateCdpRunner, handleEmulate } from "./emulate";
import { classifyCdpError } from "./errors"; import { classifyCdpError } from "./errors";
import { handleEvaluate } from "./evaluate"; import { handleEvaluate } from "./evaluate";
import { handleRequestHelp } from "./human-loop"; import { handleRequestHelp } from "./human-loop";
import { handleClick, handleFill, handleHover, handlePress, handleSelect } from "./interaction"; import {
handleBlur,
handleClick,
handleFill,
handleFocus,
handleHover,
handlePress,
handleSelect,
} from "./interaction";
import { import {
handleNavigate, handleNavigate,
handleNavigateBack, handleNavigateBack,
@@ -515,6 +525,28 @@ export class ToolDispatcher {
); );
return this.rememberHover((req.params as HoverParams).session_id, result); return this.rememberHover((req.params as HoverParams).session_id, result);
} }
case "tool.focus":
return this.withHoverReleaseForRequest(
req.params as FocusParams,
() =>
handleFocus(
this.sessions,
req.params as FocusParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
),
signal,
);
case "tool.blur":
return this.withHoverReleaseForRequest(
req.params as BlurParams,
() =>
handleBlur(
this.sessions,
req.params as BlurParams,
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
),
signal,
);
case "tool.fill": case "tool.fill":
return this.withHoverReleaseForRequest( return this.withHoverReleaseForRequest(
req.params as FillParams, req.params as FillParams,
@@ -785,6 +817,8 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
case "tool.reload": case "tool.reload":
case "tool.click": case "tool.click":
case "tool.hover": case "tool.hover":
case "tool.focus":
case "tool.blur":
case "tool.fill": case "tool.fill":
case "tool.press": case "tool.press":
case "tool.select": case "tool.select":
+206 -2
View File
@@ -1,5 +1,4 @@
// DOM interaction tools — `tool.click`, `tool.fill`, `tool.press`, and // DOM interaction tools — click, hover, focus/blur, fill, press, and select.
// `tool.select`.
// //
// All interaction tools: // All interaction tools:
// 1. Resolve target tab (sandbox: must be inside Agent Window). // 1. Resolve target tab (sandbox: must be inside Agent Window).
@@ -14,10 +13,14 @@ import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
import type { CdpTarget } from "@/browser-driver/frame-graph"; import type { CdpTarget } from "@/browser-driver/frame-graph";
import type { SessionContext, SessionManager } from "@/session-manager/manager"; import type { SessionContext, SessionManager } from "@/session-manager/manager";
import type { import type {
BlurParams,
BlurResult,
ClickParams, ClickParams,
ClickResult, ClickResult,
FillParams, FillParams,
FillResult, FillResult,
FocusParams,
FocusResult,
HoverParams, HoverParams,
HoverResult, HoverResult,
KeyModifier, KeyModifier,
@@ -227,6 +230,207 @@ export async function resolveBackendNode(
} }
} }
// Check the target's own root, then its hosts: closed shadow roots are not
// reachable via host.shadowRoot. DOM focus remains meaningful in background tabs.
const FOCUS_CHECK = `function() {
if (!this.isConnected) return false;
let element = this;
while (element) {
const root = element.getRootNode();
if (root.activeElement !== element) return false;
element = root.host;
}
return true;
}`;
const BLUR_TARGET = `function() {
if (!this.isConnected) throw new Error('blur target is detached');
const wasFocused = (${FOCUS_CHECK}).call(this);
if (typeof this.blur !== 'function') return { ok: false, was_focused: wasFocused };
this.blur();
return { ok: true, was_focused: wasFocused };
}`;
interface FocusScriptReply<T> {
result?: { value?: T };
exceptionDetails?: { text?: string; exception?: { description?: string } };
}
function checkFocusAbort(signal: AbortSignal | undefined): void {
if (signal?.aborted) throw new DOMException("interaction aborted", "AbortError");
}
// Only focus/blur use this view. Guard every CDP boundary, including commands
// inside selector resolution and frame scrolling, without changing other tools.
function focusCdp(cdp: CdpRunner, signal: AbortSignal | undefined): CdpRunner {
const send = <T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T> => {
checkFocusAbort(signal);
return cdpRunnerForTarget(cdp, target).send<T>(target.tabId, method, params);
};
return {
send: (tabId, method, params) => send({ tabId }, method, params),
sendToTarget: send,
trackSessionTab: cdp.trackSessionTab?.bind(cdp),
getFrameGraph: cdp.getFrameGraph
? (tabId) => {
checkFocusAbort(signal);
return cdp.getFrameGraph!(tabId);
}
: undefined,
};
}
// ---------------------------------------------------------------------------
// tool.focus / tool.blur
// ---------------------------------------------------------------------------
export function handleFocus(
manager: SessionManager,
params: FocusParams,
deps: InteractionDeps = getDefaultDeps(),
): Promise<FocusResult | RpcError> {
return changeFocus(manager, params, deps, "focus");
}
export function handleBlur(
manager: SessionManager,
params: BlurParams,
deps: InteractionDeps = getDefaultDeps(),
): Promise<BlurResult | RpcError> {
return changeFocus(manager, params, deps, "blur");
}
function changeFocus(
manager: SessionManager,
params: FocusParams,
deps: InteractionDeps,
action: "focus",
): Promise<FocusResult | RpcError>;
function changeFocus(
manager: SessionManager,
params: BlurParams,
deps: InteractionDeps,
action: "blur",
): Promise<BlurResult | RpcError>;
async function changeFocus(
manager: SessionManager,
params: FocusParams | BlurParams,
deps: InteractionDeps,
action: "focus" | "blur",
): Promise<FocusResult | BlurResult | RpcError> {
const ctx = lookupSession(manager, params, action);
if (isRpcError(ctx)) return ctx;
const cdp = focusCdp(deps.cdp, deps.signal);
let objectId: string | undefined;
let cleanupCdp: CdpRunner | undefined;
let tabId: number | undefined;
try {
checkFocusAbort(deps.signal);
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
checkFocusAbort(deps.signal);
if (isRpcError(target)) return target;
const denied = enforceAgentWindow(ctx, target, action);
if (denied) return denied;
tabId = target.tabId;
const dialogCursor = markDialogCursor(deps.cdp, tabId);
const node = await resolveBackendNode(cdp, ctx, target, params, action);
checkFocusAbort(deps.signal);
if (isRpcError(node)) return node;
const nodeCdp = cdpRunnerForTarget(cdp, node.cdpTarget);
cleanupCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
deps.cdp.trackSessionTab?.(ctx.sessionId, tabId);
if (action === "focus") {
const scrollErr = await scrollElementAndFramesIntoView(
cdp,
tabId,
node.cdpTarget,
node.backendNodeId,
node.frameId,
);
checkFocusAbort(deps.signal);
if (scrollErr) return scrollErr;
}
const resolved = await backendNodeToObject(nodeCdp, tabId, node.backendNodeId);
// Keep the handle before checking cancellation so finally can release it.
if (!isRpcError(resolved)) objectId = resolved;
checkFocusAbort(deps.signal);
if (isRpcError(resolved)) return resolved;
const runScript = async <T>(functionDeclaration: string): Promise<T | undefined> => {
const reply = await nodeCdp.send<FocusScriptReply<T>>(
target.tabId,
"Runtime.callFunctionOn",
{
objectId,
functionDeclaration,
returnByValue: true,
},
);
checkFocusAbort(deps.signal);
if (reply.exceptionDetails) {
const details = reply.exceptionDetails;
throw new Error(
`${action} script failed: ${details.exception?.description ?? details.text ?? "unknown exception"}`,
);
}
return reply.result?.value;
};
let wasFocused: boolean | undefined;
if (action === "focus") {
await nodeCdp.send(tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
checkFocusAbort(deps.signal);
} else {
const mutation = await runScript<{ ok: boolean; was_focused: boolean }>(BLUR_TARGET);
if (typeof mutation?.ok !== "boolean" || typeof mutation.was_focused !== "boolean") {
throw new Error("blur script returned an unexpected result");
}
if (!mutation.ok) {
return { code: "invalid_params", message: "target element does not support blur()" };
}
wasFocused = mutation.was_focused;
}
// Use a separate call so microtasks from focus/blur handlers finish before
// we report success. In particular, a blur handler can restore focus.
const focused = await runScript<boolean>(FOCUS_CHECK);
if (typeof focused !== "boolean") {
throw new Error(`${action} verification returned an unexpected result`);
}
if (focused !== (action === "focus")) {
throw new Error(
action === "focus"
? "target element did not become focused"
: "target element remained focused after blur()",
);
}
return attachDialogs(deps.cdp, tabId, dialogCursor, {
tab_id: tabId,
used_ref: node.usedRef,
used_selector: node.usedSelector,
focused,
...(action === "blur" ? { was_focused: wasFocused! } : {}),
});
} catch (err) {
return (
throwIfAborted(deps.signal) ?? {
code: "cdp_failed",
message: err instanceof Error ? err.message : String(err),
}
);
} finally {
if (objectId !== undefined && cleanupCdp && tabId !== undefined) {
// Cleanup must also run after cancellation. Navigation may have already
// disposed the object, which must not replace the operation's result.
try {
await cleanupCdp.send(tabId, "Runtime.releaseObject", { objectId });
} catch {
// The target or execution context may no longer exist.
}
}
}
}
export async function resolveActionTarget( export async function resolveActionTarget(
cdp: CdpRunner, cdp: CdpRunner,
ctx: SessionContext, ctx: SessionContext,
+33
View File
@@ -470,6 +470,39 @@ export interface HoverResult {
dialogs?: JavaScriptDialogInfo[]; dialogs?: JavaScriptDialogInfo[];
} }
export interface FocusParams {
session_id: string;
ref?: string;
selector?: string;
tab_id?: number;
timeout_ms?: number;
}
export interface FocusResult {
tab_id: number;
used_ref?: string;
used_selector?: string;
focused: boolean;
dialogs?: JavaScriptDialogInfo[];
}
export interface BlurParams {
session_id: string;
ref?: string;
selector?: string;
tab_id?: number;
timeout_ms?: number;
}
export interface BlurResult {
tab_id: number;
used_ref?: string;
used_selector?: string;
was_focused: boolean;
focused: boolean;
dialogs?: JavaScriptDialogInfo[];
}
export interface FillParams { export interface FillParams {
session_id: string; session_id: string;
value: string; value: string;
+5 -2
View File
@@ -55,10 +55,13 @@ Use this default loop:
```text ```text
bsk navigate <url> --session <id> bsk navigate <url> --session <id>
bsk observe --session <id> bsk observe --session <id>
bsk click|hover|fill|select|press ... --session <id> bsk click|hover|focus|blur|fill|select|press ... --session <id>
bsk observe --session <id> # after navigation or a meaningful DOM change bsk observe --session <id> # after navigation or a meaningful DOM change
``` ```
`bsk focus <ref>` explicitly focuses a target; `bsk blur <ref>` removes focus and reports whether
it was focused. Use these for UI states triggered by focus changes.
Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also
make them stale. Observe again before the next interaction. make them stale. Observe again before the next interaction.
@@ -114,7 +117,7 @@ This list of names is complete. Never invent a command outside it; read
session start|stop|list browsers status doctor update logs session start|stop|list browsers status doctor update logs
navigate navigate-back navigate-forward reload wait-for-navigation wait-ms navigate navigate-back navigate-forward reload wait-for-navigation wait-ms
observe snapshot get-html screenshot console network observe snapshot get-html screenshot console network
click hover fill select press evaluate click hover focus blur fill select press evaluate
tab list|create|close|select|borrow|return window resize emulate tab list|create|close|select|borrow|return window resize emulate
upload download request-help record start|stop upload download request-help record start|stop
``` ```
+123 -3
View File
@@ -1,4 +1,5 @@
//! `bsk click` / `bsk fill` / `bsk press` (M7 interaction tools). //! `bsk click` / `bsk focus` / `bsk blur` / `bsk fill` / `bsk press`
//! interaction tools.
//! //!
//! The `<target>` positional accepts either a snapshot ref (`@e3`, //! The `<target>` positional accepts either a snapshot ref (`@e3`,
//! `e3`) or a CSS selector. Use `--ref` / `--selector` explicitly when //! `e3`) or a CSS selector. Use `--ref` / `--selector` explicitly when
@@ -10,8 +11,9 @@ use std::time::Duration;
use anyhow::Context; use anyhow::Context;
use bsk_protocol::Method; use bsk_protocol::Method;
use bsk_protocol::tools::{ use bsk_protocol::tools::{
ClickParams, ClickResult, FillParams, FillResult, HoverParams, HoverResult, KeyModifier, BlurParams, BlurResult, ClickParams, ClickResult, FillParams, FillResult, FocusParams,
MouseButton, PressParams, PressResult, SelectParams, SelectResult, FocusResult, HoverParams, HoverResult, KeyModifier, MouseButton, PressParams, PressResult,
SelectParams, SelectResult,
}; };
use clap::{Args, ValueEnum}; use clap::{Args, ValueEnum};
@@ -274,6 +276,124 @@ pub fn dispatch_hover(args: HoverArgs, format: Format) -> Result<(), CliError> {
Ok(()) Ok(())
} }
// ---------------------------------------------------------------------------
// bsk focus / bsk blur
// ---------------------------------------------------------------------------
#[derive(Debug, Clone, Args)]
pub struct FocusArgs {
/// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used.
pub target: Option<String>,
#[arg(long = "ref")]
pub ref_: Option<String>,
#[arg(long = "selector")]
pub selector: Option<String>,
#[arg(long)]
pub session: String,
#[arg(long = "tab-id")]
pub tab_id: Option<i64>,
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
pub timeout: u32,
}
pub fn dispatch_focus(args: FocusArgs, format: Format) -> Result<(), CliError> {
let info = ensure_daemon().context("ensure daemon is running")?;
let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?;
let params = FocusParams {
session_id: args.session,
ref_,
selector,
tab_id: args.tab_id,
timeout_ms: Some(args.timeout),
};
let reply: FocusResult = call(
info.sock_path,
Method::ToolFocus,
params,
"focus-1",
args.timeout,
)?;
match format {
Format::Json => println!(
"{}",
serde_json::to_string_pretty(&reply)
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
),
Format::Human => {
let target =
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref());
println!(
"focus ok tab={} target={target} focused={}",
reply.tab_id, reply.focused
);
print_dialog_summaries(&reply.dialogs);
}
}
Ok(())
}
#[derive(Debug, Clone, Args)]
pub struct BlurArgs {
/// Snapshot ref (`@e3`, `e3`) or CSS selector. Optional when `--ref`/`--selector` is used.
pub target: Option<String>,
#[arg(long = "ref")]
pub ref_: Option<String>,
#[arg(long = "selector")]
pub selector: Option<String>,
#[arg(long)]
pub session: String,
#[arg(long = "tab-id")]
pub tab_id: Option<i64>,
#[arg(long, default_value = "30s", value_parser = parse_timeout_ms)]
pub timeout: u32,
}
pub fn dispatch_blur(args: BlurArgs, format: Format) -> Result<(), CliError> {
let info = ensure_daemon().context("ensure daemon is running")?;
let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?;
let params = BlurParams {
session_id: args.session,
ref_,
selector,
tab_id: args.tab_id,
timeout_ms: Some(args.timeout),
};
let reply: BlurResult = call(
info.sock_path,
Method::ToolBlur,
params,
"blur-1",
args.timeout,
)?;
match format {
Format::Json => println!(
"{}",
serde_json::to_string_pretty(&reply)
.map_err(|e| CliError::Local(anyhow::anyhow!(e)))?
),
Format::Human => {
let target =
format_used_target(reply.used_ref.as_deref(), reply.used_selector.as_deref());
println!(
"blur ok tab={} target={target} was_focused={} focused={}",
reply.tab_id, reply.was_focused, reply.focused
);
print_dialog_summaries(&reply.dialogs);
}
}
Ok(())
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// bsk fill // bsk fill
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
+9 -1
View File
@@ -47,7 +47,9 @@ use crate::cli::evaluate::EvaluateArgs;
use crate::cli::get_html::GetHtmlArgs; use crate::cli::get_html::GetHtmlArgs;
use crate::cli::human_loop::RequestHelpArgs; use crate::cli::human_loop::RequestHelpArgs;
use crate::cli::install_skill::InstallSkillArgs; use crate::cli::install_skill::InstallSkillArgs;
use crate::cli::interaction::{ClickArgs, FillArgs, HoverArgs, PressArgs, SelectArgs}; use crate::cli::interaction::{
BlurArgs, ClickArgs, FillArgs, FocusArgs, HoverArgs, PressArgs, SelectArgs,
};
use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs}; use crate::cli::navigate::{NavigateCommand, NavigateHistoryArgs, ReloadArgs};
use crate::cli::network::NetworkArgs; use crate::cli::network::NetworkArgs;
use crate::cli::observe::ObserveArgs; use crate::cli::observe::ObserveArgs;
@@ -173,6 +175,12 @@ pub enum Command {
/// Hover a snapshot ref or CSS selector. /// Hover a snapshot ref or CSS selector.
Hover(HoverArgs), Hover(HoverArgs),
/// Focus a snapshot ref or CSS selector.
Focus(FocusArgs),
/// Remove focus from a snapshot ref or CSS selector.
Blur(BlurArgs),
/// Fill an input / textarea / contenteditable. /// Fill an input / textarea / contenteditable.
Fill(FillArgs), Fill(FillArgs),
+2
View File
@@ -253,6 +253,8 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
| Method::ToolReload | Method::ToolReload
| Method::ToolClick | Method::ToolClick
| Method::ToolHover | Method::ToolHover
| Method::ToolFocus
| Method::ToolBlur
| Method::ToolFill | Method::ToolFill
| Method::ToolPress | Method::ToolPress
| Method::ToolSelect | Method::ToolSelect
+2
View File
@@ -94,6 +94,8 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
Command::Reload(args) => cli::navigate::dispatch_reload(args, format), Command::Reload(args) => cli::navigate::dispatch_reload(args, format),
Command::Click(args) => cli::interaction::dispatch_click(args, format), Command::Click(args) => cli::interaction::dispatch_click(args, format),
Command::Hover(args) => cli::interaction::dispatch_hover(args, format), Command::Hover(args) => cli::interaction::dispatch_hover(args, format),
Command::Focus(args) => cli::interaction::dispatch_focus(args, format),
Command::Blur(args) => cli::interaction::dispatch_blur(args, format),
Command::Fill(args) => cli::interaction::dispatch_fill(args, format), Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
Command::Press(args) => cli::interaction::dispatch_press(args, format), Command::Press(args) => cli::interaction::dispatch_press(args, format),
Command::Select(args) => cli::interaction::dispatch_select(args, format), Command::Select(args) => cli::interaction::dispatch_select(args, format),
+18
View File
@@ -255,6 +255,24 @@ fn parses_hover_with_settle() {
assert_eq!(args.settle, 300); assert_eq!(args.settle, 300);
} }
#[test]
fn parses_focus_target() {
let cli = parse(&["bsk", "focus", "@e2", "--session", "s1"]);
let Command::Focus(args) = cli.command else {
panic!("expected focus command");
};
assert_eq!(args.target.as_deref(), Some("@e2"));
}
#[test]
fn parses_blur_selector() {
let cli = parse(&["bsk", "blur", "--selector", "#search", "--session", "s1"]);
let Command::Blur(args) = cli.command else {
panic!("expected blur command");
};
assert_eq!(args.selector.as_deref(), Some("#search"));
}
#[test] #[test]
fn rejects_zero_click_count() { fn rejects_zero_click_count() {
assert!( assert!(
+92 -6
View File
@@ -1,5 +1,4 @@
//! M7 integration: drive the 7 new tool RPCs (navigate / //! M7 integration: drive interaction and navigation tool RPCs
//! navigate_back / navigate_forward / reload / click / fill / press)
//! through the IPC + per-session queue + fake extension and assert //! through the IPC + per-session queue + fake extension and assert
//! the wire shapes line up. The extension stub mirrors each method's //! the wire shapes line up. The extension stub mirrors each method's
//! `*Result` so we exercise the daemon's serialise → forward → //! `*Result` so we exercise the daemon's serialise → forward →
@@ -13,10 +12,11 @@ use bsk::daemon::{self, DaemonConfig};
use bsk::ipc_client::IpcClient; use bsk::ipc_client::IpcClient;
use bsk_protocol::system::{HandshakeParams, HandshakeResult}; use bsk_protocol::system::{HandshakeParams, HandshakeResult};
use bsk_protocol::tools::{ use bsk_protocol::tools::{
ClickParams, ClickResult, FillParams, FillResult, KeyModifier, MouseButton, NavigateBackParams, BlurParams, BlurResult, ClickParams, ClickResult, FillParams, FillResult, FocusParams,
NavigateBackResult, NavigateForwardParams, NavigateForwardResult, NavigateParams, FocusResult, KeyModifier, MouseButton, NavigateBackParams, NavigateBackResult,
NavigateResult, PressParams, PressResult, ReloadParams, ReloadResult, SelectParams, NavigateForwardParams, NavigateForwardResult, NavigateParams, NavigateResult, PressParams,
SelectResult, SessionStartParams, SessionStartResult, WaitUntil, PressResult, ReloadParams, ReloadResult, SelectParams, SelectResult, SessionStartParams,
SessionStartResult, WaitUntil,
}; };
use bsk_protocol::{ use bsk_protocol::{
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError, BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
@@ -382,6 +382,92 @@ async fn click_round_trips_ref_and_modifiers() {
handle.shutdown().await; handle.shutdown().await;
} }
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn focus_round_trips_verified_state() {
let (handle, sock) = spawn_daemon().await;
let mut ws = connect_ext(handle.ws_addr()).await;
let _ = do_handshake(&mut ws).await;
run_extension(ws, |req| {
assert_eq!(req.method, Method::ToolFocus);
let params = req.params.clone().unwrap();
let p: FocusParams = serde_json::from_value(params).unwrap();
assert_eq!(p.ref_.as_deref(), Some("@e3"));
ResponseBody::Ok(
serde_json::to_value(FocusResult {
tab_id: 9,
used_ref: Some("e3".into()),
used_selector: None,
focused: true,
dialogs: vec![],
})
.unwrap(),
)
});
let session_id = ipc_session_start(&sock).await;
let result: FocusResult = ipc_tool_call(
&sock,
Method::ToolFocus,
FocusParams {
session_id,
ref_: Some("@e3".into()),
selector: None,
tab_id: None,
timeout_ms: Some(5_000),
},
)
.await
.expect("focus ok");
assert!(result.focused);
assert_eq!(result.used_ref.as_deref(), Some("e3"));
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn blur_round_trips_previous_focus_state() {
let (handle, sock) = spawn_daemon().await;
let mut ws = connect_ext(handle.ws_addr()).await;
let _ = do_handshake(&mut ws).await;
run_extension(ws, |req| {
assert_eq!(req.method, Method::ToolBlur);
let params = req.params.clone().unwrap();
let p: BlurParams = serde_json::from_value(params).unwrap();
assert_eq!(p.selector.as_deref(), Some("#search"));
ResponseBody::Ok(
serde_json::to_value(BlurResult {
tab_id: 9,
used_ref: None,
used_selector: Some("#search".into()),
was_focused: true,
focused: false,
dialogs: vec![],
})
.unwrap(),
)
});
let session_id = ipc_session_start(&sock).await;
let result: BlurResult = ipc_tool_call(
&sock,
Method::ToolBlur,
BlurParams {
session_id,
ref_: None,
selector: Some("#search".into()),
tab_id: None,
timeout_ms: Some(5_000),
},
)
.await
.expect("blur ok");
assert!(result.was_focused);
assert!(!result.focused);
assert_eq!(result.used_selector.as_deref(), Some("#search"));
handle.shutdown().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
async fn fill_round_trips_clear_before_default() { async fn fill_round_trips_clear_before_default() {
let (handle, sock) = spawn_daemon().await; let (handle, sock) = spawn_daemon().await;
@@ -0,0 +1,42 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "BlurParams",
"type": "object",
"required": [
"session_id"
],
"properties": {
"ref": {
"description": "Optional `@e<N>` ref allocated by the last observation. Mutually exclusive with `selector`.",
"type": [
"string",
"null"
]
},
"selector": {
"type": [
"string",
"null"
]
},
"session_id": {
"type": "string"
},
"tab_id": {
"description": "Target tab. Defaults to the Agent Window's active tab.",
"type": [
"integer",
"null"
],
"format": "int64"
},
"timeout_ms": {
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 1.0
}
}
}
@@ -0,0 +1,112 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "BlurResult",
"type": "object",
"required": [
"focused",
"tab_id",
"was_focused"
],
"properties": {
"dialogs": {
"type": "array",
"items": {
"$ref": "#/definitions/JavaScriptDialogInfo"
}
},
"focused": {
"description": "Always false for a successful blur; included for machine verification.",
"type": "boolean"
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"used_ref": {
"type": [
"string",
"null"
]
},
"used_selector": {
"type": [
"string",
"null"
]
},
"was_focused": {
"description": "Whether the target held focus before the call.",
"type": "boolean"
}
},
"definitions": {
"JavaScriptDialogHandledAction": {
"description": "How the extension resolved the dialog so CDP could continue.",
"type": "string",
"enum": [
"accepted",
"dismissed"
]
},
"JavaScriptDialogInfo": {
"description": "One observed + handled JavaScript dialog during a tool call.",
"type": "object",
"required": [
"handled",
"message",
"sequence",
"tab_id",
"type"
],
"properties": {
"default_prompt": {
"type": [
"string",
"null"
]
},
"handled": {
"$ref": "#/definitions/JavaScriptDialogHandledAction"
},
"has_browser_handler": {
"type": [
"boolean",
"null"
]
},
"message": {
"type": "string"
},
"sequence": {
"description": "Monotonic per-tab sequence for ordering within a session.",
"type": "integer",
"format": "uint64",
"minimum": 0.0
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"type": {
"$ref": "#/definitions/JavaScriptDialogType"
},
"url": {
"type": [
"string",
"null"
]
}
}
},
"JavaScriptDialogType": {
"description": "Native JS dialog kind reported by CDP.",
"type": "string",
"enum": [
"alert",
"confirm",
"prompt",
"beforeunload"
]
}
}
}
@@ -0,0 +1,42 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "FocusParams",
"type": "object",
"required": [
"session_id"
],
"properties": {
"ref": {
"description": "Optional `@e<N>` ref allocated by the last observation. Mutually exclusive with `selector`.",
"type": [
"string",
"null"
]
},
"selector": {
"type": [
"string",
"null"
]
},
"session_id": {
"type": "string"
},
"tab_id": {
"description": "Target tab. Defaults to the Agent Window's active tab.",
"type": [
"integer",
"null"
],
"format": "int64"
},
"timeout_ms": {
"type": [
"integer",
"null"
],
"format": "uint32",
"minimum": 1.0
}
}
}
@@ -0,0 +1,107 @@
{
"$schema": "http://json-schema.org/draft-07/schema#",
"title": "FocusResult",
"type": "object",
"required": [
"focused",
"tab_id"
],
"properties": {
"dialogs": {
"type": "array",
"items": {
"$ref": "#/definitions/JavaScriptDialogInfo"
}
},
"focused": {
"description": "Whether the target holds DOM focus in its document or shadow root after the call.",
"type": "boolean"
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"used_ref": {
"type": [
"string",
"null"
]
},
"used_selector": {
"type": [
"string",
"null"
]
}
},
"definitions": {
"JavaScriptDialogHandledAction": {
"description": "How the extension resolved the dialog so CDP could continue.",
"type": "string",
"enum": [
"accepted",
"dismissed"
]
},
"JavaScriptDialogInfo": {
"description": "One observed + handled JavaScript dialog during a tool call.",
"type": "object",
"required": [
"handled",
"message",
"sequence",
"tab_id",
"type"
],
"properties": {
"default_prompt": {
"type": [
"string",
"null"
]
},
"handled": {
"$ref": "#/definitions/JavaScriptDialogHandledAction"
},
"has_browser_handler": {
"type": [
"boolean",
"null"
]
},
"message": {
"type": "string"
},
"sequence": {
"description": "Monotonic per-tab sequence for ordering within a session.",
"type": "integer",
"format": "uint64",
"minimum": 0.0
},
"tab_id": {
"type": "integer",
"format": "int64"
},
"type": {
"$ref": "#/definitions/JavaScriptDialogType"
},
"url": {
"type": [
"string",
"null"
]
}
}
},
"JavaScriptDialogType": {
"description": "Native JS dialog kind reported by CDP.",
"type": "string",
"enum": [
"alert",
"confirm",
"prompt",
"beforeunload"
]
}
}
}
@@ -79,6 +79,10 @@ fn main() {
dump!(ClickResult, "tool_click_result"); dump!(ClickResult, "tool_click_result");
dump!(HoverParams, "tool_hover_params"); dump!(HoverParams, "tool_hover_params");
dump!(HoverResult, "tool_hover_result"); dump!(HoverResult, "tool_hover_result");
dump!(FocusParams, "tool_focus_params");
dump!(FocusResult, "tool_focus_result");
dump!(BlurParams, "tool_blur_params");
dump!(BlurResult, "tool_blur_result");
dump!(FillParams, "tool_fill_params"); dump!(FillParams, "tool_fill_params");
dump!(FillResult, "tool_fill_result"); dump!(FillResult, "tool_fill_result");
dump!(PressParams, "tool_press_params"); dump!(PressParams, "tool_press_params");
+12
View File
@@ -72,6 +72,10 @@ pub enum Method {
ToolClick, ToolClick,
#[serde(rename = "tool.hover")] #[serde(rename = "tool.hover")]
ToolHover, ToolHover,
#[serde(rename = "tool.focus")]
ToolFocus,
#[serde(rename = "tool.blur")]
ToolBlur,
#[serde(rename = "tool.fill")] #[serde(rename = "tool.fill")]
ToolFill, ToolFill,
#[serde(rename = "tool.press")] #[serde(rename = "tool.press")]
@@ -168,6 +172,8 @@ impl Method {
| Method::ToolNavigateForward | Method::ToolNavigateForward
| Method::ToolReload | Method::ToolReload
| Method::ToolClick | Method::ToolClick
| Method::ToolFocus
| Method::ToolBlur
| Method::ToolFill | Method::ToolFill
| Method::ToolPress | Method::ToolPress
| Method::ToolSelect | Method::ToolSelect
@@ -305,6 +311,8 @@ mod tests {
assert!(Method::ToolNavigateForward.is_mutating()); assert!(Method::ToolNavigateForward.is_mutating());
assert!(Method::ToolReload.is_mutating()); assert!(Method::ToolReload.is_mutating());
assert!(Method::ToolClick.is_mutating()); assert!(Method::ToolClick.is_mutating());
assert!(Method::ToolFocus.is_mutating());
assert!(Method::ToolBlur.is_mutating());
assert!(Method::ToolFill.is_mutating()); assert!(Method::ToolFill.is_mutating());
assert!(Method::ToolPress.is_mutating()); assert!(Method::ToolPress.is_mutating());
assert!(Method::ToolSelect.is_mutating()); assert!(Method::ToolSelect.is_mutating());
@@ -349,6 +357,8 @@ mod tests {
assert_eq!(Method::ToolHover.effect(), MethodEffect::TransientInput); assert_eq!(Method::ToolHover.effect(), MethodEffect::TransientInput);
assert_eq!(Method::ToolObserve.effect(), MethodEffect::TransientInput); assert_eq!(Method::ToolObserve.effect(), MethodEffect::TransientInput);
assert_eq!(Method::ToolClick.effect(), MethodEffect::BrowserMutation); assert_eq!(Method::ToolClick.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::ToolFocus.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::ToolBlur.effect(), MethodEffect::BrowserMutation);
assert_eq!(Method::Cancel.effect(), MethodEffect::ControlPlane); assert_eq!(Method::Cancel.effect(), MethodEffect::ControlPlane);
} }
@@ -358,6 +368,8 @@ mod tests {
assert!(Method::ToolHover.requires_interrupt_gate()); assert!(Method::ToolHover.requires_interrupt_gate());
assert!(Method::ToolObserve.requires_interrupt_gate()); assert!(Method::ToolObserve.requires_interrupt_gate());
assert!(Method::ToolClick.requires_interrupt_gate()); assert!(Method::ToolClick.requires_interrupt_gate());
assert!(Method::ToolFocus.requires_interrupt_gate());
assert!(Method::ToolBlur.requires_interrupt_gate());
assert!(!Method::Cancel.requires_interrupt_gate()); assert!(!Method::Cancel.requires_interrupt_gate());
} }
} }
+109 -2
View File
@@ -1,5 +1,5 @@
//! DOM interaction tools (`tool.click`, `tool.fill`, `tool.press`, //! DOM interaction tools (`tool.click`, `tool.hover`, `tool.focus`,
//! `tool.select`). //! `tool.blur`, `tool.fill`, `tool.press`, `tool.select`).
//! //!
//! Element-targeted tools accept either a snapshot `ref` (`@e<N>` form, //! Element-targeted tools accept either a snapshot `ref` (`@e<N>` form,
//! normalised against the session's RefStore) **or** a CSS selector //! normalised against the session's RefStore) **or** a CSS selector
@@ -131,6 +131,82 @@ pub struct HoverResult {
pub dialogs: Vec<JavaScriptDialogInfo>, pub dialogs: Vec<JavaScriptDialogInfo>,
} }
// ---------------------------------------------------------------------------
// focus / blur
// ---------------------------------------------------------------------------
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct FocusParams {
pub session_id: String,
/// Optional `@e<N>` ref allocated by the last observation.
/// Mutually exclusive with `selector`.
#[serde(
rename = "ref",
alias = "ref_",
default,
skip_serializing_if = "Option::is_none"
)]
pub ref_: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selector: Option<String>,
/// Target tab. Defaults to the Agent Window's active tab.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tab_id: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 1))]
pub timeout_ms: Option<u32>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct FocusResult {
pub tab_id: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_ref: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_selector: Option<String>,
/// Whether the target holds DOM focus in its document or shadow root after the call.
pub focused: bool,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dialogs: Vec<JavaScriptDialogInfo>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct BlurParams {
pub session_id: String,
/// Optional `@e<N>` ref allocated by the last observation.
/// Mutually exclusive with `selector`.
#[serde(
rename = "ref",
alias = "ref_",
default,
skip_serializing_if = "Option::is_none"
)]
pub ref_: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub selector: Option<String>,
/// Target tab. Defaults to the Agent Window's active tab.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub tab_id: Option<i64>,
#[serde(default, skip_serializing_if = "Option::is_none")]
#[schemars(range(min = 1))]
pub timeout_ms: Option<u32>,
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
pub struct BlurResult {
pub tab_id: i64,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_ref: Option<String>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub used_selector: Option<String>,
/// Whether the target held focus before the call.
pub was_focused: bool,
/// Always false for a successful blur; included for machine verification.
pub focused: bool,
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub dialogs: Vec<JavaScriptDialogInfo>,
}
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
// fill // fill
// --------------------------------------------------------------------------- // ---------------------------------------------------------------------------
@@ -325,6 +401,37 @@ mod tests {
assert_eq!(round, p); assert_eq!(round, p);
} }
#[test]
fn focus_params_serialise_ref_field_name() {
let p = FocusParams {
session_id: "abcd".into(),
ref_: Some("@e4".into()),
selector: None,
tab_id: Some(42),
timeout_ms: Some(5_000),
};
let v = serde_json::to_value(&p).unwrap();
assert_eq!(v.get("ref").and_then(|v| v.as_str()), Some("@e4"));
assert!(v.get("ref_").is_none());
let round: FocusParams = serde_json::from_value(v).unwrap();
assert_eq!(round, p);
}
#[test]
fn blur_result_round_trips_focus_state() {
let result = BlurResult {
tab_id: 42,
used_ref: Some("e4".into()),
used_selector: None,
was_focused: true,
focused: false,
dialogs: vec![],
};
let value = serde_json::to_value(&result).unwrap();
let round: BlurResult = serde_json::from_value(value).unwrap();
assert_eq!(round, result);
}
#[test] #[test]
fn modifiers_render_as_lowercase_strings() { fn modifiers_render_as_lowercase_strings() {
let v = serde_json::to_value(KeyModifier::Ctrl).unwrap(); let v = serde_json::to_value(KeyModifier::Ctrl).unwrap();
+1 -1
View File
@@ -54,7 +54,7 @@ the `bsk` CLI and browser extension separately when a release requires it.
| `browser_session` | `start`, `stop`, `list` | Manage plugin-owned Agent Window sessions. | | `browser_session` | `start`, `stop`, `list` | Manage plugin-owned Agent Window sessions. |
| `browser_page` | `navigate`, `back`, `forward`, `reload`, `wait` | Navigate the active tab and wait for page lifecycle events. | | `browser_page` | `navigate`, `back`, `forward`, `reload`, `wait` | Navigate the active tab and wait for page lifecycle events. |
| `browser_inspect` | `observe`, `snapshot`, `html`, `screenshot`, `console`, `network` | Read semantic or diagnostic page state and capture screenshots. | | `browser_inspect` | `observe`, `snapshot`, `html`, `screenshot`, `console`, `network` | Read semantic or diagnostic page state and capture screenshots. |
| `browser_interact` | `click`, `hover`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. | | `browser_interact` | `click`, `hover`, `focus`, `blur`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. |
| `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. | | `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. |
| `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. | | `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. |
@@ -51,8 +51,8 @@ Use `browser_inspect` action `observe` as the primary semantic page view. It ret
text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may
also become stale after large DOM changes, so observe again before the next interaction. also become stale after large DOM changes, so observe again before the next interaction.
Use `browser_interact` for click, hover, fill, select, and key actions. An observation marks a Use `browser_interact` for click, hover, focus, blur, fill, select, and key actions. Focus and blur
hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are explicitly enter or leave UI states triggered by focus changes. An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are
labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref. labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref.
Do not click the trigger itself unless the user wants the trigger's action. Do not click the trigger itself unless the user wants the trigger's action.
@@ -167,12 +167,14 @@ const BROWSER_TOOL_SPECS: BrowserToolSpec[] = [
{ {
name: "browser_interact", name: "browser_interact",
description: description:
"Interact with an element in the active Agent Window tab. Actions: click, hover, fill, select, " + "Interact with an element in the active Agent Window tab. Actions: click, hover, focus, blur, fill, select, " +
"press. click/hover/fill/select require target; fill also requires value; select requires " + "press. click/hover/focus/blur/fill/select require target; fill also requires value; select requires " +
"values; press requires key and may optionally focus target first.", "values; press requires key and may optionally focus target first.",
actions: { actions: {
click: "interact.click", click: "interact.click",
hover: "interact.hover", hover: "interact.hover",
focus: "interact.focus",
blur: "interact.blur",
fill: "interact.fill", fill: "interact.fill",
select: "interact.select", select: "interact.select",
press: "interact.press", press: "interact.press",
@@ -562,6 +562,10 @@ export function actionForLabel(label: string): string {
return "clicking"; return "clicking";
case "hover": case "hover":
return "hovering"; return "hovering";
case "focus":
return "focusing";
case "blur":
return "blurring";
case "fill": case "fill":
return "filling"; return "filling";
case "select": case "select":
@@ -13,7 +13,7 @@ import type { ToolDeps } from "./tools";
const MODIFIERS = ["alt", "ctrl", "meta", "shift"] as const; const MODIFIERS = ["alt", "ctrl", "meta", "shift"] as const;
/** Add hover and select without bypassing session ownership or observation. */ /** Add hover, focus, blur and select without bypassing session ownership or observation. */
export function registerPhaseOneInteractionTools( export function registerPhaseOneInteractionTools(
deps: ToolDeps, deps: ToolDeps,
register: ToolRegistrar, register: ToolRegistrar,
@@ -100,6 +100,88 @@ export function registerPhaseOneInteractionTools(
}), }),
); );
for (const action of ["focus", "blur"] as const) {
register(
defineTool({
name: `interact.${action}`,
description:
action === "focus"
? "Focus an element and verify its DOM focus state."
: "Remove focus from an element and report whether it was focused.",
parameters: {
target: {
type: "string",
required: true,
description: "Snapshot ref (@e3 / e3) or CSS selector of the element.",
},
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
timeoutMs: TIMEOUT_MS_PARAM,
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
focused: { type: "boolean", required: true },
wasFocused: {
type: "boolean",
description: "Whether the target was focused before blur.",
},
},
},
render: (_args, value) => [
{
type: "text",
text:
`[session ${value.session}] ${action} on tab ${value.tabId}: focused=${value.focused}` +
(value.wasFocused === undefined ? "" : `, wasFocused=${value.wasFocused}`),
},
],
},
async execute(args, exec) {
requireNonEmpty(args.target, "target");
requirePositive(args.timeoutMs, "timeoutMs");
const sessionId = registry.resolve(args.session, `browser_interact(action=${action})`);
const cmdArgs = [action, "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`);
appendTarget(cmdArgs, args.target);
const reply = (await runtime.run(
exec,
cmdArgs,
action,
sessionId,
runnerTimeout(deps, args.timeoutMs),
)) as {
tab_id: number;
focused: boolean;
was_focused?: boolean;
};
return {
session: sessionId,
tabId: reply.tab_id,
focused: reply.focused,
...(action === "blur" ? { wasFocused: reply.was_focused } : {}),
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
action,
args.target,
"--session",
args.session ?? "(current)",
]),
description: action === "focus" ? "Focus an element" : "Remove focus from an element",
}),
presentResult: runtime.presentTerminalResult,
}),
);
}
register( register(
defineTool({ defineTool({
name: "interact.select", name: "interact.select",
@@ -102,6 +102,8 @@ const ACTION_ROUTES: Record<string, readonly [string, string]> = {
"inspect.network": ["browser_inspect", "network"], "inspect.network": ["browser_inspect", "network"],
"interact.click": ["browser_interact", "click"], "interact.click": ["browser_interact", "click"],
"interact.hover": ["browser_interact", "hover"], "interact.hover": ["browser_interact", "hover"],
"interact.focus": ["browser_interact", "focus"],
"interact.blur": ["browser_interact", "blur"],
"interact.fill": ["browser_interact", "fill"], "interact.fill": ["browser_interact", "fill"],
"interact.select": ["browser_interact", "select"], "interact.select": ["browser_interact", "select"],
"interact.press": ["browser_interact", "press"], "interact.press": ["browser_interact", "press"],
@@ -200,7 +202,7 @@ const EXPECTED_ACTIONS = {
browser_session: ["start", "stop", "list"], browser_session: ["start", "stop", "list"],
browser_page: ["navigate", "back", "forward", "reload", "wait"], browser_page: ["navigate", "back", "forward", "reload", "wait"],
browser_inspect: ["observe", "snapshot", "html", "screenshot", "console", "network"], browser_inspect: ["observe", "snapshot", "html", "screenshot", "console", "network"],
browser_interact: ["click", "hover", "fill", "select", "press"], browser_interact: ["click", "hover", "focus", "blur", "fill", "select", "press"],
browser_tabs: ["list", "create", "select", "close", "borrow", "return"], browser_tabs: ["list", "create", "select", "close", "borrow", "return"],
browser_assist: ["resize", "emulate", "request-help"], browser_assist: ["resize", "emulate", "request-help"],
} as const; } as const;
@@ -499,6 +501,60 @@ describe("interaction tools", () => {
]); ]);
}); });
it.each([
"focus",
"blur",
] as const)("interact.%s maps focus state, target, tab and timeout", async (action) => {
const { tools, calls } = setup({
"session start": START_REPLY("s1"),
[action]: {
tab_id: 7,
focused: action === "focus",
...(action === "blur" ? { was_focused: true } : {}),
},
});
await startSession(tools);
const target = action === "focus" ? "@e3" : "#field";
const value = await tools
.get(`interact.${action}`)
?.execute({ target, tabId: 7, timeoutMs: 150_000 }, makeExec());
expect(value).toEqual({
session: "s1",
tabId: 7,
focused: action === "focus",
...(action === "blur" ? { wasFocused: true } : {}),
});
expect(calls[1].args).toEqual([
action,
"--session",
"s1",
"--tab-id",
"7",
"--timeout",
"150000ms",
target,
]);
expect(calls[1].options.timeoutMs).toBe(165_000);
});
it.each([
"focus",
"blur",
] as const)("interact.%s validates arguments and session ownership before running", async (action) => {
const { tools, calls } = setup({ "session start": START_REPLY("s1") });
await startSession(tools);
const tool = tools.get(`interact.${action}`)!;
await expect(tool.execute({}, makeExec())).rejects.toThrow(/invalid arguments/i);
await expect(tool.execute({ target: " " }, makeExec())).rejects.toThrow(/non-empty/i);
await expect(tool.execute({ target: "e1", timeoutMs: 0 }, makeExec())).rejects.toThrow(
/greater than zero/i,
);
await expect(tool.execute({ target: "e1", session: "foreign" }, makeExec())).rejects.toThrow(
/session/i,
);
expect(calls).toHaveLength(1);
});
it("interact.fill passes value and --no-clear", async () => { it("interact.fill passes value and --no-clear", async () => {
const { tools, calls } = setup(responses); const { tools, calls } = setup(responses);
await startSession(tools); await startSession(tools);
+5 -2
View File
@@ -55,10 +55,13 @@ Use this default loop:
```text ```text
bsk navigate <url> --session <id> bsk navigate <url> --session <id>
bsk observe --session <id> bsk observe --session <id>
bsk click|hover|fill|select|press ... --session <id> bsk click|hover|focus|blur|fill|select|press ... --session <id>
bsk observe --session <id> # after navigation or a meaningful DOM change bsk observe --session <id> # after navigation or a meaningful DOM change
``` ```
`bsk focus <ref>` explicitly focuses a target; `bsk blur <ref>` removes focus and reports whether
it was focused. Use these for UI states triggered by focus changes.
Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also
make them stale. Observe again before the next interaction. make them stale. Observe again before the next interaction.
@@ -114,7 +117,7 @@ This list of names is complete. Never invent a command outside it; read
session start|stop|list browsers status doctor update logs session start|stop|list browsers status doctor update logs
navigate navigate-back navigate-forward reload wait-for-navigation wait-ms navigate navigate-back navigate-forward reload wait-for-navigation wait-ms
observe snapshot get-html screenshot console network observe snapshot get-html screenshot console network
click hover fill select press evaluate click hover focus blur fill select press evaluate
tab list|create|close|select|borrow|return window resize emulate tab list|create|close|select|borrow|return window resize emulate
upload download request-help record start|stop upload download request-help record start|stop
``` ```