mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(dsh-plugin): expose focus and blur actions
This commit is contained in:
@@ -54,7 +54,7 @@ the `bsk` CLI and browser extension separately when a release requires it.
|
||||
| `browser_session` | `start`, `stop`, `list` | Manage plugin-owned Agent Window sessions. |
|
||||
| `browser_page` | `navigate`, `back`, `forward`, `reload`, `wait` | Navigate the active tab and wait for page lifecycle events. |
|
||||
| `browser_inspect` | `observe`, `snapshot`, `html`, `screenshot`, `console`, `network` | Read semantic or diagnostic page state and capture screenshots. |
|
||||
| `browser_interact` | `click`, `hover`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. |
|
||||
| `browser_interact` | `click`, `hover`, `focus`, `blur`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. |
|
||||
| `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. |
|
||||
| `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. |
|
||||
|
||||
|
||||
@@ -51,8 +51,8 @@ Use `browser_inspect` action `observe` as the primary semantic page view. It ret
|
||||
text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may
|
||||
also become stale after large DOM changes, so observe again before the next interaction.
|
||||
|
||||
Use `browser_interact` for click, hover, fill, select, and key actions. An observation marks a
|
||||
hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are
|
||||
Use `browser_interact` for click, hover, focus, blur, fill, select, and key actions. Focus and blur
|
||||
explicitly enter or leave UI states triggered by focus changes. An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are
|
||||
labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref.
|
||||
Do not click the trigger itself unless the user wants the trigger's action.
|
||||
|
||||
|
||||
@@ -167,12 +167,14 @@ const BROWSER_TOOL_SPECS: BrowserToolSpec[] = [
|
||||
{
|
||||
name: "browser_interact",
|
||||
description:
|
||||
"Interact with an element in the active Agent Window tab. Actions: click, hover, fill, select, " +
|
||||
"press. click/hover/fill/select require target; fill also requires value; select requires " +
|
||||
"Interact with an element in the active Agent Window tab. Actions: click, hover, focus, blur, fill, select, " +
|
||||
"press. click/hover/focus/blur/fill/select require target; fill also requires value; select requires " +
|
||||
"values; press requires key and may optionally focus target first.",
|
||||
actions: {
|
||||
click: "interact.click",
|
||||
hover: "interact.hover",
|
||||
focus: "interact.focus",
|
||||
blur: "interact.blur",
|
||||
fill: "interact.fill",
|
||||
select: "interact.select",
|
||||
press: "interact.press",
|
||||
|
||||
@@ -562,6 +562,10 @@ export function actionForLabel(label: string): string {
|
||||
return "clicking";
|
||||
case "hover":
|
||||
return "hovering";
|
||||
case "focus":
|
||||
return "focusing";
|
||||
case "blur":
|
||||
return "blurring";
|
||||
case "fill":
|
||||
return "filling";
|
||||
case "select":
|
||||
|
||||
@@ -13,7 +13,7 @@ import type { ToolDeps } from "./tools";
|
||||
|
||||
const MODIFIERS = ["alt", "ctrl", "meta", "shift"] as const;
|
||||
|
||||
/** Add hover and select without bypassing session ownership or observation. */
|
||||
/** Add hover, focus, blur and select without bypassing session ownership or observation. */
|
||||
export function registerPhaseOneInteractionTools(
|
||||
deps: ToolDeps,
|
||||
register: ToolRegistrar,
|
||||
@@ -100,6 +100,88 @@ export function registerPhaseOneInteractionTools(
|
||||
}),
|
||||
);
|
||||
|
||||
for (const action of ["focus", "blur"] as const) {
|
||||
register(
|
||||
defineTool({
|
||||
name: `interact.${action}`,
|
||||
description:
|
||||
action === "focus"
|
||||
? "Focus an element and verify its DOM focus state."
|
||||
: "Remove focus from an element and report whether it was focused.",
|
||||
parameters: {
|
||||
target: {
|
||||
type: "string",
|
||||
required: true,
|
||||
description: "Snapshot ref (@e3 / e3) or CSS selector of the element.",
|
||||
},
|
||||
session: SESSION_PARAM,
|
||||
tabId: TAB_ID_PARAM,
|
||||
timeoutMs: TIMEOUT_MS_PARAM,
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
focused: { type: "boolean", required: true },
|
||||
wasFocused: {
|
||||
type: "boolean",
|
||||
description: "Whether the target was focused before blur.",
|
||||
},
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text:
|
||||
`[session ${value.session}] ${action} on tab ${value.tabId}: focused=${value.focused}` +
|
||||
(value.wasFocused === undefined ? "" : `, wasFocused=${value.wasFocused}`),
|
||||
},
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
requireNonEmpty(args.target, "target");
|
||||
requirePositive(args.timeoutMs, "timeoutMs");
|
||||
const sessionId = registry.resolve(args.session, `browser_interact(action=${action})`);
|
||||
const cmdArgs = [action, "--session", sessionId];
|
||||
appendTabId(cmdArgs, args.tabId);
|
||||
if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`);
|
||||
appendTarget(cmdArgs, args.target);
|
||||
const reply = (await runtime.run(
|
||||
exec,
|
||||
cmdArgs,
|
||||
action,
|
||||
sessionId,
|
||||
runnerTimeout(deps, args.timeoutMs),
|
||||
)) as {
|
||||
tab_id: number;
|
||||
focused: boolean;
|
||||
was_focused?: boolean;
|
||||
};
|
||||
return {
|
||||
session: sessionId,
|
||||
tabId: reply.tab_id,
|
||||
focused: reply.focused,
|
||||
...(action === "blur" ? { wasFocused: reply.was_focused } : {}),
|
||||
};
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: runtime.commandLine([
|
||||
action,
|
||||
args.target,
|
||||
"--session",
|
||||
args.session ?? "(current)",
|
||||
]),
|
||||
description: action === "focus" ? "Focus an element" : "Remove focus from an element",
|
||||
}),
|
||||
presentResult: runtime.presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "interact.select",
|
||||
|
||||
@@ -102,6 +102,8 @@ const ACTION_ROUTES: Record<string, readonly [string, string]> = {
|
||||
"inspect.network": ["browser_inspect", "network"],
|
||||
"interact.click": ["browser_interact", "click"],
|
||||
"interact.hover": ["browser_interact", "hover"],
|
||||
"interact.focus": ["browser_interact", "focus"],
|
||||
"interact.blur": ["browser_interact", "blur"],
|
||||
"interact.fill": ["browser_interact", "fill"],
|
||||
"interact.select": ["browser_interact", "select"],
|
||||
"interact.press": ["browser_interact", "press"],
|
||||
@@ -200,7 +202,7 @@ const EXPECTED_ACTIONS = {
|
||||
browser_session: ["start", "stop", "list"],
|
||||
browser_page: ["navigate", "back", "forward", "reload", "wait"],
|
||||
browser_inspect: ["observe", "snapshot", "html", "screenshot", "console", "network"],
|
||||
browser_interact: ["click", "hover", "fill", "select", "press"],
|
||||
browser_interact: ["click", "hover", "focus", "blur", "fill", "select", "press"],
|
||||
browser_tabs: ["list", "create", "select", "close", "borrow", "return"],
|
||||
browser_assist: ["resize", "emulate", "request-help"],
|
||||
} as const;
|
||||
@@ -499,6 +501,60 @@ describe("interaction tools", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"focus",
|
||||
"blur",
|
||||
] as const)("interact.%s maps focus state, target, tab and timeout", async (action) => {
|
||||
const { tools, calls } = setup({
|
||||
"session start": START_REPLY("s1"),
|
||||
[action]: {
|
||||
tab_id: 7,
|
||||
focused: action === "focus",
|
||||
...(action === "blur" ? { was_focused: true } : {}),
|
||||
},
|
||||
});
|
||||
await startSession(tools);
|
||||
const target = action === "focus" ? "@e3" : "#field";
|
||||
const value = await tools
|
||||
.get(`interact.${action}`)
|
||||
?.execute({ target, tabId: 7, timeoutMs: 150_000 }, makeExec());
|
||||
expect(value).toEqual({
|
||||
session: "s1",
|
||||
tabId: 7,
|
||||
focused: action === "focus",
|
||||
...(action === "blur" ? { wasFocused: true } : {}),
|
||||
});
|
||||
expect(calls[1].args).toEqual([
|
||||
action,
|
||||
"--session",
|
||||
"s1",
|
||||
"--tab-id",
|
||||
"7",
|
||||
"--timeout",
|
||||
"150000ms",
|
||||
target,
|
||||
]);
|
||||
expect(calls[1].options.timeoutMs).toBe(165_000);
|
||||
});
|
||||
|
||||
it.each([
|
||||
"focus",
|
||||
"blur",
|
||||
] as const)("interact.%s validates arguments and session ownership before running", async (action) => {
|
||||
const { tools, calls } = setup({ "session start": START_REPLY("s1") });
|
||||
await startSession(tools);
|
||||
const tool = tools.get(`interact.${action}`)!;
|
||||
await expect(tool.execute({}, makeExec())).rejects.toThrow(/invalid arguments/i);
|
||||
await expect(tool.execute({ target: " " }, makeExec())).rejects.toThrow(/non-empty/i);
|
||||
await expect(tool.execute({ target: "e1", timeoutMs: 0 }, makeExec())).rejects.toThrow(
|
||||
/greater than zero/i,
|
||||
);
|
||||
await expect(tool.execute({ target: "e1", session: "foreign" }, makeExec())).rejects.toThrow(
|
||||
/session/i,
|
||||
);
|
||||
expect(calls).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("interact.fill passes value and --no-clear", async () => {
|
||||
const { tools, calls } = setup(responses);
|
||||
await startSession(tools);
|
||||
|
||||
Reference in New Issue
Block a user