diff --git a/packages/dsh-plugin-browserskill/README.md b/packages/dsh-plugin-browserskill/README.md index 83b3a6a..e02453a 100644 --- a/packages/dsh-plugin-browserskill/README.md +++ b/packages/dsh-plugin-browserskill/README.md @@ -54,7 +54,7 @@ the `bsk` CLI and browser extension separately when a release requires it. | `browser_session` | `start`, `stop`, `list` | Manage plugin-owned Agent Window sessions. | | `browser_page` | `navigate`, `back`, `forward`, `reload`, `wait` | Navigate the active tab and wait for page lifecycle events. | | `browser_inspect` | `observe`, `snapshot`, `html`, `screenshot`, `console`, `network` | Read semantic or diagnostic page state and capture screenshots. | -| `browser_interact` | `click`, `hover`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. | +| `browser_interact` | `click`, `hover`, `focus`, `blur`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. | | `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. | | `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. | diff --git a/packages/dsh-plugin-browserskill/skill/SKILL.md b/packages/dsh-plugin-browserskill/skill/SKILL.md index 464a24e..6ca293a 100644 --- a/packages/dsh-plugin-browserskill/skill/SKILL.md +++ b/packages/dsh-plugin-browserskill/skill/SKILL.md @@ -51,8 +51,8 @@ Use `browser_inspect` action `observe` as the primary semantic page view. It ret text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may also become stale after large DOM changes, so observe again before the next interaction. -Use `browser_interact` for click, hover, fill, select, and key actions. An observation marks a -hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are +Use `browser_interact` for click, hover, focus, blur, fill, select, and key actions. Focus and blur +explicitly enter or leave UI states triggered by focus changes. An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref. Do not click the trigger itself unless the user wants the trigger's action. diff --git a/packages/dsh-plugin-browserskill/src/browser-tools.ts b/packages/dsh-plugin-browserskill/src/browser-tools.ts index b69a83d..3204eed 100644 --- a/packages/dsh-plugin-browserskill/src/browser-tools.ts +++ b/packages/dsh-plugin-browserskill/src/browser-tools.ts @@ -167,12 +167,14 @@ const BROWSER_TOOL_SPECS: BrowserToolSpec[] = [ { name: "browser_interact", description: - "Interact with an element in the active Agent Window tab. Actions: click, hover, fill, select, " + - "press. click/hover/fill/select require target; fill also requires value; select requires " + + "Interact with an element in the active Agent Window tab. Actions: click, hover, focus, blur, fill, select, " + + "press. click/hover/focus/blur/fill/select require target; fill also requires value; select requires " + "values; press requires key and may optionally focus target first.", actions: { click: "interact.click", hover: "interact.hover", + focus: "interact.focus", + blur: "interact.blur", fill: "interact.fill", select: "interact.select", press: "interact.press", diff --git a/packages/dsh-plugin-browserskill/src/observation.ts b/packages/dsh-plugin-browserskill/src/observation.ts index efdd9c1..31f13c3 100644 --- a/packages/dsh-plugin-browserskill/src/observation.ts +++ b/packages/dsh-plugin-browserskill/src/observation.ts @@ -562,6 +562,10 @@ export function actionForLabel(label: string): string { return "clicking"; case "hover": return "hovering"; + case "focus": + return "focusing"; + case "blur": + return "blurring"; case "fill": return "filling"; case "select": diff --git a/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts b/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts index 3c7c4f9..858fcc2 100644 --- a/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts +++ b/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts @@ -13,7 +13,7 @@ import type { ToolDeps } from "./tools"; const MODIFIERS = ["alt", "ctrl", "meta", "shift"] as const; -/** Add hover and select without bypassing session ownership or observation. */ +/** Add hover, focus, blur and select without bypassing session ownership or observation. */ export function registerPhaseOneInteractionTools( deps: ToolDeps, register: ToolRegistrar, @@ -100,6 +100,88 @@ export function registerPhaseOneInteractionTools( }), ); + for (const action of ["focus", "blur"] as const) { + register( + defineTool({ + name: `interact.${action}`, + description: + action === "focus" + ? "Focus an element and verify its DOM focus state." + : "Remove focus from an element and report whether it was focused.", + parameters: { + target: { + type: "string", + required: true, + description: "Snapshot ref (@e3 / e3) or CSS selector of the element.", + }, + session: SESSION_PARAM, + tabId: TAB_ID_PARAM, + timeoutMs: TIMEOUT_MS_PARAM, + }, + output: { + schema: { + type: "object", + additionalProperties: false, + properties: { + session: { type: "string", required: true }, + tabId: { type: "integer", required: true }, + focused: { type: "boolean", required: true }, + wasFocused: { + type: "boolean", + description: "Whether the target was focused before blur.", + }, + }, + }, + render: (_args, value) => [ + { + type: "text", + text: + `[session ${value.session}] ${action} on tab ${value.tabId}: focused=${value.focused}` + + (value.wasFocused === undefined ? "" : `, wasFocused=${value.wasFocused}`), + }, + ], + }, + async execute(args, exec) { + requireNonEmpty(args.target, "target"); + requirePositive(args.timeoutMs, "timeoutMs"); + const sessionId = registry.resolve(args.session, `browser_interact(action=${action})`); + const cmdArgs = [action, "--session", sessionId]; + appendTabId(cmdArgs, args.tabId); + if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`); + appendTarget(cmdArgs, args.target); + const reply = (await runtime.run( + exec, + cmdArgs, + action, + sessionId, + runnerTimeout(deps, args.timeoutMs), + )) as { + tab_id: number; + focused: boolean; + was_focused?: boolean; + }; + return { + session: sessionId, + tabId: reply.tab_id, + focused: reply.focused, + ...(action === "blur" ? { wasFocused: reply.was_focused } : {}), + }; + }, + presentCall: (args) => ({ + card: "terminal", + title: runtime.commandLine([ + action, + args.target, + "--session", + args.session ?? "(current)", + ]), + description: action === "focus" ? "Focus an element" : "Remove focus from an element", + }), + presentResult: runtime.presentTerminalResult, + }), + ); + } + register( defineTool({ name: "interact.select", diff --git a/packages/dsh-plugin-browserskill/tests/tools.test.ts b/packages/dsh-plugin-browserskill/tests/tools.test.ts index 7faf3fa..e144199 100644 --- a/packages/dsh-plugin-browserskill/tests/tools.test.ts +++ b/packages/dsh-plugin-browserskill/tests/tools.test.ts @@ -102,6 +102,8 @@ const ACTION_ROUTES: Record = { "inspect.network": ["browser_inspect", "network"], "interact.click": ["browser_interact", "click"], "interact.hover": ["browser_interact", "hover"], + "interact.focus": ["browser_interact", "focus"], + "interact.blur": ["browser_interact", "blur"], "interact.fill": ["browser_interact", "fill"], "interact.select": ["browser_interact", "select"], "interact.press": ["browser_interact", "press"], @@ -200,7 +202,7 @@ const EXPECTED_ACTIONS = { browser_session: ["start", "stop", "list"], browser_page: ["navigate", "back", "forward", "reload", "wait"], browser_inspect: ["observe", "snapshot", "html", "screenshot", "console", "network"], - browser_interact: ["click", "hover", "fill", "select", "press"], + browser_interact: ["click", "hover", "focus", "blur", "fill", "select", "press"], browser_tabs: ["list", "create", "select", "close", "borrow", "return"], browser_assist: ["resize", "emulate", "request-help"], } as const; @@ -499,6 +501,60 @@ describe("interaction tools", () => { ]); }); + it.each([ + "focus", + "blur", + ] as const)("interact.%s maps focus state, target, tab and timeout", async (action) => { + const { tools, calls } = setup({ + "session start": START_REPLY("s1"), + [action]: { + tab_id: 7, + focused: action === "focus", + ...(action === "blur" ? { was_focused: true } : {}), + }, + }); + await startSession(tools); + const target = action === "focus" ? "@e3" : "#field"; + const value = await tools + .get(`interact.${action}`) + ?.execute({ target, tabId: 7, timeoutMs: 150_000 }, makeExec()); + expect(value).toEqual({ + session: "s1", + tabId: 7, + focused: action === "focus", + ...(action === "blur" ? { wasFocused: true } : {}), + }); + expect(calls[1].args).toEqual([ + action, + "--session", + "s1", + "--tab-id", + "7", + "--timeout", + "150000ms", + target, + ]); + expect(calls[1].options.timeoutMs).toBe(165_000); + }); + + it.each([ + "focus", + "blur", + ] as const)("interact.%s validates arguments and session ownership before running", async (action) => { + const { tools, calls } = setup({ "session start": START_REPLY("s1") }); + await startSession(tools); + const tool = tools.get(`interact.${action}`)!; + await expect(tool.execute({}, makeExec())).rejects.toThrow(/invalid arguments/i); + await expect(tool.execute({ target: " " }, makeExec())).rejects.toThrow(/non-empty/i); + await expect(tool.execute({ target: "e1", timeoutMs: 0 }, makeExec())).rejects.toThrow( + /greater than zero/i, + ); + await expect(tool.execute({ target: "e1", session: "foreign" }, makeExec())).rejects.toThrow( + /session/i, + ); + expect(calls).toHaveLength(1); + }); + it("interact.fill passes value and --no-clear", async () => { const { tools, calls } = setup(responses); await startSession(tools);