From 77e3f5e1f338465c3f507cdec3a26168da8048f1 Mon Sep 17 00:00:00 2001 From: drakezhang Date: Thu, 10 Sep 2026 00:05:52 +0800 Subject: [PATCH] feat(dsh-plugin): expose scroll-to action --- packages/dsh-plugin-browserskill/README.md | 2 +- .../dsh-plugin-browserskill/skill/SKILL.md | 7 +- .../src/browser-tools.ts | 5 +- .../src/phase-one-tools-interaction.ts | 85 ++++++++++++++++++- .../tests/tools.test.ts | 28 +++++- 5 files changed, 121 insertions(+), 6 deletions(-) diff --git a/packages/dsh-plugin-browserskill/README.md b/packages/dsh-plugin-browserskill/README.md index e02453a..42e141b 100644 --- a/packages/dsh-plugin-browserskill/README.md +++ b/packages/dsh-plugin-browserskill/README.md @@ -54,7 +54,7 @@ the `bsk` CLI and browser extension separately when a release requires it. | `browser_session` | `start`, `stop`, `list` | Manage plugin-owned Agent Window sessions. | | `browser_page` | `navigate`, `back`, `forward`, `reload`, `wait` | Navigate the active tab and wait for page lifecycle events. | | `browser_inspect` | `observe`, `snapshot`, `html`, `screenshot`, `console`, `network` | Read semantic or diagnostic page state and capture screenshots. | -| `browser_interact` | `click`, `hover`, `focus`, `blur`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. | +| `browser_interact` | `click`, `hover`, `scroll-to`, `focus`, `blur`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. | | `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. | | `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. | diff --git a/packages/dsh-plugin-browserskill/skill/SKILL.md b/packages/dsh-plugin-browserskill/skill/SKILL.md index 6ca293a..55a744e 100644 --- a/packages/dsh-plugin-browserskill/skill/SKILL.md +++ b/packages/dsh-plugin-browserskill/skill/SKILL.md @@ -51,7 +51,12 @@ Use `browser_inspect` action `observe` as the primary semantic page view. It ret text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may also become stale after large DOM changes, so observe again before the next interaction. -Use `browser_interact` for click, hover, focus, blur, fill, select, and key actions. Focus and blur +Use `browser_interact` with `action=scroll-to` to bring an element into view. Its bounds are the +visible border-box portion in top-level viewport CSS pixels after ancestor clipping; partial +visibility is enough. Hidden or fully clipped targets fail. This does not test occlusion by other +elements. Use fresh element refs for iframe/shadow-root targets; selectors search the main document. + +Use `browser_interact` for click, hover, scroll-to, focus, blur, fill, select, and key actions. Focus and blur explicitly enter or leave UI states triggered by focus changes. An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref. Do not click the trigger itself unless the user wants the trigger's action. diff --git a/packages/dsh-plugin-browserskill/src/browser-tools.ts b/packages/dsh-plugin-browserskill/src/browser-tools.ts index 3204eed..d002a9a 100644 --- a/packages/dsh-plugin-browserskill/src/browser-tools.ts +++ b/packages/dsh-plugin-browserskill/src/browser-tools.ts @@ -167,12 +167,13 @@ const BROWSER_TOOL_SPECS: BrowserToolSpec[] = [ { name: "browser_interact", description: - "Interact with an element in the active Agent Window tab. Actions: click, hover, focus, blur, fill, select, " + - "press. click/hover/focus/blur/fill/select require target; fill also requires value; select requires " + + "Interact with an element in the active Agent Window tab. Actions: click, hover, scroll-to, focus, blur, fill, select, " + + "press. click/hover/scroll-to/focus/blur/fill/select require target; fill also requires value; select requires " + "values; press requires key and may optionally focus target first.", actions: { click: "interact.click", hover: "interact.hover", + "scroll-to": "interact.scroll-to", focus: "interact.focus", blur: "interact.blur", fill: "interact.fill", diff --git a/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts b/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts index 858fcc2..5048021 100644 --- a/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts +++ b/packages/dsh-plugin-browserskill/src/phase-one-tools-interaction.ts @@ -13,7 +13,7 @@ import type { ToolDeps } from "./tools"; const MODIFIERS = ["alt", "ctrl", "meta", "shift"] as const; -/** Add hover, focus, blur and select without bypassing session ownership or observation. */ +/** Add interaction primitives without bypassing session ownership or observation. */ export function registerPhaseOneInteractionTools( deps: ToolDeps, register: ToolRegistrar, @@ -100,6 +100,89 @@ export function registerPhaseOneInteractionTools( }), ); + register( + defineTool({ + name: "interact.scroll-to", + description: + "Scroll an element and its frame owners into view. Returns the visible portion's " + + "bounds in top-level viewport CSS pixels; fails if no visible area remains. " + + "Use a fresh ref for iframe targets; selectors search the main document.", + parameters: { + target: { + type: "string", + required: true, + description: "Element ref (@e3 / e3) or main-document CSS selector to scroll into view.", + }, + session: SESSION_PARAM, + tabId: TAB_ID_PARAM, + timeoutMs: TIMEOUT_MS_PARAM, + }, + output: { + schema: { + type: "object", + additionalProperties: false, + properties: { + session: { type: "string", required: true }, + tabId: { type: "integer", required: true }, + x: { type: "number", required: true }, + y: { type: "number", required: true }, + width: { type: "number", required: true }, + height: { type: "number", required: true }, + }, + }, + render: (_args, value) => [ + { + type: "text", + text: + `[session ${value.session}] scrolled into view on tab ${value.tabId}: ` + + `(${value.x}, ${value.y}, ${value.width}, ${value.height})`, + }, + ], + }, + async execute(args, exec) { + requireNonEmpty(args.target, "target"); + requirePositive(args.timeoutMs, "timeoutMs"); + const sessionId = registry.resolve(args.session, "browser_interact(action=scroll-to)"); + const cmdArgs = ["scroll-to", "--session", sessionId]; + appendTabId(cmdArgs, args.tabId); + if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`); + appendTarget(cmdArgs, args.target); + const reply = (await runtime.run( + exec, + cmdArgs, + "scroll-to", + sessionId, + runnerTimeout(deps, args.timeoutMs), + )) as { + tab_id: number; + x: number; + y: number; + width: number; + height: number; + }; + return { + session: sessionId, + tabId: reply.tab_id, + x: reply.x, + y: reply.y, + width: reply.width, + height: reply.height, + }; + }, + presentCall: (args) => ({ + card: "terminal", + title: runtime.commandLine([ + "scroll-to", + args.target, + "--session", + args.session ?? "(current)", + ]), + description: "Scroll an element into view", + }), + presentResult: runtime.presentTerminalResult, + }), + ); + for (const action of ["focus", "blur"] as const) { register( defineTool({ diff --git a/packages/dsh-plugin-browserskill/tests/tools.test.ts b/packages/dsh-plugin-browserskill/tests/tools.test.ts index e144199..ab7686c 100644 --- a/packages/dsh-plugin-browserskill/tests/tools.test.ts +++ b/packages/dsh-plugin-browserskill/tests/tools.test.ts @@ -102,6 +102,7 @@ const ACTION_ROUTES: Record = { "inspect.network": ["browser_inspect", "network"], "interact.click": ["browser_interact", "click"], "interact.hover": ["browser_interact", "hover"], + "interact.scroll-to": ["browser_interact", "scroll-to"], "interact.focus": ["browser_interact", "focus"], "interact.blur": ["browser_interact", "blur"], "interact.fill": ["browser_interact", "fill"], @@ -202,7 +203,7 @@ const EXPECTED_ACTIONS = { browser_session: ["start", "stop", "list"], browser_page: ["navigate", "back", "forward", "reload", "wait"], browser_inspect: ["observe", "snapshot", "html", "screenshot", "console", "network"], - browser_interact: ["click", "hover", "focus", "blur", "fill", "select", "press"], + browser_interact: ["click", "hover", "scroll-to", "focus", "blur", "fill", "select", "press"], browser_tabs: ["list", "create", "select", "close", "borrow", "return"], browser_assist: ["resize", "emulate", "request-help"], } as const; @@ -540,6 +541,7 @@ describe("interaction tools", () => { it.each([ "focus", "blur", + "scroll-to", ] as const)("interact.%s validates arguments and session ownership before running", async (action) => { const { tools, calls } = setup({ "session start": START_REPLY("s1") }); await startSession(tools); @@ -555,6 +557,30 @@ describe("interaction tools", () => { expect(calls).toHaveLength(1); }); + it.each(["@e3", "#target"])("interact.scroll-to maps %s, bounds and timeout", async (target) => { + const { tools, calls } = setup({ + "session start": START_REPLY("s1"), + "scroll-to": { tab_id: 7, x: 10, y: 20, width: 100, height: 80 }, + }); + await startSession(tools); + expect( + await tools + .get("interact.scroll-to") + ?.execute({ target, tabId: 7, timeoutMs: 150_000 }, makeExec()), + ).toEqual({ session: "s1", tabId: 7, x: 10, y: 20, width: 100, height: 80 }); + expect(calls[1].args).toEqual([ + "scroll-to", + "--session", + "s1", + "--tab-id", + "7", + "--timeout", + "150000ms", + target, + ]); + expect(calls[1].options.timeoutMs).toBe(165_000); + }); + it("interact.fill passes value and --no-clear", async () => { const { tools, calls } = setup(responses); await startSession(tools);