feat(dsh-plugin): expose scroll-to action

This commit is contained in:
drakezhang committed 2026-09-10 00:05:52 +08:00
1 parent 61c97c9cb5
commit 77e3f5e1f3
5 files changed
+121 -6

No files matched your search

+1 -1
View File
@@ -54,7 +54,7 @@ the `bsk` CLI and browser extension separately when a release requires it.
| `browser_session` | `start`, `stop`, `list` | Manage plugin-owned Agent Window sessions. | | `browser_session` | `start`, `stop`, `list` | Manage plugin-owned Agent Window sessions. |
| `browser_page` | `navigate`, `back`, `forward`, `reload`, `wait` | Navigate the active tab and wait for page lifecycle events. | | `browser_page` | `navigate`, `back`, `forward`, `reload`, `wait` | Navigate the active tab and wait for page lifecycle events. |
| `browser_inspect` | `observe`, `snapshot`, `html`, `screenshot`, `console`, `network` | Read semantic or diagnostic page state and capture screenshots. | | `browser_inspect` | `observe`, `snapshot`, `html`, `screenshot`, `console`, `network` | Read semantic or diagnostic page state and capture screenshots. |
| `browser_interact` | `click`, `hover`, `focus`, `blur`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. | | `browser_interact` | `click`, `hover`, `scroll-to`, `focus`, `blur`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. |
| `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. | | `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. |
| `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. | | `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. |
@@ -51,7 +51,12 @@ Use `browser_inspect` action `observe` as the primary semantic page view. It ret
text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may
also become stale after large DOM changes, so observe again before the next interaction. also become stale after large DOM changes, so observe again before the next interaction.
Use `browser_interact` for click, hover, focus, blur, fill, select, and key actions. Focus and blur Use `browser_interact` with `action=scroll-to` to bring an element into view. Its bounds are the
visible border-box portion in top-level viewport CSS pixels after ancestor clipping; partial
visibility is enough. Hidden or fully clipped targets fail. This does not test occlusion by other
elements. Use fresh element refs for iframe/shadow-root targets; selectors search the main document.
Use `browser_interact` for click, hover, scroll-to, focus, blur, fill, select, and key actions. Focus and blur
explicitly enter or leave UI states triggered by focus changes. An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are explicitly enter or leave UI states triggered by focus changes. An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are
labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref. labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref.
Do not click the trigger itself unless the user wants the trigger's action. Do not click the trigger itself unless the user wants the trigger's action.
@@ -167,12 +167,13 @@ const BROWSER_TOOL_SPECS: BrowserToolSpec[] = [
{ {
name: "browser_interact", name: "browser_interact",
description: description:
"Interact with an element in the active Agent Window tab. Actions: click, hover, focus, blur, fill, select, " + "Interact with an element in the active Agent Window tab. Actions: click, hover, scroll-to, focus, blur, fill, select, " +
"press. click/hover/focus/blur/fill/select require target; fill also requires value; select requires " + "press. click/hover/scroll-to/focus/blur/fill/select require target; fill also requires value; select requires " +
"values; press requires key and may optionally focus target first.", "values; press requires key and may optionally focus target first.",
actions: { actions: {
click: "interact.click", click: "interact.click",
hover: "interact.hover", hover: "interact.hover",
"scroll-to": "interact.scroll-to",
focus: "interact.focus", focus: "interact.focus",
blur: "interact.blur", blur: "interact.blur",
fill: "interact.fill", fill: "interact.fill",
@@ -13,7 +13,7 @@ import type { ToolDeps } from "./tools";
const MODIFIERS = ["alt", "ctrl", "meta", "shift"] as const; const MODIFIERS = ["alt", "ctrl", "meta", "shift"] as const;
/** Add hover, focus, blur and select without bypassing session ownership or observation. */ /** Add interaction primitives without bypassing session ownership or observation. */
export function registerPhaseOneInteractionTools( export function registerPhaseOneInteractionTools(
deps: ToolDeps, deps: ToolDeps,
register: ToolRegistrar, register: ToolRegistrar,
@@ -100,6 +100,89 @@ export function registerPhaseOneInteractionTools(
}), }),
); );
register(
defineTool({
name: "interact.scroll-to",
description:
"Scroll an element and its frame owners into view. Returns the visible portion's " +
"bounds in top-level viewport CSS pixels; fails if no visible area remains. " +
"Use a fresh ref for iframe targets; selectors search the main document.",
parameters: {
target: {
type: "string",
required: true,
description: "Element ref (@e3 / e3) or main-document CSS selector to scroll into view.",
},
session: SESSION_PARAM,
tabId: TAB_ID_PARAM,
timeoutMs: TIMEOUT_MS_PARAM,
},
output: {
schema: {
type: "object",
additionalProperties: false,
properties: {
session: { type: "string", required: true },
tabId: { type: "integer", required: true },
x: { type: "number", required: true },
y: { type: "number", required: true },
width: { type: "number", required: true },
height: { type: "number", required: true },
},
},
render: (_args, value) => [
{
type: "text",
text:
`[session ${value.session}] scrolled into view on tab ${value.tabId}: ` +
`(${value.x}, ${value.y}, ${value.width}, ${value.height})`,
},
],
},
async execute(args, exec) {
requireNonEmpty(args.target, "target");
requirePositive(args.timeoutMs, "timeoutMs");
const sessionId = registry.resolve(args.session, "browser_interact(action=scroll-to)");
const cmdArgs = ["scroll-to", "--session", sessionId];
appendTabId(cmdArgs, args.tabId);
if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`);
appendTarget(cmdArgs, args.target);
const reply = (await runtime.run(
exec,
cmdArgs,
"scroll-to",
sessionId,
runnerTimeout(deps, args.timeoutMs),
)) as {
tab_id: number;
x: number;
y: number;
width: number;
height: number;
};
return {
session: sessionId,
tabId: reply.tab_id,
x: reply.x,
y: reply.y,
width: reply.width,
height: reply.height,
};
},
presentCall: (args) => ({
card: "terminal",
title: runtime.commandLine([
"scroll-to",
args.target,
"--session",
args.session ?? "(current)",
]),
description: "Scroll an element into view",
}),
presentResult: runtime.presentTerminalResult,
}),
);
for (const action of ["focus", "blur"] as const) { for (const action of ["focus", "blur"] as const) {
register( register(
defineTool({ defineTool({
@@ -102,6 +102,7 @@ const ACTION_ROUTES: Record<string, readonly [string, string]> = {
"inspect.network": ["browser_inspect", "network"], "inspect.network": ["browser_inspect", "network"],
"interact.click": ["browser_interact", "click"], "interact.click": ["browser_interact", "click"],
"interact.hover": ["browser_interact", "hover"], "interact.hover": ["browser_interact", "hover"],
"interact.scroll-to": ["browser_interact", "scroll-to"],
"interact.focus": ["browser_interact", "focus"], "interact.focus": ["browser_interact", "focus"],
"interact.blur": ["browser_interact", "blur"], "interact.blur": ["browser_interact", "blur"],
"interact.fill": ["browser_interact", "fill"], "interact.fill": ["browser_interact", "fill"],
@@ -202,7 +203,7 @@ const EXPECTED_ACTIONS = {
browser_session: ["start", "stop", "list"], browser_session: ["start", "stop", "list"],
browser_page: ["navigate", "back", "forward", "reload", "wait"], browser_page: ["navigate", "back", "forward", "reload", "wait"],
browser_inspect: ["observe", "snapshot", "html", "screenshot", "console", "network"], browser_inspect: ["observe", "snapshot", "html", "screenshot", "console", "network"],
browser_interact: ["click", "hover", "focus", "blur", "fill", "select", "press"], browser_interact: ["click", "hover", "scroll-to", "focus", "blur", "fill", "select", "press"],
browser_tabs: ["list", "create", "select", "close", "borrow", "return"], browser_tabs: ["list", "create", "select", "close", "borrow", "return"],
browser_assist: ["resize", "emulate", "request-help"], browser_assist: ["resize", "emulate", "request-help"],
} as const; } as const;
@@ -540,6 +541,7 @@ describe("interaction tools", () => {
it.each([ it.each([
"focus", "focus",
"blur", "blur",
"scroll-to",
] as const)("interact.%s validates arguments and session ownership before running", async (action) => { ] as const)("interact.%s validates arguments and session ownership before running", async (action) => {
const { tools, calls } = setup({ "session start": START_REPLY("s1") }); const { tools, calls } = setup({ "session start": START_REPLY("s1") });
await startSession(tools); await startSession(tools);
@@ -555,6 +557,30 @@ describe("interaction tools", () => {
expect(calls).toHaveLength(1); expect(calls).toHaveLength(1);
}); });
it.each(["@e3", "#target"])("interact.scroll-to maps %s, bounds and timeout", async (target) => {
const { tools, calls } = setup({
"session start": START_REPLY("s1"),
"scroll-to": { tab_id: 7, x: 10, y: 20, width: 100, height: 80 },
});
await startSession(tools);
expect(
await tools
.get("interact.scroll-to")
?.execute({ target, tabId: 7, timeoutMs: 150_000 }, makeExec()),
).toEqual({ session: "s1", tabId: 7, x: 10, y: 20, width: 100, height: 80 });
expect(calls[1].args).toEqual([
"scroll-to",
"--session",
"s1",
"--tab-id",
"7",
"--timeout",
"150000ms",
target,
]);
expect(calls[1].options.timeoutMs).toBe(165_000);
});
it("interact.fill passes value and --no-clear", async () => { it("interact.fill passes value and --no-clear", async () => {
const { tools, calls } = setup(responses); const { tools, calls } = setup(responses);
await startSession(tools); await startSession(tools);