mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
feat(dsh-plugin): expose scroll-to action
This commit is contained in:
1 parent
61c97c9cb5
commit
77e3f5e1f3
5 files changed
+121
-6
No files matched your search
@@ -54,7 +54,7 @@ the `bsk` CLI and browser extension separately when a release requires it.
|
|||||||
| `browser_session` | `start`, `stop`, `list` | Manage plugin-owned Agent Window sessions. |
|
| `browser_session` | `start`, `stop`, `list` | Manage plugin-owned Agent Window sessions. |
|
||||||
| `browser_page` | `navigate`, `back`, `forward`, `reload`, `wait` | Navigate the active tab and wait for page lifecycle events. |
|
| `browser_page` | `navigate`, `back`, `forward`, `reload`, `wait` | Navigate the active tab and wait for page lifecycle events. |
|
||||||
| `browser_inspect` | `observe`, `snapshot`, `html`, `screenshot`, `console`, `network` | Read semantic or diagnostic page state and capture screenshots. |
|
| `browser_inspect` | `observe`, `snapshot`, `html`, `screenshot`, `console`, `network` | Read semantic or diagnostic page state and capture screenshots. |
|
||||||
| `browser_interact` | `click`, `hover`, `focus`, `blur`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. |
|
| `browser_interact` | `click`, `hover`, `scroll-to`, `focus`, `blur`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. |
|
||||||
| `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. |
|
| `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. |
|
||||||
| `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. |
|
| `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. |
|
||||||
|
|
||||||
|
|||||||
@@ -51,7 +51,12 @@ Use `browser_inspect` action `observe` as the primary semantic page view. It ret
|
|||||||
text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may
|
text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may
|
||||||
also become stale after large DOM changes, so observe again before the next interaction.
|
also become stale after large DOM changes, so observe again before the next interaction.
|
||||||
|
|
||||||
Use `browser_interact` for click, hover, focus, blur, fill, select, and key actions. Focus and blur
|
Use `browser_interact` with `action=scroll-to` to bring an element into view. Its bounds are the
|
||||||
|
visible border-box portion in top-level viewport CSS pixels after ancestor clipping; partial
|
||||||
|
visibility is enough. Hidden or fully clipped targets fail. This does not test occlusion by other
|
||||||
|
elements. Use fresh element refs for iframe/shadow-root targets; selectors search the main document.
|
||||||
|
|
||||||
|
Use `browser_interact` for click, hover, scroll-to, focus, blur, fill, select, and key actions. Focus and blur
|
||||||
explicitly enter or leave UI states triggered by focus changes. An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are
|
explicitly enter or leave UI states triggered by focus changes. An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are
|
||||||
labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref.
|
labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref.
|
||||||
Do not click the trigger itself unless the user wants the trigger's action.
|
Do not click the trigger itself unless the user wants the trigger's action.
|
||||||
|
|||||||
@@ -167,12 +167,13 @@ const BROWSER_TOOL_SPECS: BrowserToolSpec[] = [
|
|||||||
{
|
{
|
||||||
name: "browser_interact",
|
name: "browser_interact",
|
||||||
description:
|
description:
|
||||||
"Interact with an element in the active Agent Window tab. Actions: click, hover, focus, blur, fill, select, " +
|
"Interact with an element in the active Agent Window tab. Actions: click, hover, scroll-to, focus, blur, fill, select, " +
|
||||||
"press. click/hover/focus/blur/fill/select require target; fill also requires value; select requires " +
|
"press. click/hover/scroll-to/focus/blur/fill/select require target; fill also requires value; select requires " +
|
||||||
"values; press requires key and may optionally focus target first.",
|
"values; press requires key and may optionally focus target first.",
|
||||||
actions: {
|
actions: {
|
||||||
click: "interact.click",
|
click: "interact.click",
|
||||||
hover: "interact.hover",
|
hover: "interact.hover",
|
||||||
|
"scroll-to": "interact.scroll-to",
|
||||||
focus: "interact.focus",
|
focus: "interact.focus",
|
||||||
blur: "interact.blur",
|
blur: "interact.blur",
|
||||||
fill: "interact.fill",
|
fill: "interact.fill",
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import type { ToolDeps } from "./tools";
|
|||||||
|
|
||||||
const MODIFIERS = ["alt", "ctrl", "meta", "shift"] as const;
|
const MODIFIERS = ["alt", "ctrl", "meta", "shift"] as const;
|
||||||
|
|
||||||
/** Add hover, focus, blur and select without bypassing session ownership or observation. */
|
/** Add interaction primitives without bypassing session ownership or observation. */
|
||||||
export function registerPhaseOneInteractionTools(
|
export function registerPhaseOneInteractionTools(
|
||||||
deps: ToolDeps,
|
deps: ToolDeps,
|
||||||
register: ToolRegistrar,
|
register: ToolRegistrar,
|
||||||
@@ -100,6 +100,89 @@ export function registerPhaseOneInteractionTools(
|
|||||||
}),
|
}),
|
||||||
);
|
);
|
||||||
|
|
||||||
|
register(
|
||||||
|
defineTool({
|
||||||
|
name: "interact.scroll-to",
|
||||||
|
description:
|
||||||
|
"Scroll an element and its frame owners into view. Returns the visible portion's " +
|
||||||
|
"bounds in top-level viewport CSS pixels; fails if no visible area remains. " +
|
||||||
|
"Use a fresh ref for iframe targets; selectors search the main document.",
|
||||||
|
parameters: {
|
||||||
|
target: {
|
||||||
|
type: "string",
|
||||||
|
required: true,
|
||||||
|
description: "Element ref (@e3 / e3) or main-document CSS selector to scroll into view.",
|
||||||
|
},
|
||||||
|
session: SESSION_PARAM,
|
||||||
|
tabId: TAB_ID_PARAM,
|
||||||
|
timeoutMs: TIMEOUT_MS_PARAM,
|
||||||
|
},
|
||||||
|
output: {
|
||||||
|
schema: {
|
||||||
|
type: "object",
|
||||||
|
additionalProperties: false,
|
||||||
|
properties: {
|
||||||
|
session: { type: "string", required: true },
|
||||||
|
tabId: { type: "integer", required: true },
|
||||||
|
x: { type: "number", required: true },
|
||||||
|
y: { type: "number", required: true },
|
||||||
|
width: { type: "number", required: true },
|
||||||
|
height: { type: "number", required: true },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
render: (_args, value) => [
|
||||||
|
{
|
||||||
|
type: "text",
|
||||||
|
text:
|
||||||
|
`[session ${value.session}] scrolled into view on tab ${value.tabId}: ` +
|
||||||
|
`(${value.x}, ${value.y}, ${value.width}, ${value.height})`,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
},
|
||||||
|
async execute(args, exec) {
|
||||||
|
requireNonEmpty(args.target, "target");
|
||||||
|
requirePositive(args.timeoutMs, "timeoutMs");
|
||||||
|
const sessionId = registry.resolve(args.session, "browser_interact(action=scroll-to)");
|
||||||
|
const cmdArgs = ["scroll-to", "--session", sessionId];
|
||||||
|
appendTabId(cmdArgs, args.tabId);
|
||||||
|
if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`);
|
||||||
|
appendTarget(cmdArgs, args.target);
|
||||||
|
const reply = (await runtime.run(
|
||||||
|
exec,
|
||||||
|
cmdArgs,
|
||||||
|
"scroll-to",
|
||||||
|
sessionId,
|
||||||
|
runnerTimeout(deps, args.timeoutMs),
|
||||||
|
)) as {
|
||||||
|
tab_id: number;
|
||||||
|
x: number;
|
||||||
|
y: number;
|
||||||
|
width: number;
|
||||||
|
height: number;
|
||||||
|
};
|
||||||
|
return {
|
||||||
|
session: sessionId,
|
||||||
|
tabId: reply.tab_id,
|
||||||
|
x: reply.x,
|
||||||
|
y: reply.y,
|
||||||
|
width: reply.width,
|
||||||
|
height: reply.height,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
presentCall: (args) => ({
|
||||||
|
card: "terminal",
|
||||||
|
title: runtime.commandLine([
|
||||||
|
"scroll-to",
|
||||||
|
args.target,
|
||||||
|
"--session",
|
||||||
|
args.session ?? "(current)",
|
||||||
|
]),
|
||||||
|
description: "Scroll an element into view",
|
||||||
|
}),
|
||||||
|
presentResult: runtime.presentTerminalResult,
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
|
||||||
for (const action of ["focus", "blur"] as const) {
|
for (const action of ["focus", "blur"] as const) {
|
||||||
register(
|
register(
|
||||||
defineTool({
|
defineTool({
|
||||||
|
|||||||
@@ -102,6 +102,7 @@ const ACTION_ROUTES: Record<string, readonly [string, string]> = {
|
|||||||
"inspect.network": ["browser_inspect", "network"],
|
"inspect.network": ["browser_inspect", "network"],
|
||||||
"interact.click": ["browser_interact", "click"],
|
"interact.click": ["browser_interact", "click"],
|
||||||
"interact.hover": ["browser_interact", "hover"],
|
"interact.hover": ["browser_interact", "hover"],
|
||||||
|
"interact.scroll-to": ["browser_interact", "scroll-to"],
|
||||||
"interact.focus": ["browser_interact", "focus"],
|
"interact.focus": ["browser_interact", "focus"],
|
||||||
"interact.blur": ["browser_interact", "blur"],
|
"interact.blur": ["browser_interact", "blur"],
|
||||||
"interact.fill": ["browser_interact", "fill"],
|
"interact.fill": ["browser_interact", "fill"],
|
||||||
@@ -202,7 +203,7 @@ const EXPECTED_ACTIONS = {
|
|||||||
browser_session: ["start", "stop", "list"],
|
browser_session: ["start", "stop", "list"],
|
||||||
browser_page: ["navigate", "back", "forward", "reload", "wait"],
|
browser_page: ["navigate", "back", "forward", "reload", "wait"],
|
||||||
browser_inspect: ["observe", "snapshot", "html", "screenshot", "console", "network"],
|
browser_inspect: ["observe", "snapshot", "html", "screenshot", "console", "network"],
|
||||||
browser_interact: ["click", "hover", "focus", "blur", "fill", "select", "press"],
|
browser_interact: ["click", "hover", "scroll-to", "focus", "blur", "fill", "select", "press"],
|
||||||
browser_tabs: ["list", "create", "select", "close", "borrow", "return"],
|
browser_tabs: ["list", "create", "select", "close", "borrow", "return"],
|
||||||
browser_assist: ["resize", "emulate", "request-help"],
|
browser_assist: ["resize", "emulate", "request-help"],
|
||||||
} as const;
|
} as const;
|
||||||
@@ -540,6 +541,7 @@ describe("interaction tools", () => {
|
|||||||
it.each([
|
it.each([
|
||||||
"focus",
|
"focus",
|
||||||
"blur",
|
"blur",
|
||||||
|
"scroll-to",
|
||||||
] as const)("interact.%s validates arguments and session ownership before running", async (action) => {
|
] as const)("interact.%s validates arguments and session ownership before running", async (action) => {
|
||||||
const { tools, calls } = setup({ "session start": START_REPLY("s1") });
|
const { tools, calls } = setup({ "session start": START_REPLY("s1") });
|
||||||
await startSession(tools);
|
await startSession(tools);
|
||||||
@@ -555,6 +557,30 @@ describe("interaction tools", () => {
|
|||||||
expect(calls).toHaveLength(1);
|
expect(calls).toHaveLength(1);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it.each(["@e3", "#target"])("interact.scroll-to maps %s, bounds and timeout", async (target) => {
|
||||||
|
const { tools, calls } = setup({
|
||||||
|
"session start": START_REPLY("s1"),
|
||||||
|
"scroll-to": { tab_id: 7, x: 10, y: 20, width: 100, height: 80 },
|
||||||
|
});
|
||||||
|
await startSession(tools);
|
||||||
|
expect(
|
||||||
|
await tools
|
||||||
|
.get("interact.scroll-to")
|
||||||
|
?.execute({ target, tabId: 7, timeoutMs: 150_000 }, makeExec()),
|
||||||
|
).toEqual({ session: "s1", tabId: 7, x: 10, y: 20, width: 100, height: 80 });
|
||||||
|
expect(calls[1].args).toEqual([
|
||||||
|
"scroll-to",
|
||||||
|
"--session",
|
||||||
|
"s1",
|
||||||
|
"--tab-id",
|
||||||
|
"7",
|
||||||
|
"--timeout",
|
||||||
|
"150000ms",
|
||||||
|
target,
|
||||||
|
]);
|
||||||
|
expect(calls[1].options.timeoutMs).toBe(165_000);
|
||||||
|
});
|
||||||
|
|
||||||
it("interact.fill passes value and --no-clear", async () => {
|
it("interact.fill passes value and --no-clear", async () => {
|
||||||
const { tools, calls } = setup(responses);
|
const { tools, calls } = setup(responses);
|
||||||
await startSession(tools);
|
await startSession(tools);
|
||||||
|
|||||||
Reference in new issue
Block a user