mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Merge pull request #157 from Tencent/refactor/simplify-dsh-commands
Refactor/simplify dsh commands
This commit is contained in:
Zhang GH
@@ -5,41 +5,20 @@ npm: [`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wx
|
||||
A [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (dsh) tool plugin that exposes
|
||||
[BrowserSkill](https://github.com/Tencent/BrowserSkill) (`bsk`) browser automation to the model.
|
||||
|
||||
Each tool maps to one `bsk <cmd> --json` invocation: the plugin spawns the bsk CLI, parses its
|
||||
Each action maps to one `bsk <cmd> --json` invocation: the plugin spawns the bsk CLI, parses its
|
||||
structured JSON output, and returns a canonical typed value. The bsk daemon, browser, and browser
|
||||
extension keep owning the actual browser control — this package is a thin, well-typed bridge.
|
||||
|
||||
## Tools
|
||||
|
||||
| Tool | bsk command | Purpose |
|
||||
| Tool | Actions | Purpose |
|
||||
| --- | --- | --- |
|
||||
| `browser_session_start` | `bsk session start` | Open an Agent Window session; optional initial URL, window size, and device emulation preset. Returns the session id and makes it the current session. |
|
||||
| `browser_session_stop` | `bsk session stop` | Stop a session (the current one by default) and close its Agent Window. Only plugin-created sessions can be stopped. |
|
||||
| `browser_session_list` | — (registry only) | List the sessions this plugin created, marking the current one. Foreign daemon sessions are never shown. |
|
||||
| `browser_navigate` | `bsk navigate` | Navigate the active tab, with `waitUntil` / timeout control. |
|
||||
| `browser_snapshot` | `bsk snapshot` | Indented aria-tree snapshot with `@eN` refs for interaction tools. |
|
||||
| `browser_observe` | `bsk observe` | Semantic VOM observation (read-only) with `@eN` refs. |
|
||||
| `browser_click` | `bsk click` | Click a snapshot ref or CSS selector (button / click-count options). |
|
||||
| `browser_hover` | `bsk hover` | Hover a ref or selector and allow hover-triggered UI to settle. |
|
||||
| `browser_fill` | `bsk fill` | Fill an input / textarea / contenteditable (clears first by default). |
|
||||
| `browser_select` | `bsk select` | Select one or more option values on a `<select>`. |
|
||||
| `browser_press` | `bsk press` | Dispatch a key or combo, optionally focusing a target first. |
|
||||
| `browser_screenshot` | `bsk screenshot` | PNG capture of the tab or a ref-cropped element; inlines the image when the deployment supports image input, otherwise returns a file path. |
|
||||
| `browser_emulate` | `bsk emulate` | Apply or clear mobile device emulation on the active tab. |
|
||||
| `browser_tab_list` | `bsk tab list` | List user and/or Agent Window tabs visible to an owned session. |
|
||||
| `browser_tab_create` | `bsk tab create` | Create a foreground or background tab in the Agent Window. |
|
||||
| `browser_tab_select` | `bsk tab select` | Focus an Agent Window tab. |
|
||||
| `browser_tab_close` | `bsk tab close` | Close an Agent Window tab. |
|
||||
| `browser_tab_borrow` | `bsk tab borrow` | Move a user tab into the Agent Window for controlled interaction. |
|
||||
| `browser_tab_return` | `bsk tab return` | Return a borrowed tab to its original user window. |
|
||||
| `browser_navigate_back` / `browser_navigate_forward` | `bsk navigate-back` / `bsk navigate-forward` | Traverse tab history and wait for a lifecycle phase. |
|
||||
| `browser_reload` | `bsk reload` | Reload the active tab, optionally bypassing cache. |
|
||||
| `browser_wait_for_navigation` | `bsk wait-for-navigation` | Wait explicitly for a page lifecycle event. |
|
||||
| `browser_request_help` | `bsk request-help` | Pause for login, captcha, OTP, confirmation, or another human browser step. |
|
||||
| `browser_get_html` | `bsk get-html` | Read raw document or ref-scoped HTML after semantic observation is insufficient. |
|
||||
| `browser_console` | `bsk console` | Read buffered console/log/exception entries without evaluating JavaScript. |
|
||||
| `browser_network` | `bsk network` | Read buffered network response/failure metadata. |
|
||||
| `browser_window_resize` | `bsk window resize` | Resize the owned session's Agent Window. |
|
||||
| `browser_session` | `start`, `stop`, `list` | Manage plugin-owned Agent Window sessions. |
|
||||
| `browser_page` | `navigate`, `back`, `forward`, `reload`, `wait` | Navigate the active tab and wait for page lifecycle events. |
|
||||
| `browser_inspect` | `observe`, `snapshot`, `html`, `screenshot`, `console`, `network` | Read semantic or diagnostic page state and capture screenshots. |
|
||||
| `browser_interact` | `click`, `hover`, `fill`, `select`, `press` | Interact with controls using fresh refs or selectors. |
|
||||
| `browser_tabs` | `list`, `create`, `select`, `close`, `borrow`, `return` | Manage Agent Window tabs and temporarily borrow user tabs. |
|
||||
| `browser_assist` | `resize`, `emulate`, `request-help` | Resize or emulate the browser and pause for human-only steps. |
|
||||
|
||||
`bsk evaluate` and `bsk record` are intentionally not exposed by this plugin: arbitrary page
|
||||
evaluation is a higher-risk capability, while recording is long-running and needs a dedicated
|
||||
@@ -63,7 +42,7 @@ seam degrade silently.
|
||||
|
||||
One agent conversation can drive several browser sessions at once:
|
||||
|
||||
- `browser_session_start` returns the session id and makes it the **current session**.
|
||||
- `browser_session` with `action: start` returns the session id and makes it the **current session**.
|
||||
- Every operation tool accepts an optional `session` argument. When omitted, the call acts on the
|
||||
current session (the one most recently started or used); when given, that session becomes current.
|
||||
- Every tool result echoes the session it actually acted on, so the model never has to guess.
|
||||
@@ -72,9 +51,9 @@ One agent conversation can drive several browser sessions at once:
|
||||
|
||||
**Ownership boundary**: the bsk daemon may be shared with other agents, terminals, or dsh
|
||||
instances. The plugin therefore only ever sees and operates on sessions it created itself —
|
||||
an explicit `session` argument naming a foreign or unknown id is rejected, `browser_session_list`
|
||||
shows plugin-created sessions only (no daemon-wide view), and stop/unload cleanup can never touch
|
||||
a session owned by another program.
|
||||
an explicit `session` argument naming a foreign or unknown id is rejected, the `list` action on
|
||||
`browser_session` shows plugin-created sessions only (no daemon-wide view), and stop/unload cleanup
|
||||
can never touch a session owned by another program.
|
||||
|
||||
## Installation
|
||||
|
||||
@@ -109,7 +88,7 @@ All fields are optional and validated through the plugin's Schemastery `Config`:
|
||||
# lazyTools: true # reveal browser_* tools only after the skill is invoked
|
||||
```
|
||||
|
||||
- **`lazyTools` (default `true`)** — the final progressive-disclosure stage: the 28
|
||||
- **`lazyTools` (default `true`)** — the final progressive-disclosure stage: the six
|
||||
`browser_*` tool schemas stay OUT of the system prompt (zero schema tokens) until the
|
||||
`browser-skill` skill is actually invoked — the skill catalog entry is the only
|
||||
advertisement. One successful invocation (model tool call, or a `/browser-skill` user
|
||||
@@ -131,7 +110,7 @@ like the BrowserSkill extension without leaking a single selector into the host
|
||||
shell's theme cannot bleed back in.
|
||||
|
||||
- **Lifecycle**: hidden while the plugin owns no sessions; appears on the first
|
||||
`browser_session_start`; disappears when all sessions stop (or the plugin unloads).
|
||||
`browser_session` start action; disappears when all sessions stop (or the plugin unloads).
|
||||
- **Focus view**: status row (green/idle/red dot + session + action + mm:ss), the latest page
|
||||
frame (refreshes every ~1.5s while active, ~8s when idle; the last good frame stays on
|
||||
stage while the next one loads, and is kept on errors so the card does not flash),
|
||||
@@ -173,11 +152,11 @@ shell's theme cannot bleed back in.
|
||||
card). Screenshots additionally attach the image itself when the host mounts an attachment store
|
||||
and the active model route declares image input; otherwise the PNG path is returned.
|
||||
- **Web UI toolview (browser half)**: the package is dual-face. `dsh.client` (platform `web`) ships
|
||||
`lib/client.cjs`, which registers a keyed `tool.call.toolview` view for `browser_screenshot`. The
|
||||
custom view keeps the terminal block (command + output) and, when the settled result carries an
|
||||
image block, resolves the durable attachment through the client session's authorized
|
||||
`lib/client.cjs`, which registers a keyed `tool.call.toolview` view for `browser_inspect`. The
|
||||
custom view keeps a terminal block for every inspect action and, when a screenshot result carries
|
||||
an image block, resolves the durable attachment through the client session's authorized
|
||||
`readAttachment` RPC and renders it with the shared `MessageImage` thumbnail/lightbox atoms.
|
||||
Every other `browser_*` tool keeps the stock terminal card. The bundle follows the dsh client
|
||||
The other five tools keep the stock terminal card. The bundle follows the dsh client
|
||||
contract: a CJS closure factory handed to `window.__ModuleLoader__.load`, platform modules
|
||||
(`react`, `dsh-client-ui-*`) external, everything else inlined, CSS Modules compiled by
|
||||
lightningcss.
|
||||
|
||||
@@ -18,37 +18,15 @@ function stripFrontmatter(markdown) {
|
||||
}
|
||||
|
||||
const SUPPORTED_BROWSER_TOOLS = [
|
||||
"browser_session_start",
|
||||
"browser_session_stop",
|
||||
"browser_session_list",
|
||||
"browser_navigate",
|
||||
"browser_navigate_back",
|
||||
"browser_navigate_forward",
|
||||
"browser_reload",
|
||||
"browser_wait_for_navigation",
|
||||
"browser_snapshot",
|
||||
"browser_observe",
|
||||
"browser_get_html",
|
||||
"browser_screenshot",
|
||||
"browser_console",
|
||||
"browser_network",
|
||||
"browser_click",
|
||||
"browser_hover",
|
||||
"browser_fill",
|
||||
"browser_select",
|
||||
"browser_press",
|
||||
"browser_tab_list",
|
||||
"browser_tab_create",
|
||||
"browser_tab_select",
|
||||
"browser_tab_close",
|
||||
"browser_tab_borrow",
|
||||
"browser_tab_return",
|
||||
"browser_request_help",
|
||||
"browser_emulate",
|
||||
"browser_window_resize",
|
||||
"browser_session",
|
||||
"browser_page",
|
||||
"browser_inspect",
|
||||
"browser_interact",
|
||||
"browser_tabs",
|
||||
"browser_assist",
|
||||
];
|
||||
|
||||
function validateModelFacingSkill(markdown) {
|
||||
function validateModelFacingSkill(markdown, supportedTools) {
|
||||
if (/\bbsk\b/i.test(markdown)) {
|
||||
throw new Error("DSH skill must not expose the internal CLI name");
|
||||
}
|
||||
@@ -57,9 +35,9 @@ function validateModelFacingSkill(markdown) {
|
||||
}
|
||||
|
||||
const mentioned = new Set(markdown.match(/\bbrowser_[a-z][a-z_]*\b/g) ?? []);
|
||||
const supported = new Set(SUPPORTED_BROWSER_TOOLS);
|
||||
const supported = new Set(supportedTools);
|
||||
const unknown = [...mentioned].filter((name) => !supported.has(name));
|
||||
const missing = SUPPORTED_BROWSER_TOOLS.filter((name) => !mentioned.has(name));
|
||||
const missing = supportedTools.filter((name) => !mentioned.has(name));
|
||||
if (unknown.length > 0) {
|
||||
throw new Error(`DSH skill mentions unsupported browser tools: ${unknown.join(", ")}`);
|
||||
}
|
||||
@@ -75,11 +53,11 @@ const DESCRIPTION =
|
||||
"debug page activity, or smoke-test UI changes in a managed Agent Window.";
|
||||
|
||||
const content = `${stripFrontmatter(source).trim()}\n`;
|
||||
validateModelFacingSkill(content);
|
||||
validateModelFacingSkill(content, SUPPORTED_BROWSER_TOOLS);
|
||||
|
||||
const banner =
|
||||
"// GENERATED by scripts/build-skill-content.mjs — do not edit.\n" +
|
||||
"// Source: skill/SKILL.md (DeepSeek Harness browser_* tool instructions).\n";
|
||||
"// Source: skill/SKILL.md (DeepSeek Harness six-tool instructions).\n";
|
||||
|
||||
writeFileSync(
|
||||
join(pkg, "src", "skill-content.generated.ts"),
|
||||
|
||||
@@ -1,271 +1,146 @@
|
||||
---
|
||||
name: browser-skill
|
||||
description: |
|
||||
Use the injected browser_* tools to automate the user's logged-in Chromium
|
||||
browser in a managed Agent Window.
|
||||
description: Browser automation through six injected domain tools.
|
||||
---
|
||||
|
||||
# browser-skill for DeepSeek Harness
|
||||
|
||||
Drive the user's real Chromium browser through the structured `browser_*` tools provided by this
|
||||
Drive the user's real Chromium browser through the six structured domain tools provided by this
|
||||
plugin. The browser keeps the user's existing logins and cookies while isolating automation in an
|
||||
Agent Window. Use the tool schemas exactly as exposed by DeepSeek Harness; arguments are structured
|
||||
fields, not command-line flags.
|
||||
Agent Window. Every call must include the tool's `action` field; other fields depend on that action.
|
||||
|
||||
## Tool availability
|
||||
|
||||
Loading this skill reveals the browser tools for the rest of the conversation. If the tools were
|
||||
hidden before this skill was loaded, start browser work only after the skill invocation succeeds.
|
||||
Loading this skill reveals the browser tools for the rest of the conversation. All browser work
|
||||
must use the injected tools directly. Do not invoke another process to control the browser. Direct
|
||||
tool calls preserve session ownership, the live observation window, cancellation, attachments, and
|
||||
cleanup.
|
||||
|
||||
All browser operations must use the injected tools directly. Do not translate the examples in this
|
||||
document into terminal commands or invoke another process to control the browser. Direct tool calls
|
||||
preserve session ownership, the live observation window, tool cards, cancellation, and cleanup.
|
||||
The six tools are:
|
||||
|
||||
## When to use
|
||||
|
||||
- Visit and read pages that require the user's existing browser session.
|
||||
- Fill forms, select options, click through bounded flows, or smoke-test a web UI.
|
||||
- Inspect tabs, page structure, screenshots, console messages, or network metadata.
|
||||
- Work with a user tab after explicitly borrowing it into the Agent Window.
|
||||
|
||||
Do not use browser automation for tasks that need only local files, a purpose-built API, or an
|
||||
explanation. Never extract credentials, cookies, tokens, or other secrets. Do not keep control of a
|
||||
personal tab longer than the immediate task requires.
|
||||
- `browser_session`: start, stop, or list plugin-owned sessions.
|
||||
- `browser_page`: navigate, back, forward, reload, or wait for navigation.
|
||||
- `browser_inspect`: observe, snapshot, html, screenshot, console, or network.
|
||||
- `browser_interact`: click, hover, fill, select, or press.
|
||||
- `browser_tabs`: list, create, select, close, borrow, or return tabs.
|
||||
- `browser_assist`: resize the Agent Window, emulate a device, or request help from the user.
|
||||
|
||||
## Mandatory workflow
|
||||
|
||||
Every browser task follows this lifecycle:
|
||||
|
||||
1. Call `browser_session_start` and retain its returned `sessionId`.
|
||||
2. Use that session for every browser operation. Pass `session` explicitly when more than one
|
||||
session exists.
|
||||
3. Call `browser_session_stop` when the bounded goal is reached or an error ends the task. Treat
|
||||
cleanup like a `finally` path unless the user explicitly asks to keep the session open.
|
||||
1. Call `browser_session` with `action: "start"` and retain its returned `sessionId`.
|
||||
2. Use that session for every later call. Pass `session` explicitly when more than one exists.
|
||||
3. Call `browser_session` with `action: "stop"` when the goal is reached or an error ends the task.
|
||||
Treat cleanup like a finally path unless the user explicitly asks to keep the session open.
|
||||
|
||||
Conceptual tool-call sequence:
|
||||
Conceptual sequence:
|
||||
|
||||
```text
|
||||
browser_session_start({ url: "https://example.com" })
|
||||
browser_observe({ session: sessionId })
|
||||
browser_click({ session: sessionId, target: "@e4" })
|
||||
browser_observe({ session: sessionId })
|
||||
browser_session_stop({ session: sessionId })
|
||||
browser_session({ action: "start", url: "https://example.com" })
|
||||
browser_inspect({ action: "observe", session: sessionId })
|
||||
browser_interact({ action: "click", session: sessionId, target: "@e4" })
|
||||
browser_inspect({ action: "observe", session: sessionId })
|
||||
browser_session({ action: "stop", session: sessionId })
|
||||
```
|
||||
|
||||
The plugin owns only sessions created by `browser_session_start`. `browser_session_list` reports
|
||||
only those owned sessions. Never guess or reuse a session id from another program. When `session`
|
||||
is omitted, most tools use the most recently started or used owned session; explicit ids are safer
|
||||
for multi-session work.
|
||||
The plugin owns only sessions it starts. Never guess or reuse a session id from another program.
|
||||
When `session` is omitted, most actions use the current owned session, but explicit ids are safer.
|
||||
|
||||
## Stop when the goal is met
|
||||
|
||||
Browser work is a bounded task, not open-ended browsing.
|
||||
Browser work is bounded. Define an observable success condition, take the shortest path, and stop
|
||||
as soon as success is visible. Do not refresh or click after success. If an action fails twice
|
||||
without progress, pause instead of brute-forcing. If state is ambiguous, make at most one fresh
|
||||
observation and then ask the user.
|
||||
|
||||
1. Define one observable success condition from the user's request.
|
||||
2. Take the shortest path: observe, act, then perform at most one relevant check.
|
||||
3. Stop immediately when success is visible. Do not refresh, navigate, click, or perform extra
|
||||
verification after success.
|
||||
4. If a human-only step appears, use `browser_request_help`. If an action fails twice without
|
||||
progress, pause instead of brute-forcing.
|
||||
5. When uncertain, make at most one fresh observation. If the state is still ambiguous, ask the
|
||||
user rather than continuing to click.
|
||||
## Observe and interact
|
||||
|
||||
## Core interaction loop
|
||||
Use `browser_inspect` with `action: "observe"` as the primary semantic page view. It returns roles,
|
||||
states, text, and `@eN` refs. Use `action: "snapshot"` when a static accessibility tree is more useful.
|
||||
Prefer a fresh ref over a raw CSS selector for interactions.
|
||||
|
||||
Use `browser_navigate` to open a destination in the active Agent Window tab, then inspect the page
|
||||
before interacting:
|
||||
Refs invalidate after navigation and may become stale after large DOM changes. Observe again after
|
||||
navigation or a meaningful DOM update before using another ref. If an observation identifies a
|
||||
hover-triggered surface, hover the trigger, observe again, then interact with the revealed control.
|
||||
|
||||
1. `browser_observe` is the primary semantic view. It returns roles, states, text, and `@eN` refs,
|
||||
including hints about conditional hover or focus surfaces.
|
||||
2. `browser_snapshot` is the static accessibility-tree fallback when the semantic view is
|
||||
insufficient or a stricter tree is useful.
|
||||
3. Prefer a fresh `@eN` ref over a raw CSS selector for `browser_click`, `browser_hover`,
|
||||
`browser_fill`, `browser_select`, and targeted `browser_press` calls.
|
||||
4. Observe again after navigation or a meaningful DOM change before using another ref.
|
||||
Use `browser_interact` as follows:
|
||||
|
||||
Refs invalidate after navigation and may also become stale after large DOM changes. Never carry a
|
||||
ref to a new page. If an observation marks a control as hover-triggered, call `browser_hover` on the
|
||||
trigger, observe again, and then act on the newly visible ref.
|
||||
- `action: "click"`: requires `target`; optionally accepts `button` and `clickCount`.
|
||||
- `action: "hover"`: requires `target`; optionally accepts `modifiers`, `settleMs`, and `timeoutMs`.
|
||||
- `action: "fill"`: requires `target` and `value`; `noClear` appends instead of replacing.
|
||||
- `action: "select"`: requires `target` and a non-empty `values` array. Values are option value
|
||||
attributes, not necessarily the visible labels.
|
||||
- `action: "press"`: requires `key`; optionally focus `target` first and set `holdMs`.
|
||||
|
||||
Use interaction tools according to their schemas:
|
||||
## Navigate and wait
|
||||
|
||||
- `browser_click`: click a ref or selector; optionally choose mouse button and click count.
|
||||
- `browser_hover`: move to a ref or selector and allow hover UI to settle; then observe again.
|
||||
- `browser_fill`: clear and fill a text-capable element; set `noClear` only when appending is
|
||||
intentional.
|
||||
- `browser_select`: pass `values` as an array, including for a single option.
|
||||
- `browser_press`: send a key or combo, optionally focusing `target` first.
|
||||
Use `browser_page` with these actions:
|
||||
|
||||
Actions may trigger a navigation after the action itself returns. Use
|
||||
`browser_wait_for_navigation` only when that lifecycle wait is actually needed, then observe the
|
||||
new page.
|
||||
- `navigate` requires `url` and optionally accepts `waitUntil` and `timeoutMs`.
|
||||
- `back` and `forward` move one history entry and optionally accept lifecycle wait fields.
|
||||
- `reload` refreshes the current tab; set `hard: true` only when bypassing cache is necessary.
|
||||
- `wait` waits for an expected navigation after an interaction that did not already wait.
|
||||
|
||||
## Observation and page reading priority
|
||||
Avoid speculative waits when no navigation is expected. After any page change, discard old refs and
|
||||
observe again.
|
||||
|
||||
## Reading priority
|
||||
|
||||
Escalate page reading only as needed:
|
||||
|
||||
1. `browser_observe` for normal semantic understanding and interaction refs.
|
||||
2. `browser_snapshot` for a stricter static accessibility tree.
|
||||
3. `browser_get_html` for hidden DOM, metadata, or exact markup that the semantic views cannot
|
||||
provide. A `ref` must come from a fresh observation; keep `maxBytes` bounded.
|
||||
4. `browser_screenshot` for layout, styling, canvas, image content, or another genuinely visual
|
||||
question. Pass a fresh `ref` to crop to one element, or omit it for the visible tab.
|
||||
1. `browser_inspect` with `action: "observe"` for normal understanding and refs.
|
||||
2. The same tool with `action: "snapshot"` for a stricter static tree.
|
||||
3. `action: "html"` for exact markup that semantic views cannot provide; keep `maxBytes` bounded and
|
||||
use only a fresh ref when scoping a subtree.
|
||||
4. `action: "screenshot"` for layout, styling, canvas, or genuinely visual evidence. A fresh `ref`
|
||||
crops to an element; omit it for the visible tab.
|
||||
|
||||
Do not start with raw HTML or a screenshot merely to discover ordinary controls. Screenshots may
|
||||
return an inline image or a saved path depending on the deployment.
|
||||
Do not start with raw HTML or a screenshot merely to discover ordinary controls.
|
||||
|
||||
## Tabs and the Agent Window boundary
|
||||
|
||||
Write actions normally affect only Agent Window tabs.
|
||||
Use `browser_tabs` with these actions:
|
||||
|
||||
- `browser_tab_list` lists `agent`, `user`, or `all` tabs visible to the owned session. Use
|
||||
`scope: "user"` to locate a user tab before borrowing it.
|
||||
- `browser_tab_create` creates an Agent Window tab. It is active by default; set `active: false`
|
||||
for a background tab.
|
||||
- `browser_tab_select` focuses an Agent Window tab.
|
||||
- `browser_tab_close` closes an Agent Window tab.
|
||||
- `browser_tab_borrow` moves a chosen user tab into the Agent Window before any write action.
|
||||
- `browser_tab_return` returns a borrowed tab to its original window and position. Return it as
|
||||
soon as the immediate task is finished; stopping the session is only the fallback cleanup.
|
||||
- `list` returns visible tabs; `scope: "user"` finds a user tab before borrowing.
|
||||
- `create` makes an Agent Window tab and optionally accepts `url`, `active`, and `index`.
|
||||
- `select` focuses an Agent Window tab and requires a returned `tabId`.
|
||||
- `close` closes an Agent Window tab and requires a returned `tabId`.
|
||||
- `borrow` moves a user tab into the Agent Window and requires a listed user `tabId`.
|
||||
- `return` restores a borrowed tab and requires that borrowed `tabId`.
|
||||
|
||||
Use only a `tabId` returned by `browser_tab_list` or `browser_tab_create`. Do not assume a tab id
|
||||
from page position or title.
|
||||
Return borrowed tabs as soon as the immediate task finishes. Stopping the session is only fallback
|
||||
cleanup. Never invent a tab id.
|
||||
|
||||
## Navigation and lifecycle
|
||||
## Human help and display controls
|
||||
|
||||
- `browser_navigate` opens a URL and can wait for a requested lifecycle phase.
|
||||
- `browser_navigate_back` and `browser_navigate_forward` move one history entry.
|
||||
- `browser_reload` reloads the current tab; use `hard: true` only when bypassing cache is necessary.
|
||||
- `browser_wait_for_navigation` waits for a lifecycle event after an action that did not already
|
||||
wait. Avoid speculative waits when no navigation is expected.
|
||||
Use `browser_assist` with `action: "request-help"` for login, captcha, OTP, payment confirmation,
|
||||
consent, or another human-only step. `prompt` is required. Optional `targets` highlight fresh refs or
|
||||
selectors. `completionCriteria` can detect explicit stable success through URL, selector, or text
|
||||
conditions. After the user continues or criteria complete, observe again before reasoning about the
|
||||
new state.
|
||||
|
||||
After navigation, history traversal, or reload, discard old refs and call `browser_observe` or
|
||||
`browser_snapshot` again.
|
||||
Use `action: "resize"` with `width` and `height` to resize the outer Agent Window. Use
|
||||
`action: "emulate"` with a device preset or explicit width and height; `mobile: true` also requires
|
||||
dimensions. Use `off: true` alone to clear emulation. Emulation is per tab.
|
||||
|
||||
## Ask the user for browser help
|
||||
## Read-only diagnostics
|
||||
|
||||
Call `browser_request_help` for login, captcha, OTP, payment confirmation, consent, or another step
|
||||
that requires the user. Give a clear `prompt`; when the prompt refers to a concrete control, pass
|
||||
its fresh ref or selector in `targets` so the overlay can highlight it. Use `title` only when it
|
||||
adds clarity.
|
||||
Use `browser_inspect` with `action: "console"` for buffered logs and exceptions, and
|
||||
`action: "network"` for response and failure metadata. Both accept `tabId`, `since`, `limit`, and
|
||||
`maxTextChars`; continue with the returned sequence cursor rather than rereading the same entries.
|
||||
Console additionally accepts `includeStack`.
|
||||
|
||||
Optional `completionCriteria` can finish the pause automatically when the page has an explicit,
|
||||
stable success signal. It supports `any` or `all` conditions using `urlContains`, `urlMatches`,
|
||||
`selectorExists`, `selectorMissing`, `textExists`, or `textMissing`, plus `stableForMs`.
|
||||
|
||||
```text
|
||||
browser_request_help({
|
||||
session: sessionId,
|
||||
title: "Sign in required",
|
||||
prompt: "Complete sign-in, then return control after the account page appears.",
|
||||
targets: ["@e7"],
|
||||
timeoutMs: 300000,
|
||||
completionCriteria: {
|
||||
any: [{ urlContains: "/account" }, { selectorExists: "[data-testid='account-menu']" }],
|
||||
stableForMs: 1000
|
||||
}
|
||||
})
|
||||
```
|
||||
|
||||
Interpret outcomes deliberately:
|
||||
|
||||
- `continued` or `completed`: the user returned control or the explicit detector matched. Observe
|
||||
again before reasoning about the new state or using refs.
|
||||
- `cancelled`: stop the requested flow.
|
||||
- `timed_out`: report the timeout; do not repeatedly reopen the prompt.
|
||||
- `disabled`: unattended mode does not allow help; proceed only if safe or stop gracefully.
|
||||
- `navigated`: legacy outcome; do not treat navigation alone as proof of success.
|
||||
|
||||
## Debugging without page scripting
|
||||
|
||||
Use the read-only debugging tools only when relevant to the user's task:
|
||||
|
||||
- `browser_console` reads buffered console messages, browser logs, and JavaScript exceptions.
|
||||
Set `includeStack: true` when stack frames are useful.
|
||||
- `browser_network` reads buffered response and failure metadata. It does not return request or
|
||||
response headers or bodies and does not modify traffic.
|
||||
|
||||
Both tools accept `tabId`, `since`, `limit`, and `maxTextChars`. Their result returns `nextSince`;
|
||||
pass that value as the next `since` cursor instead of repeatedly requesting the same entries.
|
||||
|
||||
Arbitrary page script evaluation and interaction recording are intentionally unsupported in this
|
||||
DeepSeek Harness plugin. Do not invent tool names or route around this limitation. If the task
|
||||
strictly requires either capability, explain that it is not available in this phase.
|
||||
|
||||
## Window and device controls
|
||||
|
||||
- `browser_window_resize` changes the Agent Window's outer `width` and `height`; each dimension
|
||||
must be from 100 through 7680 CSS pixels.
|
||||
- `browser_emulate` applies a built-in device preset or explicit viewport settings to the active
|
||||
tab. Explicit `width` and `height` must be supplied together. `mobile: true` also requires both
|
||||
dimensions. Use `off: true` alone to clear emulation. Emulation is per-tab and new tabs do not
|
||||
inherit it.
|
||||
|
||||
## Tool reference
|
||||
|
||||
### Sessions
|
||||
|
||||
| Tool | Purpose |
|
||||
| --- | --- |
|
||||
| `browser_session_start` | Start an owned Agent Window session, optionally with URL, size, browser instance, focus behavior, or device preset. |
|
||||
| `browser_session_stop` | Stop an owned session and close its Agent Window. |
|
||||
| `browser_session_list` | List owned sessions and identify the current one. |
|
||||
|
||||
### Navigation and reading
|
||||
|
||||
| Tool | Purpose |
|
||||
| --- | --- |
|
||||
| `browser_navigate` | Navigate the active tab to a URL. |
|
||||
| `browser_navigate_back` | Go back one history entry. |
|
||||
| `browser_navigate_forward` | Go forward one history entry. |
|
||||
| `browser_reload` | Reload the active tab, optionally bypassing cache. |
|
||||
| `browser_wait_for_navigation` | Wait for an expected page lifecycle event. |
|
||||
| `browser_observe` | Read the primary semantic page representation with refs. |
|
||||
| `browser_snapshot` | Read the static accessibility-tree fallback with refs. |
|
||||
| `browser_get_html` | Read raw document or ref-scoped HTML as a last resort. |
|
||||
| `browser_screenshot` | Capture the visible tab or a ref-cropped element. |
|
||||
|
||||
### Interaction
|
||||
|
||||
| Tool | Purpose |
|
||||
| --- | --- |
|
||||
| `browser_click` | Click a ref or CSS selector. |
|
||||
| `browser_hover` | Hover a ref or selector to reveal conditional UI. |
|
||||
| `browser_fill` | Fill a text-capable element. |
|
||||
| `browser_select` | Select one or more option values. |
|
||||
| `browser_press` | Send a keyboard key or combo. |
|
||||
|
||||
### Tabs
|
||||
|
||||
| Tool | Purpose |
|
||||
| --- | --- |
|
||||
| `browser_tab_list` | List visible user and Agent Window tabs. |
|
||||
| `browser_tab_create` | Create an Agent Window tab. |
|
||||
| `browser_tab_select` | Focus an Agent Window tab. |
|
||||
| `browser_tab_close` | Close an Agent Window tab. |
|
||||
| `browser_tab_borrow` | Move a user tab into the Agent Window. |
|
||||
| `browser_tab_return` | Return a borrowed tab to its original window. |
|
||||
|
||||
### Assistance, diagnostics, and display
|
||||
|
||||
| Tool | Purpose |
|
||||
| --- | --- |
|
||||
| `browser_request_help` | Pause for an explicit user browser step. |
|
||||
| `browser_console` | Read buffered console and exception entries. |
|
||||
| `browser_network` | Read buffered response and failure metadata. |
|
||||
| `browser_window_resize` | Resize the Agent Window. |
|
||||
| `browser_emulate` | Apply or clear per-tab device emulation. |
|
||||
Arbitrary page script evaluation and interaction recording are intentionally unsupported. Do not
|
||||
invent tool names or route around that limitation.
|
||||
|
||||
## Error handling and final checklist
|
||||
|
||||
On a stale ref, observe again and retry once with a fresh ref. On a closed or unknown tab, list
|
||||
tabs instead of guessing. On an unknown session, list owned sessions or start a new one; never try
|
||||
foreign ids. On timeout, decide whether one longer, purposeful wait is justified. Avoid repeated
|
||||
actions with unchanged state.
|
||||
On a stale ref, observe again and retry once. On an unknown tab, list tabs instead of guessing. On
|
||||
an unknown session, list owned sessions or start one; never try foreign ids. On timeout, decide
|
||||
whether one longer purposeful wait is justified, and never immediately repeat a failed history
|
||||
action without first checking page state.
|
||||
|
||||
Before finishing a browser task, verify that:
|
||||
|
||||
- The user's observable goal is met, or the blocker is clearly reported.
|
||||
- Any borrowed tab has been returned when practical.
|
||||
- The owned session has been stopped unless the user explicitly asked to keep it open.
|
||||
- No further page action is taken after success.
|
||||
Before finishing, verify that the observable goal is met or the blocker is clear, borrowed tabs are
|
||||
returned when practical, the owned session is stopped, and no extra page action occurs after
|
||||
success.
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
* are hidden from every surface, so their Agent Windows would otherwise
|
||||
* linger with no way back to them.
|
||||
*
|
||||
* Ownership lineage: browser_session_start records the calling agent's
|
||||
* Ownership lineage: browser_session action=start records the calling agent's
|
||||
* session id PLUS every ancestor along `header.parentSession` (a subagent's
|
||||
* browsers are reaped when any ancestor is archived, the root conversation
|
||||
* included). Lineage is resolved through the host session store; an
|
||||
|
||||
@@ -0,0 +1,334 @@
|
||||
/**
|
||||
* The six model-facing BrowserSkill tools. Each action dispatches to a private
|
||||
* operation handler, so the model receives only six schemas while every action
|
||||
* retains the existing ownership, cancellation, queuing, observation,
|
||||
* screenshot, and presentation behavior.
|
||||
*/
|
||||
|
||||
import { defineTool, type ParameterSchemaSpec, type ToolDefinition } from "@deepseek-ai/dsh-tools";
|
||||
import { SESSION_PARAM, TAB_ID_PARAM, TIMEOUT_MS_PARAM, WAIT_UNTIL_PARAM } from "./tool-params";
|
||||
import { createBrowserOperationDefinitions, type ToolDeps } from "./tools";
|
||||
|
||||
const DEVICE_PRESETS = [
|
||||
"iphone-14",
|
||||
"iphone-14-pro-max",
|
||||
"iphone-se",
|
||||
"pixel-7",
|
||||
"galaxy-s23",
|
||||
"ipad-mini",
|
||||
"galaxy-tab-s8",
|
||||
] as const;
|
||||
|
||||
const TARGET_PARAM = {
|
||||
type: "string",
|
||||
description: "Snapshot ref such as @e3, or a CSS selector.",
|
||||
} as const;
|
||||
|
||||
const DOMAIN_RESULT = { type: "json" } as const;
|
||||
|
||||
interface BrowserToolSpec {
|
||||
name: string;
|
||||
description: string;
|
||||
actions: Record<string, string>;
|
||||
parameters: ParameterSchemaSpec;
|
||||
}
|
||||
|
||||
function operationArgs(args: unknown): Record<string, unknown> {
|
||||
const { action: _action, ...rest } = args as Record<string, unknown>;
|
||||
return rest;
|
||||
}
|
||||
|
||||
function definitionFor(
|
||||
definitions: ReadonlyMap<string, ToolDefinition>,
|
||||
actions: Record<string, string>,
|
||||
action: unknown,
|
||||
): ToolDefinition {
|
||||
const operationName = typeof action === "string" ? actions[action] : undefined;
|
||||
const definition = operationName === undefined ? undefined : definitions.get(operationName);
|
||||
if (definition === undefined) throw new Error(`unsupported browser action: ${String(action)}`);
|
||||
return definition;
|
||||
}
|
||||
|
||||
function defineBrowserTool(
|
||||
spec: BrowserToolSpec,
|
||||
definitions: ReadonlyMap<string, ToolDefinition>,
|
||||
): ToolDefinition {
|
||||
const actionNames = Object.keys(spec.actions);
|
||||
return defineTool({
|
||||
name: spec.name,
|
||||
description: spec.description,
|
||||
parameters: {
|
||||
action: {
|
||||
type: "string",
|
||||
required: true,
|
||||
enum: actionNames,
|
||||
description:
|
||||
"Operation to perform. Other arguments are action-dependent and are validated by that operation.",
|
||||
},
|
||||
...spec.parameters,
|
||||
},
|
||||
output: {
|
||||
schema: DOMAIN_RESULT,
|
||||
render(args, value) {
|
||||
const definition = definitionFor(definitions, spec.actions, args.action);
|
||||
return definition.output.render(operationArgs(args), value);
|
||||
},
|
||||
},
|
||||
isConcurrencySafe(args) {
|
||||
const definition = definitionFor(definitions, spec.actions, args.action);
|
||||
return definition.isConcurrencySafe?.(operationArgs(args)) === true;
|
||||
},
|
||||
async execute(args, exec) {
|
||||
const definition = definitionFor(definitions, spec.actions, args.action);
|
||||
return (await definition.execute(operationArgs(args), exec)) as never;
|
||||
},
|
||||
presentCall(args) {
|
||||
const definition = definitionFor(definitions, spec.actions, args.action);
|
||||
return definition.presentCall?.(operationArgs(args));
|
||||
},
|
||||
presentResult(args, result) {
|
||||
const definition = definitionFor(definitions, spec.actions, args.action);
|
||||
return definition.presentResult?.(operationArgs(args), result);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
const BROWSER_TOOL_SPECS: BrowserToolSpec[] = [
|
||||
{
|
||||
name: "browser_session",
|
||||
description:
|
||||
"Manage plugin-owned browser sessions. Actions: start opens an Agent Window; stop closes an " +
|
||||
"owned session; list returns owned sessions. For start, url/device/width/height/noFocus/browser " +
|
||||
"are optional. For stop, session is optional and defaults to the current owned session.",
|
||||
actions: {
|
||||
start: "session.start",
|
||||
stop: "session.stop",
|
||||
list: "session.list",
|
||||
},
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
url: { type: "string", description: "Initial URL for start." },
|
||||
width: { type: "integer", description: "Agent Window width; start requires height too." },
|
||||
height: { type: "integer", description: "Agent Window height; start requires width too." },
|
||||
noFocus: { type: "boolean", description: "Start the Agent Window in the background." },
|
||||
browser: { type: "string", description: "Browser instance id for start." },
|
||||
device: { type: "string", enum: DEVICE_PRESETS, description: "Device preset for start." },
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "browser_page",
|
||||
description:
|
||||
"Navigate and wait on the active Agent Window tab. Actions: navigate, back, forward, reload, " +
|
||||
"wait. navigate requires url; reload optionally accepts hard; navigation actions accept " +
|
||||
"waitUntil/timeoutMs. Observe again after a meaningful page change before reusing refs.",
|
||||
actions: {
|
||||
navigate: "page.navigate",
|
||||
back: "page.back",
|
||||
forward: "page.forward",
|
||||
reload: "page.reload",
|
||||
wait: "page.wait",
|
||||
},
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
tabId: TAB_ID_PARAM,
|
||||
url: { type: "string", description: "Destination URL; required for navigate." },
|
||||
waitUntil: WAIT_UNTIL_PARAM,
|
||||
timeoutMs: TIMEOUT_MS_PARAM,
|
||||
hard: { type: "boolean", description: "Bypass cache for reload." },
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "browser_inspect",
|
||||
description:
|
||||
"Read page state without arbitrary script execution. Actions: observe, snapshot, html, " +
|
||||
"screenshot, console, network. Prefer observe, then snapshot, then bounded html; use screenshot " +
|
||||
"for visual evidence. console/network support cursor fields since/limit/maxTextChars.",
|
||||
actions: {
|
||||
observe: "inspect.observe",
|
||||
snapshot: "inspect.snapshot",
|
||||
html: "inspect.html",
|
||||
screenshot: "inspect.screenshot",
|
||||
console: "inspect.console",
|
||||
network: "inspect.network",
|
||||
},
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
tabId: TAB_ID_PARAM,
|
||||
maxDepth: { type: "integer", description: "Tree depth cap for observe/snapshot." },
|
||||
maxTokens: { type: "integer", description: "Token cap for observe/snapshot." },
|
||||
ref: { type: "string", description: "Fresh ref for scoped html or cropped screenshot." },
|
||||
maxBytes: { type: "integer", description: "HTML byte cap." },
|
||||
since: { type: "integer", description: "Console/network sequence cursor." },
|
||||
limit: { type: "integer", description: "Console/network entry cap." },
|
||||
maxTextChars: { type: "integer", description: "Console/network per-entry text cap." },
|
||||
includeStack: { type: "boolean", description: "Include console stack frames." },
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "browser_interact",
|
||||
description:
|
||||
"Interact with an element in the active Agent Window tab. Actions: click, hover, fill, select, " +
|
||||
"press. click/hover/fill/select require target; fill also requires value; select requires " +
|
||||
"values; press requires key and may optionally focus target first.",
|
||||
actions: {
|
||||
click: "interact.click",
|
||||
hover: "interact.hover",
|
||||
fill: "interact.fill",
|
||||
select: "interact.select",
|
||||
press: "interact.press",
|
||||
},
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
tabId: TAB_ID_PARAM,
|
||||
target: TARGET_PARAM,
|
||||
button: { type: "string", enum: ["left", "middle", "right"], description: "Click button." },
|
||||
clickCount: { type: "integer", description: "Click count." },
|
||||
value: { type: "string", description: "Text for fill." },
|
||||
noClear: { type: "boolean", description: "Append instead of clearing for fill." },
|
||||
modifiers: {
|
||||
type: "array",
|
||||
items: { type: "string", enum: ["alt", "ctrl", "meta", "shift"] },
|
||||
description: "Modifiers held during hover.",
|
||||
},
|
||||
settleMs: { type: "integer", description: "Hover settle delay." },
|
||||
timeoutMs: TIMEOUT_MS_PARAM,
|
||||
values: {
|
||||
type: "array",
|
||||
items: { type: "string" },
|
||||
description: "Option values for select; at least one is required.",
|
||||
},
|
||||
key: { type: "string", description: "Key or combo for press." },
|
||||
holdMs: { type: "integer", description: "Key hold duration for press." },
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "browser_tabs",
|
||||
description:
|
||||
"Manage tabs visible to an owned session. Actions: list, create, select, close, borrow, return. " +
|
||||
"select/close/borrow/return require tabId from list/create. Borrow moves a user tab into the " +
|
||||
"Agent Window; return it as soon as the task finishes.",
|
||||
actions: {
|
||||
list: "tabs.list",
|
||||
create: "tabs.create",
|
||||
select: "tabs.select",
|
||||
close: "tabs.close",
|
||||
borrow: "tabs.borrow",
|
||||
return: "tabs.return",
|
||||
},
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
tabId: TAB_ID_PARAM,
|
||||
scope: {
|
||||
type: "string",
|
||||
enum: ["user", "agent", "all"],
|
||||
description: "Tab scope for list.",
|
||||
},
|
||||
url: { type: "string", description: "Initial URL for create." },
|
||||
active: { type: "boolean", description: "Focus the created tab (default true)." },
|
||||
index: { type: "integer", description: "Insertion index for create." },
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "browser_assist",
|
||||
description:
|
||||
"Display and human-assistance operations. Actions: resize, emulate, request-help. resize " +
|
||||
"requires width/height; emulate accepts device or width/height/mobile, or off alone; " +
|
||||
"request-help requires prompt and can wait for explicit completion criteria.",
|
||||
actions: {
|
||||
resize: "assist.resize",
|
||||
emulate: "assist.emulate",
|
||||
"request-help": "assist.request-help",
|
||||
},
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
tabId: TAB_ID_PARAM,
|
||||
width: { type: "integer", description: "Window/viewport width." },
|
||||
height: { type: "integer", description: "Window/viewport height." },
|
||||
device: { type: "string", enum: DEVICE_PRESETS, description: "Emulation device preset." },
|
||||
mobile: { type: "boolean", description: "Enable a mobile viewport with width/height." },
|
||||
off: { type: "boolean", description: "Clear emulation; use alone." },
|
||||
prompt: { type: "string", description: "Instructions shown to the user for request-help." },
|
||||
title: { type: "string", description: "Optional request-help title." },
|
||||
targets: {
|
||||
type: "array",
|
||||
items: { type: "string" },
|
||||
description: "Refs/selectors highlighted for request-help.",
|
||||
},
|
||||
timeoutMs: TIMEOUT_MS_PARAM,
|
||||
completionCriteria: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
description: "Automatic completion detector for request-help.",
|
||||
properties: {
|
||||
any: {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
urlContains: { type: "string" },
|
||||
urlMatches: { type: "string" },
|
||||
selectorExists: { type: "string" },
|
||||
selectorMissing: { type: "string" },
|
||||
textExists: { type: "string" },
|
||||
textMissing: { type: "string" },
|
||||
},
|
||||
},
|
||||
},
|
||||
all: {
|
||||
type: "array",
|
||||
items: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
urlContains: { type: "string" },
|
||||
urlMatches: { type: "string" },
|
||||
selectorExists: { type: "string" },
|
||||
selectorMissing: { type: "string" },
|
||||
textExists: { type: "string" },
|
||||
textMissing: { type: "string" },
|
||||
},
|
||||
},
|
||||
},
|
||||
stableForMs: { type: "integer" },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
function indexOperations(definitions: ToolDefinition[]): ReadonlyMap<string, ToolDefinition> {
|
||||
const indexed = new Map<string, ToolDefinition>();
|
||||
for (const definition of definitions) {
|
||||
if (indexed.has(definition.name)) {
|
||||
throw new Error(`duplicate browser operation definition: ${definition.name}`);
|
||||
}
|
||||
indexed.set(definition.name, definition);
|
||||
}
|
||||
|
||||
const routed = new Set(BROWSER_TOOL_SPECS.flatMap((spec) => Object.values(spec.actions)));
|
||||
const missing = [...routed].filter((name) => !indexed.has(name));
|
||||
const unreachable = [...indexed.keys()].filter((name) => !routed.has(name));
|
||||
if (missing.length > 0 || unreachable.length > 0) {
|
||||
throw new Error(
|
||||
[
|
||||
missing.length > 0 ? `missing operations: ${missing.join(", ")}` : undefined,
|
||||
unreachable.length > 0 ? `unreachable operations: ${unreachable.join(", ")}` : undefined,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("; "),
|
||||
);
|
||||
}
|
||||
return indexed;
|
||||
}
|
||||
|
||||
/** Register the complete six-tool browser suite; returns its combined disposer. */
|
||||
export function registerBrowserTools(deps: ToolDeps): () => void {
|
||||
const definitions = indexOperations(createBrowserOperationDefinitions(deps));
|
||||
const disposers = BROWSER_TOOL_SPECS.map((spec) =>
|
||||
deps.ctx.tools.register(defineBrowserTool(spec, definitions)),
|
||||
).filter((dispose): dispose is () => void => typeof dispose === "function");
|
||||
return () => {
|
||||
for (const dispose of disposers.splice(0)) dispose();
|
||||
};
|
||||
}
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
/* browser_screenshot toolview: disclosure row over a terminal block plus the
|
||||
/* browser_inspect screenshot toolview: disclosure row over a terminal block plus the
|
||||
screenshot image. Semantic tokens only (--dsw-alias-*), no literal colors. */
|
||||
|
||||
.card {
|
||||
+80
-23
@@ -1,8 +1,6 @@
|
||||
// browser_screenshot keyed toolview: the terminal card's command line and
|
||||
// output stay exactly as the host renders them, and a result carrying an
|
||||
// image block additionally renders the screenshot itself (thumbnail with the
|
||||
// shared click-to-open lightbox). Pure function of the frozen call/result
|
||||
// slice — live and replay paths render identically.
|
||||
// browser_inspect keyed toolview: all inspect actions retain a terminal card;
|
||||
// screenshot results additionally render the image attachment. Pure function
|
||||
// of the frozen call/result slice, so live and replay paths stay identical.
|
||||
|
||||
import type { ImageAttachmentRef } from "@deepseek-ai/dsh-attachment";
|
||||
import {
|
||||
@@ -19,13 +17,13 @@ import {
|
||||
} from "@deepseek-ai/dsh-client-ui-primitives";
|
||||
import type { ToolCallViewProps } from "@deepseek-ai/dsh-client-ui-tool/client";
|
||||
import { useState } from "react";
|
||||
import css from "./ScreenshotToolView.module.css";
|
||||
import css from "./BrowserInspectToolView.module.css";
|
||||
|
||||
/** Resolved loader bound to the owning session at registration time. */
|
||||
export type ScreenshotImageLoader = (attachment: ImageAttachmentRef) => Promise<string>;
|
||||
export type BrowserInspectImageLoader = (attachment: ImageAttachmentRef) => Promise<string>;
|
||||
|
||||
export type ScreenshotToolViewProps = ToolCallViewProps & {
|
||||
loadImage: ScreenshotImageLoader;
|
||||
export type BrowserInspectToolViewProps = ToolCallViewProps & {
|
||||
loadImage: BrowserInspectImageLoader;
|
||||
};
|
||||
|
||||
type ViewState = "running" | "ok" | "error";
|
||||
@@ -36,6 +34,7 @@ interface ViewModel {
|
||||
readonly output: string | null;
|
||||
readonly image: ImageAttachmentRef | null;
|
||||
readonly summary: string;
|
||||
readonly title: string;
|
||||
}
|
||||
|
||||
const TERMINAL_LABELS: Partial<TerminalBlockLabels> = {
|
||||
@@ -66,18 +65,71 @@ function firstLine(text: string): string {
|
||||
}
|
||||
|
||||
/** Rebuild the command line from the logged arguments (mirrors the host presenter). */
|
||||
function commandOf(argsRaw: string, callId: string): string {
|
||||
const INSPECT_COMMANDS: Record<string, string> = {
|
||||
observe: "observe",
|
||||
snapshot: "snapshot",
|
||||
html: "get-html",
|
||||
screenshot: "screenshot",
|
||||
console: "console",
|
||||
network: "network",
|
||||
};
|
||||
|
||||
function titleOf(action: string): string {
|
||||
return action === "html" ? "HTML" : `${action.slice(0, 1).toUpperCase()}${action.slice(1)}`;
|
||||
}
|
||||
|
||||
interface InspectArgs {
|
||||
action?: unknown;
|
||||
session?: unknown;
|
||||
tabId?: unknown;
|
||||
maxDepth?: unknown;
|
||||
maxTokens?: unknown;
|
||||
ref?: unknown;
|
||||
maxBytes?: unknown;
|
||||
since?: unknown;
|
||||
limit?: unknown;
|
||||
maxTextChars?: unknown;
|
||||
includeStack?: unknown;
|
||||
}
|
||||
|
||||
function appendNumber(parts: string[], flag: string, value: unknown): void {
|
||||
if (typeof value === "number" && Number.isFinite(value)) parts.push(flag, String(value));
|
||||
}
|
||||
|
||||
function commandOf(argsRaw: string, callId: string): { command: string; title: string } {
|
||||
try {
|
||||
const args = JSON.parse(argsRaw) as { session?: unknown; ref?: unknown };
|
||||
const parts = ["bsk", "screenshot", "--session"];
|
||||
const args = JSON.parse(argsRaw) as InspectArgs;
|
||||
const action = typeof args.action === "string" ? args.action : "inspect";
|
||||
const command = INSPECT_COMMANDS[action] ?? action;
|
||||
const parts = ["bsk", command, "--session"];
|
||||
parts.push(
|
||||
typeof args.session === "string" && args.session !== "" ? args.session : "(current)",
|
||||
);
|
||||
if (typeof args.ref === "string" && args.ref !== "") parts.push("--ref", args.ref);
|
||||
return parts.join(" ");
|
||||
if (action === "observe" || action === "snapshot") {
|
||||
appendNumber(parts, "--max-depth", args.maxDepth);
|
||||
appendNumber(parts, "--max-tokens", args.maxTokens);
|
||||
}
|
||||
if (action === "html" || action === "console" || action === "network") {
|
||||
appendNumber(parts, "--tab-id", args.tabId);
|
||||
}
|
||||
if ((action === "html" || action === "screenshot") && typeof args.ref === "string") {
|
||||
if (args.ref !== "") parts.push("--ref", args.ref);
|
||||
}
|
||||
if (action === "html") appendNumber(parts, "--max-bytes", args.maxBytes);
|
||||
if (action === "console" || action === "network") {
|
||||
appendNumber(parts, "--since", args.since);
|
||||
appendNumber(parts, "--limit", args.limit);
|
||||
appendNumber(parts, "--max-text-chars", args.maxTextChars);
|
||||
}
|
||||
if (action === "console" && args.includeStack === true) parts.push("--include-stack");
|
||||
return { command: parts.join(" "), title: titleOf(action) };
|
||||
} catch {
|
||||
// Streaming can expose a truncated JSON prefix; it is still the best label.
|
||||
return argsRaw === "" ? `bsk screenshot (${callId})` : `bsk screenshot ${firstLine(argsRaw)}`;
|
||||
return {
|
||||
command:
|
||||
argsRaw === "" ? `browser_inspect (${callId})` : `browser_inspect ${firstLine(argsRaw)}`,
|
||||
title: "Inspect",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -85,9 +137,9 @@ function commandOf(argsRaw: string, callId: string): string {
|
||||
export function viewModelOf(block: ToolCallViewProps["block"]): ViewModel {
|
||||
const settled = "kind" in block;
|
||||
const argsRaw = (settled ? block.call?.argsRaw : block.argsRaw) ?? "";
|
||||
const command = commandOf(argsRaw, block.callId);
|
||||
const { command, title } = commandOf(argsRaw, block.callId);
|
||||
if (!settled) {
|
||||
return { state: "running", command, output: null, image: null, summary: command };
|
||||
return { state: "running", command, output: null, image: null, summary: command, title };
|
||||
}
|
||||
const textBlock = block.content.find((item) => item.type === "text");
|
||||
const imageBlock = block.content.find((item) => item.type === "image");
|
||||
@@ -96,7 +148,7 @@ export function viewModelOf(block: ToolCallViewProps["block"]): ViewModel {
|
||||
imageBlock !== undefined && imageBlock.type === "image" ? imageBlock.attachment : null;
|
||||
const state: ViewState = block.isError ? "error" : "ok";
|
||||
const summary = output !== null ? firstLine(output) : command;
|
||||
return { state, command, output, image, summary };
|
||||
return { state, command, output, image, summary, title };
|
||||
}
|
||||
|
||||
function dotState(state: ViewState): StateDotState {
|
||||
@@ -111,19 +163,24 @@ function dotState(state: ViewState): StateDotState {
|
||||
}
|
||||
|
||||
/**
|
||||
* Render one browser_screenshot call: a disclosure row over a terminal block
|
||||
* (command + output) plus the screenshot image when the result carries one.
|
||||
* Render one browser_inspect call: a disclosure row over a terminal block,
|
||||
* plus the screenshot image when that action returns an image attachment.
|
||||
*/
|
||||
export function ScreenshotToolView({ block, cwd, inspect, loadImage }: ScreenshotToolViewProps) {
|
||||
export function BrowserInspectToolView({
|
||||
block,
|
||||
cwd,
|
||||
inspect,
|
||||
loadImage,
|
||||
}: BrowserInspectToolViewProps) {
|
||||
const model = viewModelOf(block);
|
||||
const [expanded, setExpanded] = useState(false);
|
||||
const expandable = model.state === "running" || model.output !== null || model.image !== null;
|
||||
const open = expanded && expandable;
|
||||
return (
|
||||
<div className={css.card} data-tool="browser_screenshot" data-state={model.state}>
|
||||
<div className={css.card} data-tool="browser_inspect" data-state={model.state}>
|
||||
<DisclosureRow
|
||||
icon={<StateDot state={dotState(model.state)} />}
|
||||
title="Screenshot"
|
||||
title={model.title}
|
||||
open={open}
|
||||
expandable={expandable}
|
||||
onToggle={() => setExpanded((value) => !value)}
|
||||
@@ -1,6 +1,6 @@
|
||||
/**
|
||||
* dsh-plugin-browserskill browser half: the `browser_screenshot` keyed
|
||||
* toolview plus the observation overlay (live thumbnails + interrupt). The
|
||||
* dsh-plugin-browserskill browser half: the `browser_inspect` keyed toolview
|
||||
* plus the observation overlay (live thumbnails + interrupt). The
|
||||
* default carrier is a floating card on the `shell.overlay` seat; when the
|
||||
* dsh-better-sidebar plugin provides its `betterSidebar` service, the view
|
||||
* moves into a sidebar tab instead (see observation-sidebar.tsx).
|
||||
@@ -16,10 +16,10 @@ import { createElement } from "react";
|
||||
// <style> tags; selectors stay unhashed so `cn(..., "bsk-obs")` roots match).
|
||||
import "./bsk-tokens.nomodule.css";
|
||||
import "./bsk-ui.nomodule.css";
|
||||
import { BrowserInspectToolView } from "./BrowserInspectToolView";
|
||||
import { ObservationOverlay } from "./ObservationOverlay";
|
||||
import { type BetterSidebarLike, registerObservationSidebar } from "./observation-sidebar";
|
||||
import { type EventSourceLike, ObservationClientStore } from "./observation-store";
|
||||
import { ScreenshotToolView } from "./ScreenshotToolView";
|
||||
|
||||
/** Required services: slots, session-scoped attachment reads, and the overlay seat. */
|
||||
export const inject = ["slots", "sessions"];
|
||||
@@ -65,9 +65,9 @@ export function apply(ctx: ClientContext): void {
|
||||
const sessions = ctx.get("sessions") as unknown as ISessions;
|
||||
ctx.slots.inject("tool.call.toolview", () =>
|
||||
ctx.slots.register(
|
||||
{ name: "tool.call.toolview", key: "browser_screenshot" },
|
||||
{ name: "tool.call.toolview", key: "browser_inspect" },
|
||||
(props: ToolCallViewProps) =>
|
||||
createElement(ScreenshotToolView, {
|
||||
createElement(BrowserInspectToolView, {
|
||||
...props,
|
||||
loadImage: (attachment: ImageAttachmentRef) =>
|
||||
loadSessionImage(sessions, props.sessionId, attachment),
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
import type { Context } from "@deepseek-ai/cordis";
|
||||
import Schema from "@deepseek-ai/schemastery";
|
||||
import { armArchiveCleanup } from "./archive-cleanup";
|
||||
import { registerBrowserTools } from "./browser-tools";
|
||||
import { armLazyTools } from "./lazy-tools";
|
||||
import { ObservationService } from "./observation";
|
||||
import { registerObservationRoutes } from "./observation-http";
|
||||
@@ -20,7 +21,7 @@ import { KeyedExecutor } from "./queue";
|
||||
import { type BskRunner, createBskRunner } from "./runner";
|
||||
import { SessionRegistry } from "./sessions";
|
||||
import { armAgentScopedBskSkill, registerBskSkill } from "./skill";
|
||||
import { type PluginConfig, registerTools } from "./tools";
|
||||
import type { PluginConfig } from "./tools";
|
||||
|
||||
export const name = "@wxg-prc-cpg/browser-skill-dsh-plugin";
|
||||
export const inject = ["tools"];
|
||||
@@ -100,11 +101,9 @@ export function apply(
|
||||
// exact agent context at startup so DSH always loads the browser_* protocol
|
||||
// instructions; the shared CLI skill remains untouched for other agents.
|
||||
const disarmAgentSkill = armAgentScopedBskSkill(ctx);
|
||||
const removeSuite = resolved.lazyTools
|
||||
? armLazyTools(ctx, () =>
|
||||
registerTools({ ctx, runner, registry, config: resolved, observation, queue }),
|
||||
)
|
||||
: registerTools({ ctx, runner, registry, config: resolved, observation, queue });
|
||||
const registerSuite = () =>
|
||||
registerBrowserTools({ ctx, runner, registry, config: resolved, observation, queue });
|
||||
const removeSuite = resolved.lazyTools ? armLazyTools(ctx, registerSuite) : registerSuite();
|
||||
// Route registration rides ctx.inject: the webServer service may be provided
|
||||
// AFTER this plugin loads, and in headless compositions it never appears (the
|
||||
// callback simply never runs, leaving the rest of the plugin unaffected).
|
||||
@@ -133,7 +132,7 @@ export function apply(
|
||||
);
|
||||
|
||||
// Unload cleanup: kill in-flight children, then stop every session this
|
||||
// plugin OWNS (created via browser_session_start). Referenced or unknown
|
||||
// plugin OWNS (created via browser_session action=start). Referenced or unknown
|
||||
// sessions belonging to other programs on the shared daemon are never
|
||||
// touched; per-stop failures (already stopped externally, daemon restart)
|
||||
// are swallowed so one stale handle cannot abort the rest.
|
||||
@@ -154,6 +153,7 @@ export function apply(
|
||||
}
|
||||
|
||||
export { armArchiveCleanup, ownerSessionIds } from "./archive-cleanup";
|
||||
export { registerBrowserTools } from "./browser-tools";
|
||||
export type { ObservationEvent, ObservationOptions, SessionObservation } from "./observation";
|
||||
export { ObservationService } from "./observation";
|
||||
export { registerObservationRoutes } from "./observation-http";
|
||||
@@ -162,4 +162,3 @@ export type { BskRunner, BskRunResult, SpawnImpl } from "./runner";
|
||||
export { BskError, createBskRunner } from "./runner";
|
||||
export { SessionRegistry } from "./sessions";
|
||||
export type { PluginConfig, ToolDeps } from "./tools";
|
||||
export { registerTools } from "./tools";
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
/**
|
||||
* Lazy tool-schema injection ("progressive disclosure", final stage): with
|
||||
* `lazyTools` on (the default), the 28 browser_* tool schemas stay OUT of the
|
||||
* `lazyTools` on (the default), the six browser tool schemas stay OUT of the
|
||||
* system prompt until the `browser-skill` skill has actually been invoked —
|
||||
* the skill catalog entry is the only advertisement. One successful
|
||||
* invocation (model tool call, or a user's `/browser-skill` gesture) reveals
|
||||
@@ -96,7 +96,7 @@ export function hasSuccessfulSkillInvocation(events: readonly SessionEventLike[]
|
||||
/**
|
||||
* Arm the lazy reveal. Returns a disposer tearing down listeners and — when
|
||||
* the reveal already happened — the tool suite itself.
|
||||
* @param registerSuite - registers the 28 browser_* tools, returns their disposer.
|
||||
* @param registerSuite - registers the six browser tools and returns their disposer.
|
||||
*/
|
||||
export function armLazyTools(ctx: Context, registerSuite: () => () => void): () => void {
|
||||
let suiteDisposer: (() => void) | undefined;
|
||||
|
||||
@@ -161,7 +161,7 @@ export class ObservationService {
|
||||
this.emit({ type: "upsert", session: { ...entry } });
|
||||
}
|
||||
|
||||
/** Register a fresh owned session (called from browser_session_start). */
|
||||
/** Register a fresh owned session (called from browser_session action=start). */
|
||||
addSession(sessionId: string, url?: string): void {
|
||||
if (!this.deps.options.enabled || this.disposed) return;
|
||||
const dshSessionIds = this.deps.registry.dshOwnersOf(sessionId);
|
||||
@@ -176,7 +176,7 @@ export class ObservationService {
|
||||
this.scheduleCapture(sessionId, 0);
|
||||
}
|
||||
|
||||
/** Drop a session (called from browser_session_stop). */
|
||||
/** Drop a session (called from browser_session action=stop). */
|
||||
removeSession(sessionId: string): void {
|
||||
this.cancelCapture(sessionId);
|
||||
this.foregroundDepth.delete(sessionId);
|
||||
@@ -276,7 +276,7 @@ export class ObservationService {
|
||||
|
||||
/**
|
||||
* Stop one owned session and close its Agent Window (the overlay's stop
|
||||
* button — same end state as `browser_session_stop`). Never waits behind a
|
||||
* button — same end state as `browser_session` action=stop). Never waits behind a
|
||||
* hung in-flight command: tool children are killed first so the session's
|
||||
* keyed queue drains immediately, and no further captures queue up. A
|
||||
* session the daemon already forgot stops idempotently — the goal state
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import type { ToolDefinition, ToolResult, ToolRunContext } from "@deepseek-ai/dsh-tools";
|
||||
import type { ToolDeps } from "./tools";
|
||||
|
||||
export type ToolRegistrar = (definition: ToolDefinition) => () => void;
|
||||
export type ToolRegistrar = (definition: ToolDefinition) => void;
|
||||
|
||||
type TerminalPresentation = { card: "terminal"; output: string; exitCode: number } | undefined;
|
||||
|
||||
|
||||
@@ -23,10 +23,10 @@ export function registerPhaseOneInteractionTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_hover",
|
||||
name: "interact.hover",
|
||||
description:
|
||||
"Move the mouse over a snapshot ref or CSS selector to reveal hover-triggered UI. " +
|
||||
"Run browser_observe or browser_snapshot again afterwards to discover newly visible refs.",
|
||||
"Run browser_inspect action=observe or snapshot afterwards to discover newly visible refs.",
|
||||
parameters: {
|
||||
target: {
|
||||
type: "string",
|
||||
@@ -68,7 +68,7 @@ export function registerPhaseOneInteractionTools(
|
||||
requireNonEmpty(args.target, "target");
|
||||
requirePositive(args.timeoutMs, "timeoutMs");
|
||||
requirePositive(args.settleMs, "settleMs");
|
||||
const sessionId = registry.resolve(args.session, "browser_hover");
|
||||
const sessionId = registry.resolve(args.session, "browser_interact(action=hover)");
|
||||
const cmdArgs = ["hover", "--session", sessionId];
|
||||
appendTabId(cmdArgs, args.tabId);
|
||||
if (args.modifiers !== undefined && args.modifiers.length > 0) {
|
||||
@@ -102,7 +102,7 @@ export function registerPhaseOneInteractionTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_select",
|
||||
name: "interact.select",
|
||||
description:
|
||||
"Set a select element's option values. Pass one value for a normal select or multiple " +
|
||||
"values for a multi-select; values replace the current selection.",
|
||||
@@ -155,7 +155,7 @@ export function registerPhaseOneInteractionTools(
|
||||
requireNonEmpty(args.target, "target");
|
||||
requirePositive(args.timeoutMs, "timeoutMs");
|
||||
if (args.values.length === 0) throw new Error("values must contain at least one option");
|
||||
const sessionId = registry.resolve(args.session, "browser_select");
|
||||
const sessionId = registry.resolve(args.session, "browser_interact(action=select)");
|
||||
const cmdArgs = ["select", "--session", sessionId];
|
||||
appendTabId(cmdArgs, args.tabId);
|
||||
for (const value of args.values) cmdArgs.push("--value", value);
|
||||
|
||||
@@ -51,7 +51,7 @@ export function registerPhaseOneNavigationTools(
|
||||
const { registry } = deps;
|
||||
|
||||
const registerHistory = (direction: "back" | "forward") => {
|
||||
const toolName = `browser_navigate_${direction}` as const;
|
||||
const toolName = `page.${direction}` as const;
|
||||
const command = `navigate-${direction}`;
|
||||
register(
|
||||
defineTool({
|
||||
@@ -106,7 +106,7 @@ export function registerPhaseOneNavigationTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_reload",
|
||||
name: "page.reload",
|
||||
description:
|
||||
"Reload the active Agent Window tab and wait for a lifecycle phase. Set hard=true to " +
|
||||
"bypass the HTTP cache.",
|
||||
@@ -132,7 +132,7 @@ export function registerPhaseOneNavigationTools(
|
||||
},
|
||||
async execute(args, exec) {
|
||||
requirePositive(args.timeoutMs, "timeoutMs");
|
||||
const sessionId = registry.resolve(args.session, "browser_reload");
|
||||
const sessionId = registry.resolve(args.session, "browser_page(action=reload)");
|
||||
const cmdArgs = ["reload", "--session", sessionId];
|
||||
appendTabId(cmdArgs, args.tabId);
|
||||
appendWaitOptions(cmdArgs, args.waitUntil, args.timeoutMs);
|
||||
@@ -163,7 +163,7 @@ export function registerPhaseOneNavigationTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_wait_for_navigation",
|
||||
name: "page.wait",
|
||||
description:
|
||||
"Wait for a page lifecycle event on the active Agent Window tab. Use after an action " +
|
||||
"that may navigate when the action result itself does not wait for navigation.",
|
||||
@@ -195,7 +195,7 @@ export function registerPhaseOneNavigationTools(
|
||||
},
|
||||
async execute(args, exec) {
|
||||
requirePositive(args.timeoutMs, "timeoutMs");
|
||||
const sessionId = registry.resolve(args.session, "browser_wait_for_navigation");
|
||||
const sessionId = registry.resolve(args.session, "browser_page(action=wait)");
|
||||
const timeoutMs = args.timeoutMs ?? 30_000;
|
||||
const cmdArgs = ["wait-for-navigation", "--session", sessionId];
|
||||
appendTabId(cmdArgs, args.tabId);
|
||||
|
||||
@@ -143,7 +143,7 @@ export function registerPhaseOneSupportTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_request_help",
|
||||
name: "assist.request-help",
|
||||
description:
|
||||
"Pause browser automation and ask the user to complete an in-page step such as login, " +
|
||||
"captcha, OTP, or confirmation. After continued/completed, observe again before using refs.",
|
||||
@@ -233,7 +233,7 @@ export function registerPhaseOneSupportTools(
|
||||
if (criteria?.stableForMs !== undefined && criteria.stableForMs < 0) {
|
||||
throw new Error("completionCriteria.stableForMs must be zero or greater");
|
||||
}
|
||||
const sessionId = registry.resolve(args.session, "browser_request_help");
|
||||
const sessionId = registry.resolve(args.session, "browser_assist(action=request-help)");
|
||||
const timeoutMs = args.timeoutMs ?? 300_000;
|
||||
const cmdArgs = [
|
||||
"request-help",
|
||||
@@ -285,10 +285,10 @@ export function registerPhaseOneSupportTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_get_html",
|
||||
name: "inspect.html",
|
||||
description:
|
||||
"Read raw DOM HTML from the active Agent Window tab, optionally scoped to a fresh " +
|
||||
"snapshot ref. Use only when browser_observe/browser_snapshot cannot answer the question.",
|
||||
"snapshot ref. Use only when browser_inspect observe/snapshot cannot answer the question.",
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
tabId: TAB_ID_PARAM,
|
||||
@@ -326,7 +326,7 @@ export function registerPhaseOneSupportTools(
|
||||
if (!isSnapshotRef(args.ref)) throw new Error("ref must be a snapshot ref such as @e3");
|
||||
}
|
||||
requirePositive(args.maxBytes, "maxBytes");
|
||||
const sessionId = registry.resolve(args.session, "browser_get_html");
|
||||
const sessionId = registry.resolve(args.session, "browser_inspect(action=html)");
|
||||
const cmdArgs = ["get-html", "--session", sessionId];
|
||||
appendTabId(cmdArgs, args.tabId);
|
||||
if (args.ref !== undefined) cmdArgs.push("--ref", args.ref);
|
||||
@@ -361,7 +361,7 @@ export function registerPhaseOneSupportTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_console",
|
||||
name: "inspect.console",
|
||||
description:
|
||||
"Read buffered console messages, browser log entries, and JavaScript exceptions from a tab. " +
|
||||
"This is read-only and does not evaluate JavaScript.",
|
||||
@@ -448,7 +448,7 @@ export function registerPhaseOneSupportTools(
|
||||
isConcurrencySafe: () => true,
|
||||
async execute(args, exec) {
|
||||
validateDebugArgs(args);
|
||||
const sessionId = registry.resolve(args.session, "browser_console");
|
||||
const sessionId = registry.resolve(args.session, "browser_inspect(action=console)");
|
||||
const cmdArgs = ["console", "--session", sessionId];
|
||||
appendDebugOptions(cmdArgs, args);
|
||||
if (args.includeStack === true) cmdArgs.push("--include-stack");
|
||||
@@ -493,7 +493,7 @@ export function registerPhaseOneSupportTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_network",
|
||||
name: "inspect.network",
|
||||
description:
|
||||
"Read buffered network responses and failures from a tab. Returns metadata only; request " +
|
||||
"and response headers and bodies are not captured.",
|
||||
@@ -552,7 +552,7 @@ export function registerPhaseOneSupportTools(
|
||||
isConcurrencySafe: () => true,
|
||||
async execute(args, exec) {
|
||||
validateDebugArgs(args);
|
||||
const sessionId = registry.resolve(args.session, "browser_network");
|
||||
const sessionId = registry.resolve(args.session, "browser_inspect(action=network)");
|
||||
const cmdArgs = ["network", "--session", sessionId];
|
||||
appendDebugOptions(cmdArgs, args);
|
||||
const reply = (await runtime.run(exec, cmdArgs, "network", sessionId)) as {
|
||||
@@ -592,7 +592,7 @@ export function registerPhaseOneSupportTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_window_resize",
|
||||
name: "assist.resize",
|
||||
description: "Resize the owned session's Agent Window using outer CSS-pixel dimensions.",
|
||||
parameters: {
|
||||
width: {
|
||||
@@ -629,7 +629,7 @@ export function registerPhaseOneSupportTools(
|
||||
if (args.width < 100 || args.width > 7680 || args.height < 100 || args.height > 7680) {
|
||||
throw new Error("width and height must each be in the range 100..=7680");
|
||||
}
|
||||
const sessionId = registry.resolve(args.session, "browser_window_resize");
|
||||
const sessionId = registry.resolve(args.session, "browser_assist(action=resize)");
|
||||
const reply = (await runtime.run(
|
||||
exec,
|
||||
[
|
||||
|
||||
@@ -7,7 +7,7 @@ import type { ToolDeps } from "./tools";
|
||||
const TAB_ID_REQUIRED = {
|
||||
type: "integer",
|
||||
required: true,
|
||||
description: "Chrome tab id returned by browser_tab_list or browser_tab_create.",
|
||||
description: "Chrome tab id returned by browser_tabs list or create.",
|
||||
} as const;
|
||||
|
||||
interface RawTabInfo {
|
||||
@@ -29,7 +29,7 @@ export function registerPhaseOneTabTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_tab_list",
|
||||
name: "tabs.list",
|
||||
description:
|
||||
"List tabs visible to an owned browser session. Other sessions' Agent Windows remain " +
|
||||
"hidden; use scope=user to find a user tab before borrowing it.",
|
||||
@@ -83,7 +83,7 @@ export function registerPhaseOneTabTools(
|
||||
},
|
||||
isConcurrencySafe: () => true,
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, "browser_tab_list");
|
||||
const sessionId = registry.resolve(args.session, "browser_tabs(action=list)");
|
||||
const cmdArgs = ["tab", "list", "--session", sessionId];
|
||||
if (args.scope !== undefined) cmdArgs.push("--scope", args.scope);
|
||||
const reply = (await runtime.run(exec, cmdArgs, "tab list", sessionId)) as {
|
||||
@@ -118,7 +118,7 @@ export function registerPhaseOneTabTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_tab_create",
|
||||
name: "tabs.create",
|
||||
description:
|
||||
"Create a tab inside the owned session's Agent Window. The tab is focused by default; " +
|
||||
"set active=false to open it in the background.",
|
||||
@@ -148,7 +148,7 @@ export function registerPhaseOneTabTools(
|
||||
},
|
||||
async execute(args, exec) {
|
||||
if (args.url !== undefined) requireNonEmpty(args.url, "url");
|
||||
const sessionId = registry.resolve(args.session, "browser_tab_create");
|
||||
const sessionId = registry.resolve(args.session, "browser_tabs(action=create)");
|
||||
const cmdArgs = ["tab", "create", "--session", sessionId];
|
||||
if (args.url !== undefined) cmdArgs.push("--url", args.url);
|
||||
if (args.active === false) cmdArgs.push("--no-active");
|
||||
@@ -185,7 +185,7 @@ export function registerPhaseOneTabTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_tab_close",
|
||||
name: "tabs.close",
|
||||
description: "Close a tab in the owned session's Agent Window.",
|
||||
parameters: { tabId: TAB_ID_REQUIRED, session: SESSION_PARAM },
|
||||
output: {
|
||||
@@ -202,7 +202,7 @@ export function registerPhaseOneTabTools(
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, "browser_tab_close");
|
||||
const sessionId = registry.resolve(args.session, "browser_tabs(action=close)");
|
||||
const reply = (await runtime.run(
|
||||
exec,
|
||||
["tab", "close", String(args.tabId), "--session", sessionId],
|
||||
@@ -228,7 +228,7 @@ export function registerPhaseOneTabTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_tab_select",
|
||||
name: "tabs.select",
|
||||
description: "Focus a tab in the owned session's Agent Window.",
|
||||
parameters: { tabId: TAB_ID_REQUIRED, session: SESSION_PARAM },
|
||||
output: {
|
||||
@@ -249,7 +249,7 @@ export function registerPhaseOneTabTools(
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, "browser_tab_select");
|
||||
const sessionId = registry.resolve(args.session, "browser_tabs(action=select)");
|
||||
const reply = (await runtime.run(
|
||||
exec,
|
||||
["tab", "select", String(args.tabId), "--session", sessionId],
|
||||
@@ -275,10 +275,10 @@ export function registerPhaseOneTabTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_tab_borrow",
|
||||
name: "tabs.borrow",
|
||||
description:
|
||||
"Move a user-window tab into the owned session's Agent Window for controlled interaction. " +
|
||||
"Return it with browser_tab_return when finished; stopping the session also auto-returns it.",
|
||||
"Return it with browser_tabs action=return; stopping the session also auto-returns it.",
|
||||
parameters: { tabId: TAB_ID_REQUIRED, session: SESSION_PARAM },
|
||||
output: {
|
||||
schema: {
|
||||
@@ -302,7 +302,7 @@ export function registerPhaseOneTabTools(
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, "browser_tab_borrow");
|
||||
const sessionId = registry.resolve(args.session, "browser_tabs(action=borrow)");
|
||||
const reply = (await runtime.run(
|
||||
exec,
|
||||
["tab", "borrow", String(args.tabId), "--session", sessionId],
|
||||
@@ -339,7 +339,7 @@ export function registerPhaseOneTabTools(
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_tab_return",
|
||||
name: "tabs.return",
|
||||
description: "Return a borrowed tab to its original user window and tab-strip position.",
|
||||
parameters: { tabId: TAB_ID_REQUIRED, session: SESSION_PARAM },
|
||||
output: {
|
||||
@@ -365,7 +365,7 @@ export function registerPhaseOneTabTools(
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, "browser_tab_return");
|
||||
const sessionId = registry.resolve(args.session, "browser_tabs(action=return)");
|
||||
const reply = (await runtime.run(
|
||||
exec,
|
||||
["tab", "return", String(args.tabId), "--session", sessionId],
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
*
|
||||
* Strict ownership boundary (the bsk daemon may be shared with other agents,
|
||||
* terminals, or dsh instances): the registry ONLY ever holds sessions created
|
||||
* by this plugin's browser_session_start. Tools cannot see or act on any
|
||||
* by this plugin's browser_session start action. Tools cannot see or act on any
|
||||
* other session — an explicit `session` argument naming a foreign id is an
|
||||
* error, the list tool shows owned sessions only, and stop/unload cleanup
|
||||
* can never touch a session this plugin did not create.
|
||||
@@ -48,7 +48,7 @@ export class SessionRegistry {
|
||||
if (this.sessions.size + this.pendingStarts >= this.maxSessions) {
|
||||
throw new Error(
|
||||
`session limit reached (${this.maxSessions} concurrent sessions); ` +
|
||||
"stop one with browser_session_stop before starting another",
|
||||
"stop one with browser_session action=stop before starting another",
|
||||
);
|
||||
}
|
||||
this.pendingStarts += 1;
|
||||
@@ -69,7 +69,7 @@ export class SessionRegistry {
|
||||
if (!this.sessions.has(session.sessionId) && this.sessions.size >= this.maxSessions) {
|
||||
throw new Error(
|
||||
`session limit reached (${this.maxSessions} concurrent sessions); ` +
|
||||
"stop one with browser_session_stop before starting another",
|
||||
"stop one with browser_session action=stop before starting another",
|
||||
);
|
||||
}
|
||||
this.sessions.set(session.sessionId, { ...session, owned: true });
|
||||
@@ -149,7 +149,7 @@ export class SessionRegistry {
|
||||
private foreignError(sessionId: string, toolName: string): Error {
|
||||
return new Error(
|
||||
`${toolName}: session "${sessionId}" does not belong to this plugin — only sessions ` +
|
||||
"created by browser_session_start are visible and operable here",
|
||||
"created by browser_session action=start are visible and operable here",
|
||||
);
|
||||
}
|
||||
|
||||
@@ -168,7 +168,7 @@ export class SessionRegistry {
|
||||
const current = this.current();
|
||||
if (current === undefined) {
|
||||
throw new Error(
|
||||
`${toolName} needs a session but none is active — start one with browser_session_start`,
|
||||
`${toolName} needs a session but none is active — use browser_session action=start`,
|
||||
);
|
||||
}
|
||||
this.touch(current);
|
||||
@@ -184,10 +184,11 @@ export class SessionRegistry {
|
||||
explicit !== undefined && explicit.trim().length > 0 ? explicit : this.current();
|
||||
if (candidate === undefined) {
|
||||
throw new Error(
|
||||
"browser_session_stop needs a session but none is active — start one with browser_session_start",
|
||||
"browser_session action=stop needs a session but none is active — use action=start first",
|
||||
);
|
||||
}
|
||||
if (!this.isOwned(candidate)) throw this.foreignError(candidate, "browser_session_stop");
|
||||
if (!this.isOwned(candidate))
|
||||
throw this.foreignError(candidate, "browser_session(action=stop)");
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
export const SESSION_PARAM = {
|
||||
type: "string",
|
||||
description:
|
||||
"bsk session id to act on; must be one created by this plugin's browser_session_start. " +
|
||||
"bsk session id to act on; must be one created by browser_session with action=start. " +
|
||||
"Omit to use the current session (the one most recently started or used).",
|
||||
} as const;
|
||||
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
/**
|
||||
* Model-facing tool definitions. Each tool maps to one `bsk <cmd> --json`
|
||||
* invocation: spawn via the runner, parse the JSON envelope, and return a
|
||||
* canonical value whose `output.render` produces the model-facing text.
|
||||
* UI cards reuse the terminal style: the pending card is the command line,
|
||||
* the completed card carries the output.
|
||||
* Internal browser operation definitions shared by the six model-facing tools.
|
||||
* These definitions are never registered with `ctx.tools`: they provide the
|
||||
* action-level validation, execution, rendering, and UI presentation reused by
|
||||
* the domain tools without exposing one schema per operation to the model.
|
||||
*/
|
||||
|
||||
import { readFile, unlink } from "node:fs/promises";
|
||||
@@ -158,22 +157,15 @@ function abortError(): Error {
|
||||
return error;
|
||||
}
|
||||
|
||||
/** Register the full browser tool suite; returns the combined unregister disposer. */
|
||||
export function registerTools(deps: ToolDeps): () => void {
|
||||
const { ctx, registry } = deps;
|
||||
// Track every registration without touching the cordis context itself
|
||||
// (contexts are fiber-owned; the lazy path registers from a listener fiber).
|
||||
const disposers: (() => void)[] = [];
|
||||
const register = (definition: ToolDefinition): (() => void) => {
|
||||
const dispose = ctx.tools.register(definition);
|
||||
// Test doubles occasionally return the registry map instead of a disposer.
|
||||
if (typeof dispose === "function") disposers.push(dispose);
|
||||
return dispose;
|
||||
};
|
||||
type DefinitionRegistrar = (definition: ToolDefinition) => void;
|
||||
|
||||
/** Define the private action handlers through an injected collector. */
|
||||
function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar): void {
|
||||
const { registry } = deps;
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_session_start",
|
||||
name: "session.start",
|
||||
description:
|
||||
"Start a new browser session: opens an Agent Window in the connected browser and returns " +
|
||||
"its session id. The new session becomes the current session for subsequent browser_* calls. " +
|
||||
@@ -314,10 +306,10 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_session_stop",
|
||||
name: "session.stop",
|
||||
description:
|
||||
"Stop a browser session and close its Agent Window. Stops the given session, or the " +
|
||||
"current session when `session` is omitted. Only sessions created by browser_session_start " +
|
||||
"current session when `session` is omitted. Only plugin-created sessions " +
|
||||
"can be stopped — sessions owned by other programs sharing the bsk daemon are refused.",
|
||||
parameters: { session: SESSION_PARAM },
|
||||
output: {
|
||||
@@ -354,7 +346,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_session_list",
|
||||
name: "session.list",
|
||||
description:
|
||||
"List the browser sessions created by this plugin. Sessions owned by other programs on the " +
|
||||
"shared bsk daemon are not visible here. The session marked `current` is the one browser_* " +
|
||||
@@ -419,7 +411,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_navigate",
|
||||
name: "page.navigate",
|
||||
description:
|
||||
"Navigate the session's active tab to a URL and wait for a page lifecycle phase " +
|
||||
"(default: load). Returns the final URL after redirects.",
|
||||
@@ -461,7 +453,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
},
|
||||
async execute(args, exec) {
|
||||
if (args.url.trim().length === 0) throw new Error("url must be a non-empty string");
|
||||
const sessionId = registry.resolve(args.session, "browser_navigate");
|
||||
const sessionId = registry.resolve(args.session, "browser_page(action=navigate)");
|
||||
const cmdArgs = ["navigate", "--session", sessionId];
|
||||
if (args.waitUntil !== undefined) cmdArgs.push("--wait-until", args.waitUntil);
|
||||
if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`);
|
||||
@@ -492,12 +484,12 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
|
||||
const registerObservationTool = (kind: "snapshot" | "observe") => {
|
||||
const isSnapshot = kind === "snapshot";
|
||||
const name = isSnapshot ? "browser_snapshot" : "browser_observe";
|
||||
const name = isSnapshot ? "inspect.snapshot" : "inspect.observe";
|
||||
const description = isSnapshot
|
||||
? "Capture an indented aria-tree snapshot of the session's active tab. Interactive elements " +
|
||||
"carry @eN refs for browser_click / browser_fill. Prefer browser_observe for a richer semantic view."
|
||||
"carry @eN refs for browser_interact actions. Prefer browser_inspect action=observe for a richer semantic view."
|
||||
: "Produce a semantic VOM observation of the session's active tab (roles, states, perception " +
|
||||
"probes) with @eN refs for browser_click / browser_fill. Read-only: never submits input.";
|
||||
"probes) with @eN refs for browser_interact actions. Read-only: never submits input.";
|
||||
register(
|
||||
defineTool({
|
||||
name,
|
||||
@@ -566,10 +558,10 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_click",
|
||||
name: "interact.click",
|
||||
description:
|
||||
"Click an element in the session's active tab. Target is a snapshot ref (@e3) from the " +
|
||||
"last browser_snapshot/browser_observe, or a CSS selector.",
|
||||
"last browser_inspect observation, or a CSS selector.",
|
||||
parameters: {
|
||||
target: {
|
||||
type: "string",
|
||||
@@ -607,7 +599,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
},
|
||||
async execute(args, exec) {
|
||||
if (args.target.trim().length === 0) throw new Error("target must be a non-empty string");
|
||||
const sessionId = registry.resolve(args.session, "browser_click");
|
||||
const sessionId = registry.resolve(args.session, "browser_interact(action=click)");
|
||||
const cmdArgs = ["click", "--session", sessionId];
|
||||
if (args.button !== undefined) cmdArgs.push("--button", args.button);
|
||||
if (args.clickCount !== undefined) cmdArgs.push("--click-count", String(args.clickCount));
|
||||
@@ -630,7 +622,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_fill",
|
||||
name: "interact.fill",
|
||||
description:
|
||||
"Fill an input / textarea / contenteditable element, clearing it first by default. " +
|
||||
"Target is a snapshot ref (@e3) or a CSS selector.",
|
||||
@@ -666,7 +658,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
},
|
||||
async execute(args, exec) {
|
||||
if (args.target.trim().length === 0) throw new Error("target must be a non-empty string");
|
||||
const sessionId = registry.resolve(args.session, "browser_fill");
|
||||
const sessionId = registry.resolve(args.session, "browser_interact(action=fill)");
|
||||
const cmdArgs = ["fill", "--session", sessionId, "--value", args.value];
|
||||
if (args.noClear === true) cmdArgs.push("--no-clear");
|
||||
cmdArgs.push(args.target);
|
||||
@@ -687,7 +679,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_press",
|
||||
name: "interact.press",
|
||||
description:
|
||||
"Dispatch a keyboard key or combo (Enter, Escape, ArrowDown, Ctrl+A, …) to the session's " +
|
||||
"active tab, optionally focusing a target element first.",
|
||||
@@ -726,7 +718,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
},
|
||||
async execute(args, exec) {
|
||||
if (args.key.trim().length === 0) throw new Error("key must be a non-empty string");
|
||||
const sessionId = registry.resolve(args.session, "browser_press");
|
||||
const sessionId = registry.resolve(args.session, "browser_interact(action=press)");
|
||||
const cmdArgs = ["press", "--session", sessionId];
|
||||
if (args.target !== undefined) {
|
||||
if (args.target.trim().length === 0) throw new Error("target must be a non-empty string");
|
||||
@@ -757,7 +749,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_screenshot",
|
||||
name: "inspect.screenshot",
|
||||
description:
|
||||
"Capture a PNG screenshot of the session's active tab, or crop to a snapshot ref element. " +
|
||||
"Returns the image itself when the deployment supports image input, otherwise a file path.",
|
||||
@@ -822,7 +814,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
},
|
||||
isConcurrencySafe: () => true,
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, "browser_screenshot");
|
||||
const sessionId = registry.resolve(args.session, "browser_inspect(action=screenshot)");
|
||||
const outPath = join(
|
||||
tmpdir(),
|
||||
`bsk-screenshot-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.png`,
|
||||
@@ -887,7 +879,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_emulate",
|
||||
name: "assist.emulate",
|
||||
description:
|
||||
"Apply (or clear) mobile device emulation on the session's active tab: viewport metrics, " +
|
||||
"user agent, and touch. Overrides are per-tab and not inherited by new tabs.",
|
||||
@@ -928,7 +920,7 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, "browser_emulate");
|
||||
const sessionId = registry.resolve(args.session, "browser_assist(action=emulate)");
|
||||
if (args.off === true) {
|
||||
if (args.device !== undefined || args.width !== undefined || args.height !== undefined) {
|
||||
throw new Error("off is mutually exclusive with device/width/height");
|
||||
@@ -977,7 +969,17 @@ export function registerTools(deps: ToolDeps): () => void {
|
||||
commandLine: (args) => cmdline(deps, args),
|
||||
presentTerminalResult,
|
||||
});
|
||||
return () => {
|
||||
for (const dispose of disposers.splice(0)) dispose();
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the private action handlers without publishing them to the model. The
|
||||
* six browser tools dispatch through these handlers, preserving validation,
|
||||
* rendering, cancellation, ownership, UI presentation, and attachments.
|
||||
*/
|
||||
export function createBrowserOperationDefinitions(deps: ToolDeps): ToolDefinition[] {
|
||||
const definitions: ToolDefinition[] = [];
|
||||
defineBrowserOperations(deps, (definition) => {
|
||||
definitions.push(definition);
|
||||
});
|
||||
return definitions;
|
||||
}
|
||||
|
||||
+48
-22
@@ -1,17 +1,17 @@
|
||||
// @vitest-environment happy-dom
|
||||
// browser_screenshot keyed toolview: view-model derivation from frozen blocks,
|
||||
// browser_inspect keyed toolview: view-model derivation from frozen blocks,
|
||||
// image/path-only/running/error rendering, the registration key, and the
|
||||
// session-bound attachment loader.
|
||||
|
||||
import type { RunningToolCall, ToolResultNode } from "@deepseek-ai/dsh-client-runtime/client";
|
||||
import { cleanup, render, screen, waitFor } from "@testing-library/react";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { apply } from "../../src/client/index";
|
||||
import {
|
||||
ScreenshotToolView,
|
||||
type ScreenshotToolViewProps,
|
||||
BrowserInspectToolView,
|
||||
type BrowserInspectToolViewProps,
|
||||
viewModelOf,
|
||||
} from "../../src/client/ScreenshotToolView";
|
||||
} from "../../src/client/BrowserInspectToolView";
|
||||
import { apply } from "../../src/client/index";
|
||||
|
||||
const ATTACHMENT = {
|
||||
attachmentId: "sha256:abc123",
|
||||
@@ -25,7 +25,7 @@ const ATTACHMENT = {
|
||||
function runningBlock(argsRaw: string): RunningToolCall {
|
||||
return {
|
||||
callId: "c1",
|
||||
name: "browser_screenshot",
|
||||
name: "browser_inspect",
|
||||
argsRaw,
|
||||
turn: 1,
|
||||
step: 1,
|
||||
@@ -41,7 +41,7 @@ function settledBlock(argsRaw: string, content: unknown[], isError = false): Too
|
||||
seq: 1,
|
||||
time: 0,
|
||||
callId: "c1",
|
||||
call: { name: "browser_screenshot", argsRaw },
|
||||
call: { name: "browser_inspect", argsRaw },
|
||||
callTime: 0,
|
||||
content,
|
||||
isError,
|
||||
@@ -58,14 +58,14 @@ afterEach(cleanup);
|
||||
|
||||
describe("viewModelOf", () => {
|
||||
it("derives the running model from the call frame", () => {
|
||||
const model = viewModelOf(runningBlock('{"session":"s1"}'));
|
||||
const model = viewModelOf(runningBlock('{"action":"screenshot","session":"s1"}'));
|
||||
expect(model.state).toBe("running");
|
||||
expect(model.command).toBe("bsk screenshot --session s1");
|
||||
expect(model.image).toBeNull();
|
||||
});
|
||||
|
||||
it("marks settled results carrying an image block", () => {
|
||||
const block = settledBlock("{}", [
|
||||
const block = settledBlock('{"action":"screenshot"}', [
|
||||
{ type: "text", text: IMAGE_TEXT },
|
||||
{ type: "image", attachment: ATTACHMENT },
|
||||
]);
|
||||
@@ -77,30 +77,56 @@ describe("viewModelOf", () => {
|
||||
});
|
||||
|
||||
it("keeps the path-only form image-free", () => {
|
||||
const model = viewModelOf(settledBlock("{}", [{ type: "text", text: PATH_TEXT }]));
|
||||
const model = viewModelOf(
|
||||
settledBlock('{"action":"screenshot"}', [{ type: "text", text: PATH_TEXT }]),
|
||||
);
|
||||
expect(model.state).toBe("ok");
|
||||
expect(model.image).toBeNull();
|
||||
expect(model.summary).toBe(PATH_TEXT);
|
||||
});
|
||||
|
||||
it("marks error results", () => {
|
||||
const model = viewModelOf(settledBlock("{}", [{ type: "text", text: "Error: boom" }], true));
|
||||
const model = viewModelOf(
|
||||
settledBlock('{"action":"screenshot"}', [{ type: "text", text: "Error: boom" }], true),
|
||||
);
|
||||
expect(model.state).toBe("error");
|
||||
});
|
||||
|
||||
it("renders non-screenshot inspect actions as ordinary terminal calls", () => {
|
||||
const model = viewModelOf(
|
||||
settledBlock('{"action":"observe","session":"s1"}', [
|
||||
{ type: "text", text: "page observation" },
|
||||
]),
|
||||
);
|
||||
expect(model.title).toBe("Observe");
|
||||
expect(model.command).toBe("bsk observe --session s1");
|
||||
expect(model.image).toBeNull();
|
||||
});
|
||||
|
||||
it("preserves action-specific arguments in diagnostic command cards", () => {
|
||||
const model = viewModelOf(
|
||||
runningBlock(
|
||||
'{"action":"console","session":"s1","tabId":7,"since":4,"limit":20,"maxTextChars":500,"includeStack":true}',
|
||||
),
|
||||
);
|
||||
expect(model.command).toBe(
|
||||
"bsk console --session s1 --tab-id 7 --since 4 --limit 20 --max-text-chars 500 --include-stack",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("ScreenshotToolView", () => {
|
||||
const baseProps = { callId: "c1", toolName: "browser_screenshot", openFile: () => {} };
|
||||
describe("BrowserInspectToolView", () => {
|
||||
const baseProps = { callId: "c1", toolName: "browser_inspect", openFile: () => {} };
|
||||
const renderView = (
|
||||
block: ScreenshotToolViewProps["block"],
|
||||
loadImage: ScreenshotToolViewProps["loadImage"],
|
||||
block: BrowserInspectToolViewProps["block"],
|
||||
loadImage: BrowserInspectToolViewProps["loadImage"],
|
||||
) => {
|
||||
const props = { ...baseProps, block, loadImage } as unknown as ScreenshotToolViewProps;
|
||||
return render(<ScreenshotToolView {...props} />);
|
||||
const props = { ...baseProps, block, loadImage } as unknown as BrowserInspectToolViewProps;
|
||||
return render(<BrowserInspectToolView {...props} />);
|
||||
};
|
||||
|
||||
it("renders the screenshot image through the loader", async () => {
|
||||
const block = settledBlock("{}", [
|
||||
const block = settledBlock('{"action":"screenshot"}', [
|
||||
{ type: "text", text: IMAGE_TEXT },
|
||||
{ type: "image", attachment: ATTACHMENT },
|
||||
]);
|
||||
@@ -115,7 +141,7 @@ describe("ScreenshotToolView", () => {
|
||||
});
|
||||
|
||||
it("renders the path-only form without touching the loader", async () => {
|
||||
const block = settledBlock("{}", [{ type: "text", text: PATH_TEXT }]);
|
||||
const block = settledBlock('{"action":"screenshot"}', [{ type: "text", text: PATH_TEXT }]);
|
||||
const loadImage = vi.fn(async () => "blob:unused");
|
||||
renderView(block, loadImage);
|
||||
screen.getByRole("button", { name: /screenshot/i }).click();
|
||||
@@ -125,14 +151,14 @@ describe("ScreenshotToolView", () => {
|
||||
|
||||
it("renders the running form without output", () => {
|
||||
const loadImage = vi.fn();
|
||||
renderView(runningBlock("{}"), loadImage);
|
||||
renderView(runningBlock('{"action":"screenshot"}'), loadImage);
|
||||
expect(screen.getByText("bsk screenshot --session (current)")).toBeTruthy();
|
||||
expect(loadImage).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("client plugin registration", () => {
|
||||
it("registers the browser_screenshot toolview and the shell observation overlay", () => {
|
||||
it("registers the browser_inspect toolview and the shell observation overlay", () => {
|
||||
const registrations: { name: string; key?: string; id?: string }[] = [];
|
||||
const sessions = { binding: () => undefined };
|
||||
const ctx = {
|
||||
@@ -150,7 +176,7 @@ describe("client plugin registration", () => {
|
||||
};
|
||||
apply(ctx as never);
|
||||
expect(registrations).toEqual([
|
||||
{ name: "tool.call.toolview", key: "browser_screenshot" },
|
||||
{ name: "tool.call.toolview", key: "browser_inspect" },
|
||||
{ name: "shell.overlay", id: "bsk-observation" },
|
||||
]);
|
||||
});
|
||||
@@ -73,15 +73,15 @@ describe("dispose cleanup ownership", () => {
|
||||
};
|
||||
apply(ctx as never, { maxSessions: 5, lazyTools: false }, { runnerFactory: () => runner });
|
||||
|
||||
const start = tools.get("browser_session_start");
|
||||
const snapshot = tools.get("browser_snapshot");
|
||||
if (start === undefined || snapshot === undefined) throw new Error("tools not registered");
|
||||
await start.execute({}, makeExec());
|
||||
await start.execute({}, makeExec());
|
||||
const session = tools.get("browser_session");
|
||||
const inspect = tools.get("browser_inspect");
|
||||
if (session === undefined || inspect === undefined) throw new Error("tools not registered");
|
||||
await session.execute({ action: "start" }, makeExec());
|
||||
await session.execute({ action: "start" }, makeExec());
|
||||
// A foreign session cannot even be referenced any more.
|
||||
await expect(snapshot.execute({ session: "ext9" }, makeExec())).rejects.toThrow(
|
||||
/does not belong to this plugin/,
|
||||
);
|
||||
await expect(
|
||||
inspect.execute({ action: "snapshot", session: "ext9" }, makeExec()),
|
||||
).rejects.toThrow(/does not belong to this plugin/);
|
||||
|
||||
await Promise.all(disposers.map((dispose) => dispose()));
|
||||
|
||||
|
||||
@@ -211,19 +211,25 @@ describe("lazyTools wiring in apply()", () => {
|
||||
|
||||
it("lazyTools: false registers the suite at apply time", () => {
|
||||
const { tools } = applyHarness({ lazyTools: false });
|
||||
expect(tools.has("browser_session_start")).toBe(true);
|
||||
expect(tools.has("browser_emulate")).toBe(true);
|
||||
expect([...tools.keys()].sort()).toEqual([
|
||||
"browser_assist",
|
||||
"browser_inspect",
|
||||
"browser_interact",
|
||||
"browser_page",
|
||||
"browser_session",
|
||||
"browser_tabs",
|
||||
]);
|
||||
});
|
||||
|
||||
it("lazyTools: true hides the suite until the skill fires", () => {
|
||||
const { tools, listeners } = applyHarness({ lazyTools: true });
|
||||
expect(tools.has("browser_session_start")).toBe(false);
|
||||
expect(tools.has("browser_session")).toBe(false);
|
||||
callListeners(
|
||||
listeners,
|
||||
"tools/result",
|
||||
{ name: "skill", arguments: { name: "browser-skill" } },
|
||||
{ isError: false },
|
||||
);
|
||||
expect(tools.has("browser_session_start")).toBe(true);
|
||||
expect(tools.has("browser_session")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -71,7 +71,9 @@ describe("SessionRegistry", () => {
|
||||
|
||||
it("resolve throws actionable guidance when no session exists", () => {
|
||||
const registry = new SessionRegistry(5);
|
||||
expect(() => registry.resolve(undefined, "browser_click")).toThrow(/browser_session_start/);
|
||||
expect(() => registry.resolve(undefined, "browser_interact(action=click)")).toThrow(
|
||||
/browser_session action=start/,
|
||||
);
|
||||
});
|
||||
|
||||
it("recency order survives remove of the current session", () => {
|
||||
|
||||
@@ -32,44 +32,22 @@ describe("registerBskSkill", () => {
|
||||
const content = String(skill.content);
|
||||
const mentionedTools = [...new Set(content.match(/\bbrowser_[a-z][a-z_]*\b/g) ?? [])].sort();
|
||||
expect(mentionedTools).toEqual([
|
||||
"browser_click",
|
||||
"browser_console",
|
||||
"browser_emulate",
|
||||
"browser_fill",
|
||||
"browser_get_html",
|
||||
"browser_hover",
|
||||
"browser_navigate",
|
||||
"browser_navigate_back",
|
||||
"browser_navigate_forward",
|
||||
"browser_network",
|
||||
"browser_observe",
|
||||
"browser_press",
|
||||
"browser_reload",
|
||||
"browser_request_help",
|
||||
"browser_screenshot",
|
||||
"browser_select",
|
||||
"browser_session_list",
|
||||
"browser_session_start",
|
||||
"browser_session_stop",
|
||||
"browser_snapshot",
|
||||
"browser_tab_borrow",
|
||||
"browser_tab_close",
|
||||
"browser_tab_create",
|
||||
"browser_tab_list",
|
||||
"browser_tab_return",
|
||||
"browser_tab_select",
|
||||
"browser_wait_for_navigation",
|
||||
"browser_window_resize",
|
||||
"browser_assist",
|
||||
"browser_inspect",
|
||||
"browser_interact",
|
||||
"browser_page",
|
||||
"browser_session",
|
||||
"browser_tabs",
|
||||
]);
|
||||
expect(String(skill.description)).not.toMatch(/\bbsk\b/i);
|
||||
expect(content).not.toMatch(/\bbsk\b/i);
|
||||
expect(content).not.toMatch(/```(?:bash|sh|shell)\b/i);
|
||||
expect(content).not.toMatch(/--[a-z]/);
|
||||
expect(content).toContain("All browser operations must use the injected tools directly");
|
||||
expect(content).toMatch(/All browser work\s+must use the injected tools directly/);
|
||||
expect(content).toContain("Mandatory workflow");
|
||||
expect(content).toContain("Refs invalidate after navigation");
|
||||
expect(content).toContain("evaluation and interaction recording are intentionally unsupported");
|
||||
expect(content.length).toBeGreaterThan(10_000);
|
||||
expect(content.length).toBeGreaterThan(6_000);
|
||||
expect(skill.source).toBe("bundled");
|
||||
// Source frontmatter is registration metadata and must not leak into the body.
|
||||
expect(content.startsWith("---")).toBe(false);
|
||||
@@ -144,7 +122,7 @@ describe("armAgentScopedBskSkill", () => {
|
||||
});
|
||||
|
||||
const after = await skillFiber.ctx.skills.get("browser-skill", { scope: agentKey });
|
||||
expect(after?.content).toContain("All browser operations must use the injected tools directly");
|
||||
expect(after?.content).toMatch(/All browser work\s+must use the injected tools directly/);
|
||||
expect(after?.content).not.toMatch(/\bbsk\b/i);
|
||||
expect(after?.source).toBe("bundled");
|
||||
|
||||
|
||||
@@ -3,11 +3,12 @@ import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { ToolDefinition, ToolRunContext } from "@deepseek-ai/dsh-tools";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { registerBrowserTools } from "../src/browser-tools";
|
||||
import { ObservationService } from "../src/observation";
|
||||
import { KeyedExecutor } from "../src/queue";
|
||||
import type { BskRunner, BskRunOptions, BskRunResult } from "../src/runner";
|
||||
import { SessionRegistry } from "../src/sessions";
|
||||
import { type PluginConfig, registerTools } from "../src/tools";
|
||||
import type { PluginConfig } from "../src/tools";
|
||||
|
||||
const CONFIG: PluginConfig = {
|
||||
bskPath: "bsk",
|
||||
@@ -84,8 +85,66 @@ function fakeRunner(responses: Record<string, unknown>) {
|
||||
};
|
||||
}
|
||||
|
||||
const ACTION_ROUTES: Record<string, readonly [string, string]> = {
|
||||
"session.start": ["browser_session", "start"],
|
||||
"session.stop": ["browser_session", "stop"],
|
||||
"session.list": ["browser_session", "list"],
|
||||
"page.navigate": ["browser_page", "navigate"],
|
||||
"page.back": ["browser_page", "back"],
|
||||
"page.forward": ["browser_page", "forward"],
|
||||
"page.reload": ["browser_page", "reload"],
|
||||
"page.wait": ["browser_page", "wait"],
|
||||
"inspect.observe": ["browser_inspect", "observe"],
|
||||
"inspect.snapshot": ["browser_inspect", "snapshot"],
|
||||
"inspect.html": ["browser_inspect", "html"],
|
||||
"inspect.screenshot": ["browser_inspect", "screenshot"],
|
||||
"inspect.console": ["browser_inspect", "console"],
|
||||
"inspect.network": ["browser_inspect", "network"],
|
||||
"interact.click": ["browser_interact", "click"],
|
||||
"interact.hover": ["browser_interact", "hover"],
|
||||
"interact.fill": ["browser_interact", "fill"],
|
||||
"interact.select": ["browser_interact", "select"],
|
||||
"interact.press": ["browser_interact", "press"],
|
||||
"tabs.list": ["browser_tabs", "list"],
|
||||
"tabs.create": ["browser_tabs", "create"],
|
||||
"tabs.select": ["browser_tabs", "select"],
|
||||
"tabs.close": ["browser_tabs", "close"],
|
||||
"tabs.borrow": ["browser_tabs", "borrow"],
|
||||
"tabs.return": ["browser_tabs", "return"],
|
||||
"assist.resize": ["browser_assist", "resize"],
|
||||
"assist.emulate": ["browser_assist", "emulate"],
|
||||
"assist.request-help": ["browser_assist", "request-help"],
|
||||
};
|
||||
|
||||
class TestToolRegistry extends Map<string, ToolDefinition> {
|
||||
override get(name: string): ToolDefinition | undefined {
|
||||
const direct = super.get(name);
|
||||
if (direct !== undefined) return direct;
|
||||
const route = ACTION_ROUTES[name];
|
||||
if (route === undefined) return undefined;
|
||||
const [toolName, action] = route;
|
||||
const tool = super.get(toolName);
|
||||
if (tool === undefined) return undefined;
|
||||
const withAction = (args: unknown) => ({
|
||||
action,
|
||||
...(args as Record<string, unknown>),
|
||||
});
|
||||
return {
|
||||
...tool,
|
||||
output: {
|
||||
...tool.output,
|
||||
render: (args, value) => tool.output.render(withAction(args) as never, value),
|
||||
},
|
||||
execute: (args, exec) => tool.execute(withAction(args) as never, exec),
|
||||
isConcurrencySafe: (args) => tool.isConcurrencySafe?.(withAction(args) as never) === true,
|
||||
presentCall: (args) => tool.presentCall?.(withAction(args) as never),
|
||||
presentResult: (args, result) => tool.presentResult?.(withAction(args) as never, result),
|
||||
} as ToolDefinition;
|
||||
}
|
||||
}
|
||||
|
||||
function makeCtx(services: Record<string, unknown> = {}) {
|
||||
const tools = new Map<string, ToolDefinition>();
|
||||
const tools = new TestToolRegistry();
|
||||
const ctx = {
|
||||
tools: { register: (def: ToolDefinition) => tools.set(def.name, def) },
|
||||
get: (key: string) => services[key],
|
||||
@@ -118,7 +177,7 @@ function setup(
|
||||
const { runner, calls } = fakeRunner(responses);
|
||||
const registry = new SessionRegistry(maxSessions);
|
||||
const config = { ...CONFIG, maxSessions };
|
||||
registerTools({
|
||||
registerBrowserTools({
|
||||
ctx: ctx as never,
|
||||
runner: runner as BskRunner,
|
||||
registry,
|
||||
@@ -130,56 +189,98 @@ function setup(
|
||||
}
|
||||
|
||||
async function startSession(tools: Map<string, ToolDefinition>, sessionId = "s1") {
|
||||
const tool = tools.get("browser_session_start");
|
||||
if (tool === undefined) throw new Error("browser_session_start not registered");
|
||||
const tool = tools.get("session.start");
|
||||
if (tool === undefined) throw new Error("session.start not registered");
|
||||
return tool.execute({}, makeExec()) as Promise<{ sessionId: string }>;
|
||||
}
|
||||
|
||||
const START_REPLY = (id: string) => ({ session_id: id, browser_instance_id: "chrome-1" });
|
||||
|
||||
const EXPECTED_ACTIONS = {
|
||||
browser_session: ["start", "stop", "list"],
|
||||
browser_page: ["navigate", "back", "forward", "reload", "wait"],
|
||||
browser_inspect: ["observe", "snapshot", "html", "screenshot", "console", "network"],
|
||||
browser_interact: ["click", "hover", "fill", "select", "press"],
|
||||
browser_tabs: ["list", "create", "select", "close", "borrow", "return"],
|
||||
browser_assist: ["resize", "emulate", "request-help"],
|
||||
} as const;
|
||||
|
||||
describe("tool registration", () => {
|
||||
it("registers the full browser tool suite", () => {
|
||||
it("registers exactly six public schemas with the complete action contract", () => {
|
||||
const { tools } = setup({});
|
||||
expect([...tools.keys()].sort()).toEqual([
|
||||
"browser_click",
|
||||
"browser_console",
|
||||
"browser_emulate",
|
||||
"browser_fill",
|
||||
"browser_get_html",
|
||||
"browser_hover",
|
||||
"browser_navigate",
|
||||
"browser_navigate_back",
|
||||
"browser_navigate_forward",
|
||||
"browser_network",
|
||||
"browser_observe",
|
||||
"browser_press",
|
||||
"browser_reload",
|
||||
"browser_request_help",
|
||||
"browser_screenshot",
|
||||
"browser_select",
|
||||
"browser_session_list",
|
||||
"browser_session_start",
|
||||
"browser_session_stop",
|
||||
"browser_snapshot",
|
||||
"browser_tab_borrow",
|
||||
"browser_tab_close",
|
||||
"browser_tab_create",
|
||||
"browser_tab_list",
|
||||
"browser_tab_return",
|
||||
"browser_tab_select",
|
||||
"browser_wait_for_navigation",
|
||||
"browser_window_resize",
|
||||
]);
|
||||
expect([...tools.keys()].sort()).toEqual(Object.keys(EXPECTED_ACTIONS).sort());
|
||||
for (const [name, actions] of Object.entries(EXPECTED_ACTIONS)) {
|
||||
const properties = tools.get(name)?.parameters.properties as
|
||||
| Record<string, unknown>
|
||||
| undefined;
|
||||
expect(properties?.action).toMatchObject({ enum: [...actions] });
|
||||
}
|
||||
});
|
||||
|
||||
it("requires an action on every public tool", async () => {
|
||||
const { tools } = setup({});
|
||||
for (const name of Object.keys(EXPECTED_ACTIONS)) {
|
||||
await expect(tools.get(name)?.execute({}, makeExec())).rejects.toThrow(/invalid arguments/i);
|
||||
}
|
||||
});
|
||||
|
||||
it("validates model args through defineTool (missing required param)", async () => {
|
||||
const { tools } = setup({});
|
||||
const navigate = tools.get("browser_navigate");
|
||||
const navigate = tools.get("page.navigate");
|
||||
await expect(navigate?.execute({}, makeExec())).rejects.toThrow(/invalid arguments/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("browser_session_start", () => {
|
||||
describe("action dispatch", () => {
|
||||
it("rejects actions outside the public contract", async () => {
|
||||
const { tools } = setup({});
|
||||
await expect(
|
||||
tools.get("browser_page")?.execute({ action: "unsupported" }, makeExec()),
|
||||
).rejects.toThrow(/invalid arguments|unsupported browser action/i);
|
||||
});
|
||||
|
||||
it("dispatches through the existing operation definitions", async () => {
|
||||
const { tools, calls, registry } = setup({
|
||||
"session start": START_REPLY("s1"),
|
||||
navigate: { tab_id: 7, url: "https://example.test/", reached: "load" },
|
||||
observe: SNAPSHOT_REPLY,
|
||||
fill: { tab_id: 7, value_length: 5 },
|
||||
"tab list": { tabs: [] },
|
||||
"window resize": { window_id: 9, width: 900, height: 700 },
|
||||
});
|
||||
await tools.get("browser_session")?.execute({ action: "start" }, makeExec());
|
||||
await tools
|
||||
.get("browser_page")
|
||||
?.execute({ action: "navigate", url: "https://example.test/" }, makeExec());
|
||||
await tools.get("browser_inspect")?.execute({ action: "observe" }, makeExec());
|
||||
await tools
|
||||
.get("browser_interact")
|
||||
?.execute({ action: "fill", target: "@e1", value: "hello" }, makeExec());
|
||||
await tools.get("browser_tabs")?.execute({ action: "list" }, makeExec());
|
||||
await tools
|
||||
.get("browser_assist")
|
||||
?.execute({ action: "resize", width: 900, height: 700 }, makeExec());
|
||||
|
||||
expect(registry.current()).toBe("s1");
|
||||
expect(calls.map(({ args }) => args)).toEqual([
|
||||
["session", "start"],
|
||||
["navigate", "--session", "s1", "https://example.test/"],
|
||||
["observe", "--session", "s1"],
|
||||
["fill", "--session", "s1", "--value", "hello", "@e1"],
|
||||
["tab", "list", "--session", "s1"],
|
||||
["window", "resize", "--session", "s1", "--width", "900", "--height", "700"],
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps action-specific validation in the private operation", async () => {
|
||||
const { tools } = setup({});
|
||||
await expect(
|
||||
tools.get("browser_page")?.execute({ action: "navigate" }, makeExec()),
|
||||
).rejects.toThrow(/url.*required|invalid arguments/i);
|
||||
});
|
||||
});
|
||||
|
||||
describe("session.start", () => {
|
||||
it("maps the start reply and tracks the session as current", async () => {
|
||||
const { tools, registry } = setup({ "session start": START_REPLY("s1") });
|
||||
const value = await startSession(tools);
|
||||
@@ -193,7 +294,7 @@ describe("browser_session_start", () => {
|
||||
emulate: { tab_id: 7, cleared: false },
|
||||
navigate: { tab_id: 7, url: "https://example.com", reached: "load" },
|
||||
});
|
||||
const tool = tools.get("browser_session_start");
|
||||
const tool = tools.get("session.start");
|
||||
const value = (await tool?.execute(
|
||||
{ width: 1280, height: 800, url: "https://example.com", device: "iphone-14", noFocus: true },
|
||||
makeExec(),
|
||||
@@ -218,7 +319,7 @@ describe("browser_session_start", () => {
|
||||
|
||||
it("requires width and height together", async () => {
|
||||
const { tools } = setup({ "session start": START_REPLY("s1") });
|
||||
const tool = tools.get("browser_session_start");
|
||||
const tool = tools.get("session.start");
|
||||
await expect(tool?.execute({ width: 1280 }, makeExec())).rejects.toThrow(/together/);
|
||||
});
|
||||
|
||||
@@ -228,7 +329,7 @@ describe("browser_session_start", () => {
|
||||
"session stop": {},
|
||||
});
|
||||
// navigate has no canned response -> non-zero exit -> error.
|
||||
const tool = tools.get("browser_session_start");
|
||||
const tool = tools.get("session.start");
|
||||
await expect(tool?.execute({ url: "https://example.com" }, makeExec())).rejects.toThrow(
|
||||
/navigate/,
|
||||
);
|
||||
@@ -246,7 +347,7 @@ describe("multi-session behavior", () => {
|
||||
it("operation tools default to the current session and echo it back", async () => {
|
||||
const { tools, calls } = setup(responses);
|
||||
await startSession(tools);
|
||||
const snapshot = tools.get("browser_snapshot");
|
||||
const snapshot = tools.get("inspect.snapshot");
|
||||
const value = (await snapshot?.execute({}, makeExec())) as {
|
||||
session: string;
|
||||
refCount: number;
|
||||
@@ -262,7 +363,7 @@ describe("multi-session behavior", () => {
|
||||
});
|
||||
await startSession(tools, "s1");
|
||||
await startSession(tools, "s2");
|
||||
const snapshot = tools.get("browser_snapshot");
|
||||
const snapshot = tools.get("inspect.snapshot");
|
||||
const value = (await snapshot?.execute({ session: "s1" }, makeExec())) as {
|
||||
session: string;
|
||||
};
|
||||
@@ -275,7 +376,7 @@ describe("multi-session behavior", () => {
|
||||
const { tools, calls } = setup(responses);
|
||||
await startSession(tools);
|
||||
const callsBefore = calls.length;
|
||||
const snapshot = tools.get("browser_snapshot");
|
||||
const snapshot = tools.get("inspect.snapshot");
|
||||
await expect(snapshot?.execute({ session: "other" }, makeExec())).rejects.toThrow(
|
||||
/does not belong to this plugin/,
|
||||
);
|
||||
@@ -284,8 +385,8 @@ describe("multi-session behavior", () => {
|
||||
|
||||
it("fails with guidance when no session exists", async () => {
|
||||
const { tools } = setup(responses);
|
||||
const snapshot = tools.get("browser_snapshot");
|
||||
await expect(snapshot?.execute({}, makeExec())).rejects.toThrow(/browser_session_start/);
|
||||
const snapshot = tools.get("inspect.snapshot");
|
||||
await expect(snapshot?.execute({}, makeExec())).rejects.toThrow(/browser_session action=start/);
|
||||
});
|
||||
|
||||
it("enforces the configured session cap before spawning", async () => {
|
||||
@@ -295,7 +396,7 @@ describe("multi-session behavior", () => {
|
||||
1,
|
||||
);
|
||||
await startSession(tools);
|
||||
const start = tools.get("browser_session_start");
|
||||
const start = tools.get("session.start");
|
||||
await expect(start?.execute({}, makeExec())).rejects.toThrow(/session limit/);
|
||||
// A rejected start must not spawn another bsk session (no leaked session).
|
||||
expect(calls.filter((c) => c.args.join(" ").startsWith("session start"))).toHaveLength(1);
|
||||
@@ -307,7 +408,7 @@ describe("multi-session behavior", () => {
|
||||
{},
|
||||
1,
|
||||
);
|
||||
const start = tools.get("browser_session_start");
|
||||
const start = tools.get("session.start");
|
||||
if (start === undefined) throw new Error("not registered");
|
||||
// Two truly concurrent starts against a cap of 1: exactly one may spawn.
|
||||
const outcomes = await Promise.allSettled([
|
||||
@@ -325,7 +426,7 @@ describe("multi-session behavior", () => {
|
||||
it("refuses to stop a session the plugin did not create", async () => {
|
||||
const { tools, calls } = setup(responses);
|
||||
await startSession(tools);
|
||||
const stop = tools.get("browser_session_stop");
|
||||
const stop = tools.get("session.stop");
|
||||
await expect(stop?.execute({ session: "ext7" }, makeExec())).rejects.toThrow(
|
||||
/does not belong to this plugin/,
|
||||
);
|
||||
@@ -334,14 +435,14 @@ describe("multi-session behavior", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("browser_session_stop / list", () => {
|
||||
describe("session.stop / list", () => {
|
||||
it("stops the current session and clears the registry", async () => {
|
||||
const { tools, calls, registry } = setup({
|
||||
"session start": START_REPLY("s1"),
|
||||
"session stop": {},
|
||||
});
|
||||
await startSession(tools);
|
||||
const stop = tools.get("browser_session_stop");
|
||||
const stop = tools.get("session.stop");
|
||||
const value = (await stop?.execute({}, makeExec())) as { stopped: string };
|
||||
expect(value.stopped).toBe("s1");
|
||||
expect(calls[1].args).toEqual(["session", "stop", "s1"]);
|
||||
@@ -354,7 +455,7 @@ describe("browser_session_stop / list", () => {
|
||||
});
|
||||
await startSession(tools);
|
||||
await startSession(tools);
|
||||
const list = tools.get("browser_session_list");
|
||||
const list = tools.get("session.list");
|
||||
const value = (await list?.execute({}, makeExec())) as {
|
||||
sessions: { sessionId: string; current: boolean }[];
|
||||
};
|
||||
@@ -377,10 +478,10 @@ describe("interaction tools", () => {
|
||||
emulate: { tab_id: 7, cleared: false },
|
||||
};
|
||||
|
||||
it("browser_click maps target, button, and click count", async () => {
|
||||
it("interact.click maps target, button, and click count", async () => {
|
||||
const { tools, calls } = setup(responses);
|
||||
await startSession(tools);
|
||||
const click = tools.get("browser_click");
|
||||
const click = tools.get("interact.click");
|
||||
const value = (await click?.execute(
|
||||
{ target: "@e1", button: "right", clickCount: 2 },
|
||||
makeExec(),
|
||||
@@ -398,10 +499,10 @@ describe("interaction tools", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("browser_fill passes value and --no-clear", async () => {
|
||||
it("interact.fill passes value and --no-clear", async () => {
|
||||
const { tools, calls } = setup(responses);
|
||||
await startSession(tools);
|
||||
const fill = tools.get("browser_fill");
|
||||
const fill = tools.get("interact.fill");
|
||||
await fill?.execute({ target: "e2", value: "hello", noClear: true }, makeExec());
|
||||
expect(calls[1].args).toEqual([
|
||||
"fill",
|
||||
@@ -414,20 +515,20 @@ describe("interaction tools", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("browser_press picks --ref for snapshot refs and --selector otherwise", async () => {
|
||||
it("interact.press picks --ref for snapshot refs and --selector otherwise", async () => {
|
||||
const { tools, calls } = setup(responses);
|
||||
await startSession(tools);
|
||||
const press = tools.get("browser_press");
|
||||
const press = tools.get("interact.press");
|
||||
await press?.execute({ key: "Enter", target: "@e3" }, makeExec());
|
||||
expect(calls[1].args).toEqual(["press", "--session", "s1", "--ref", "@e3", "Enter"]);
|
||||
await press?.execute({ key: "Ctrl+A", target: "#field" }, makeExec());
|
||||
expect(calls[2].args).toEqual(["press", "--session", "s1", "--selector", "#field", "Ctrl+A"]);
|
||||
});
|
||||
|
||||
it("browser_navigate converts waitUntil and timeoutMs", async () => {
|
||||
it("page.navigate converts waitUntil and timeoutMs", async () => {
|
||||
const { tools, calls } = setup(responses);
|
||||
await startSession(tools);
|
||||
const navigate = tools.get("browser_navigate");
|
||||
const navigate = tools.get("page.navigate");
|
||||
const value = (await navigate?.execute(
|
||||
{ url: "https://a.test", waitUntil: "networkidle", timeoutMs: 1500 },
|
||||
makeExec(),
|
||||
@@ -445,16 +546,16 @@ describe("interaction tools", () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it("browser_emulate rejects off combined with overrides", async () => {
|
||||
it("assist.emulate rejects off combined with overrides", async () => {
|
||||
const { tools } = setup(responses);
|
||||
await startSession(tools);
|
||||
const emulate = tools.get("browser_emulate");
|
||||
const emulate = tools.get("assist.emulate");
|
||||
await expect(emulate?.execute({ off: true, device: "iphone-14" }, makeExec())).rejects.toThrow(
|
||||
/mutually exclusive/,
|
||||
);
|
||||
});
|
||||
|
||||
it("browser_hover and browser_select map their structured arguments", async () => {
|
||||
it("interact.hover and interact.select map their structured arguments", async () => {
|
||||
const { tools, calls } = setup({
|
||||
"session start": START_REPLY("s1"),
|
||||
hover: { tab_id: 7, used_ref: "@e3", x: 12, y: 24 },
|
||||
@@ -468,7 +569,7 @@ describe("interaction tools", () => {
|
||||
});
|
||||
await startSession(tools);
|
||||
const hovered = (await tools
|
||||
.get("browser_hover")
|
||||
.get("interact.hover")
|
||||
?.execute(
|
||||
{ target: "@e3", modifiers: ["shift"], settleMs: 250, timeoutMs: 2000 },
|
||||
makeExec(),
|
||||
@@ -488,7 +589,7 @@ describe("interaction tools", () => {
|
||||
]);
|
||||
|
||||
const selected = (await tools
|
||||
.get("browser_select")
|
||||
.get("interact.select")
|
||||
?.execute({ target: "@e4", values: ["a", "b"], tabId: 7 }, makeExec())) as {
|
||||
selectedValues: string[];
|
||||
selectedLabels: string[];
|
||||
@@ -546,25 +647,25 @@ describe("phase-one tab tools", () => {
|
||||
});
|
||||
await startSession(tools);
|
||||
|
||||
const listed = (await tools
|
||||
.get("browser_tab_list")
|
||||
?.execute({ scope: "agent" }, makeExec())) as { tabs: { tabId: number }[] };
|
||||
const listed = (await tools.get("tabs.list")?.execute({ scope: "agent" }, makeExec())) as {
|
||||
tabs: { tabId: number }[];
|
||||
};
|
||||
expect(listed.tabs[0].tabId).toBe(7);
|
||||
const created = (await tools
|
||||
.get("browser_tab_create")
|
||||
.get("tabs.create")
|
||||
?.execute({ url: "https://new.test", active: false, index: 1 }, makeExec())) as {
|
||||
tabId: number;
|
||||
};
|
||||
expect(created.tabId).toBe(8);
|
||||
await tools.get("browser_tab_close")?.execute({ tabId: 8 }, makeExec());
|
||||
await tools.get("browser_tab_select")?.execute({ tabId: 7 }, makeExec());
|
||||
const borrowed = (await tools
|
||||
.get("browser_tab_borrow")
|
||||
?.execute({ tabId: 20 }, makeExec())) as { originalWindowId: number };
|
||||
await tools.get("tabs.close")?.execute({ tabId: 8 }, makeExec());
|
||||
await tools.get("tabs.select")?.execute({ tabId: 7 }, makeExec());
|
||||
const borrowed = (await tools.get("tabs.borrow")?.execute({ tabId: 20 }, makeExec())) as {
|
||||
originalWindowId: number;
|
||||
};
|
||||
expect(borrowed.originalWindowId).toBe(9);
|
||||
const returned = (await tools
|
||||
.get("browser_tab_return")
|
||||
?.execute({ tabId: 20 }, makeExec())) as { fallback: boolean };
|
||||
const returned = (await tools.get("tabs.return")?.execute({ tabId: 20 }, makeExec())) as {
|
||||
fallback: boolean;
|
||||
};
|
||||
expect(returned.fallback).toBe(false);
|
||||
|
||||
expect(calls.slice(1).map((call) => call.args)).toEqual([
|
||||
@@ -614,15 +715,14 @@ describe("phase-one navigation tools", () => {
|
||||
});
|
||||
await startSession(tools);
|
||||
await tools
|
||||
.get("browser_navigate_back")
|
||||
.get("page.back")
|
||||
?.execute({ tabId: 7, waitUntil: "commit", timeoutMs: 1000 }, makeExec());
|
||||
await tools.get("browser_navigate_forward")?.execute({}, makeExec());
|
||||
await tools
|
||||
.get("browser_reload")
|
||||
?.execute({ hard: true, waitUntil: "networkidle" }, makeExec());
|
||||
const waited = (await tools
|
||||
.get("browser_wait_for_navigation")
|
||||
?.execute({ timeoutMs: 2500 }, makeExec())) as { reached: string; errorText: string };
|
||||
await tools.get("page.forward")?.execute({}, makeExec());
|
||||
await tools.get("page.reload")?.execute({ hard: true, waitUntil: "networkidle" }, makeExec());
|
||||
const waited = (await tools.get("page.wait")?.execute({ timeoutMs: 2500 }, makeExec())) as {
|
||||
reached: string;
|
||||
errorText: string;
|
||||
};
|
||||
expect(waited).toMatchObject({ reached: "timeout", errorText: "no navigation" });
|
||||
expect(calls[1].args).toEqual([
|
||||
"navigate-back",
|
||||
@@ -668,7 +768,7 @@ describe("phase-one support tools", () => {
|
||||
},
|
||||
});
|
||||
await startSession(tools);
|
||||
const value = (await tools.get("browser_request_help")?.execute(
|
||||
const value = (await tools.get("assist.request-help")?.execute(
|
||||
{
|
||||
prompt: "Please sign in",
|
||||
title: "Login required",
|
||||
@@ -736,17 +836,17 @@ describe("phase-one support tools", () => {
|
||||
});
|
||||
await startSession(tools);
|
||||
const html = (await tools
|
||||
.get("browser_get_html")
|
||||
.get("inspect.html")
|
||||
?.execute({ ref: "@e2", maxBytes: 2048 }, makeExec())) as { html: string };
|
||||
expect(html.html).toBe("<main>Hi</main>");
|
||||
const consoleValue = (await tools
|
||||
.get("browser_console")
|
||||
.get("inspect.console")
|
||||
?.execute({ since: 1, limit: 10, includeStack: true }, makeExec())) as {
|
||||
entries: { stackTrace: { functionName: string }[] }[];
|
||||
};
|
||||
expect(consoleValue.entries[0].stackTrace[0].functionName).toBe("run");
|
||||
const networkValue = (await tools
|
||||
.get("browser_network")
|
||||
.get("inspect.network")
|
||||
?.execute({ tabId: 7, maxTextChars: 512 }, makeExec())) as {
|
||||
entries: { mimeType: string }[];
|
||||
};
|
||||
@@ -764,7 +864,7 @@ describe("phase-one support tools", () => {
|
||||
"window resize": { window_id: 2, width: 1280, height: 800 },
|
||||
});
|
||||
await startSession(tools);
|
||||
const resize = tools.get("browser_window_resize");
|
||||
const resize = tools.get("assist.resize");
|
||||
const value = (await resize?.execute({ width: 1280, height: 800 }, makeExec())) as {
|
||||
windowId: number;
|
||||
};
|
||||
@@ -785,7 +885,7 @@ describe("phase-one support tools", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("browser_screenshot", () => {
|
||||
describe("inspect.screenshot", () => {
|
||||
function pngFile(): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "bsk-test-"));
|
||||
const path = join(dir, "shot.png");
|
||||
@@ -810,7 +910,7 @@ describe("browser_screenshot", () => {
|
||||
screenshot: { tab_id: 7, width: 800, height: 600, format: "png", path, byte_size: 8 },
|
||||
});
|
||||
await startSession(tools);
|
||||
const screenshot = tools.get("browser_screenshot");
|
||||
const screenshot = tools.get("inspect.screenshot");
|
||||
const value = (await screenshot?.execute({ ref: "@e1" }, makeExec())) as {
|
||||
path: string;
|
||||
image?: unknown;
|
||||
@@ -859,7 +959,7 @@ describe("browser_screenshot", () => {
|
||||
options: {},
|
||||
},
|
||||
});
|
||||
const screenshot = tools.get("browser_screenshot");
|
||||
const screenshot = tools.get("inspect.screenshot");
|
||||
const value = (await screenshot?.execute({}, exec)) as {
|
||||
image?: { attachmentId: string; mediaType: string };
|
||||
};
|
||||
@@ -937,7 +1037,7 @@ describe("browser_screenshot", () => {
|
||||
options: {},
|
||||
},
|
||||
});
|
||||
const screenshot = tools.get("browser_screenshot");
|
||||
const screenshot = tools.get("inspect.screenshot");
|
||||
const value = (await screenshot?.execute({}, exec)) as { image?: unknown; path: string };
|
||||
expect(value.image).toBeUndefined();
|
||||
expect(value.path).toBe(path);
|
||||
@@ -950,14 +1050,14 @@ describe("error and cancellation semantics", () => {
|
||||
"session start": START_REPLY("s1"),
|
||||
});
|
||||
await startSession(tools);
|
||||
const snapshot = tools.get("browser_snapshot");
|
||||
const snapshot = tools.get("inspect.snapshot");
|
||||
await expect(snapshot?.execute({}, makeExec())).rejects.toThrow(/no canned response/);
|
||||
});
|
||||
|
||||
it("turns an aborted child into an AbortError", async () => {
|
||||
const { tools } = setup({ "session start": START_REPLY("s1") });
|
||||
await startSession(tools);
|
||||
const snapshot = tools.get("browser_snapshot");
|
||||
const snapshot = tools.get("inspect.snapshot");
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
const exec = makeExec({ signal: controller.signal });
|
||||
@@ -974,7 +1074,7 @@ describe("error and cancellation semantics", () => {
|
||||
killAll() {},
|
||||
killFor: () => 0,
|
||||
};
|
||||
registerTools({
|
||||
registerBrowserTools({
|
||||
ctx: ctx as never,
|
||||
runner,
|
||||
registry,
|
||||
@@ -982,7 +1082,7 @@ describe("error and cancellation semantics", () => {
|
||||
observation: disabledObservation({ ctx, runner, registry }),
|
||||
queue: new KeyedExecutor(),
|
||||
});
|
||||
const start = tools.get("browser_session_start");
|
||||
const start = tools.get("session.start");
|
||||
await expect(start?.execute({}, makeExec())).rejects.toThrow(/BrowserSkill must be installed/);
|
||||
});
|
||||
});
|
||||
@@ -990,7 +1090,7 @@ describe("error and cancellation semantics", () => {
|
||||
describe("presentation", () => {
|
||||
it("presents calls as terminal cards with the bsk command line", () => {
|
||||
const { tools } = setup({});
|
||||
const click = tools.get("browser_click");
|
||||
const click = tools.get("interact.click");
|
||||
const view = click?.presentCall?.({ target: "@e1" }) as {
|
||||
card: string;
|
||||
title: string;
|
||||
@@ -1001,7 +1101,7 @@ describe("presentation", () => {
|
||||
|
||||
it("presents results as terminal cards carrying the rendered text", () => {
|
||||
const { tools } = setup({});
|
||||
const click = tools.get("browser_click");
|
||||
const click = tools.get("interact.click");
|
||||
const view = click?.presentResult?.({ target: "@e1" }, {
|
||||
content: [{ type: "text", text: "clicked" }],
|
||||
isError: false,
|
||||
@@ -1011,7 +1111,7 @@ describe("presentation", () => {
|
||||
|
||||
it("projects the text block of multi-block results (screenshot text + image)", () => {
|
||||
const { tools } = setup({});
|
||||
const screenshot = tools.get("browser_screenshot");
|
||||
const screenshot = tools.get("inspect.screenshot");
|
||||
const view = screenshot?.presentResult?.({}, {
|
||||
content: [
|
||||
{ type: "text", text: "[session s1] screenshot of tab 7" },
|
||||
@@ -1070,7 +1170,7 @@ describe("screenshot scratch file lifecycle", () => {
|
||||
function setupWithRunner(runner: BskRunner, services: Record<string, unknown> = {}) {
|
||||
const { ctx, tools } = makeCtx(services);
|
||||
const registry = new SessionRegistry(5);
|
||||
registerTools({
|
||||
registerBrowserTools({
|
||||
ctx: ctx as never,
|
||||
runner,
|
||||
registry,
|
||||
@@ -1105,7 +1205,7 @@ describe("screenshot scratch file lifecycle", () => {
|
||||
options: {},
|
||||
},
|
||||
});
|
||||
const screenshot = tools.get("browser_screenshot");
|
||||
const screenshot = tools.get("inspect.screenshot");
|
||||
const value = (await screenshot?.execute({}, exec)) as { image?: unknown; path: string };
|
||||
expect(value.image).toBeDefined();
|
||||
const out = calls.find((c) => c.args[0] === "screenshot")?.args.at(-1);
|
||||
@@ -1117,7 +1217,7 @@ describe("screenshot scratch file lifecycle", () => {
|
||||
const { runner, calls } = outWritingRunner();
|
||||
const { tools } = setupWithRunner(runner);
|
||||
await startSession(tools);
|
||||
const screenshot = tools.get("browser_screenshot");
|
||||
const screenshot = tools.get("inspect.screenshot");
|
||||
const value = (await screenshot?.execute({}, makeExec())) as { image?: unknown; path: string };
|
||||
expect(value.image).toBeUndefined();
|
||||
const out = calls.find((c) => c.args[0] === "screenshot")?.args.at(-1);
|
||||
@@ -1192,11 +1292,18 @@ describe("observation action instrumentation timing", () => {
|
||||
queue,
|
||||
options: { enabled: true, thumbnailIntervalMs: 1500, idleIntervalMs: 8000 },
|
||||
});
|
||||
registerTools({ ctx: ctx as never, runner, registry, config: CONFIG, observation, queue });
|
||||
registerBrowserTools({
|
||||
ctx: ctx as never,
|
||||
runner,
|
||||
registry,
|
||||
config: CONFIG,
|
||||
observation,
|
||||
queue,
|
||||
});
|
||||
|
||||
await startSession(tools);
|
||||
const navigate = tools.get("browser_navigate");
|
||||
const snapshot = tools.get("browser_snapshot");
|
||||
const navigate = tools.get("page.navigate");
|
||||
const snapshot = tools.get("inspect.snapshot");
|
||||
const a = navigate?.execute({ url: "http://x" }, makeExec()) as Promise<unknown>;
|
||||
// A starts immediately (empty queue) and blocks on the gate.
|
||||
await vi.waitFor(() =>
|
||||
|
||||
Reference in New Issue
Block a user