mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
12
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
59d6533eff | ||
|
|
536d3a9844 | ||
|
|
f06a514bd2 | ||
|
|
1e53ee63ae | ||
|
|
120dc441bb | ||
|
|
710b1219f4 | ||
|
|
177ecd3ebf | ||
|
|
0e6b9c4542 | ||
|
|
e9f0584af3 | ||
|
|
c48a53269e | ||
|
|
3bbd6ab19f | ||
|
|
60fbf30e24 |
@@ -325,3 +325,180 @@ jobs:
|
|||||||
|
|
||||||
echo "Chrome Web Store submission created successfully:"
|
echo "Chrome Web Store submission created successfully:"
|
||||||
jq '{itemId, state, warningInfo}' "$PUBLISH_RESPONSE"
|
jq '{itemId, state, warningInfo}' "$PUBLISH_RESPONSE"
|
||||||
|
|
||||||
|
publish-edge-addons:
|
||||||
|
needs: [resolve, build-extension]
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
environment: edge-addons
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
steps:
|
||||||
|
- name: Download build artifact
|
||||||
|
uses: actions/download-artifact@v8
|
||||||
|
with:
|
||||||
|
name: extension-zip
|
||||||
|
path: dist
|
||||||
|
|
||||||
|
- name: Validate Microsoft Edge Add-ons configuration
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
API_KEY: ${{ secrets.EDGE_ADDONS_API_KEY }}
|
||||||
|
CLIENT_ID: ${{ vars.EDGE_ADDONS_CLIENT_ID }}
|
||||||
|
PRODUCT_ID: ${{ vars.EDGE_ADDONS_PRODUCT_ID }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
if [ -z "$API_KEY" ]; then
|
||||||
|
echo "::error::Environment secret EDGE_ADDONS_API_KEY is not configured"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ -z "$CLIENT_ID" ]; then
|
||||||
|
echo "::error::Environment variable EDGE_ADDONS_CLIENT_ID is not configured"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [ -z "$PRODUCT_ID" ]; then
|
||||||
|
echo "::error::Environment variable EDGE_ADDONS_PRODUCT_ID is not configured"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if ! [[ "$PRODUCT_ID" =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]]; then
|
||||||
|
echo "::error::EDGE_ADDONS_PRODUCT_ID must be the Partner Center product GUID, not the public Edge Add-ons extension ID"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
ARCHIVE="$(find dist -maxdepth 1 -name '*.zip' -type f -print -quit)"
|
||||||
|
if [ -z "$ARCHIVE" ]; then
|
||||||
|
echo "::error::No extension zip found in the downloaded artifact"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "ARCHIVE=$ARCHIVE" >> "$GITHUB_ENV"
|
||||||
|
|
||||||
|
- name: Upload and publish Microsoft Edge Add-ons extension
|
||||||
|
shell: bash
|
||||||
|
env:
|
||||||
|
API_KEY: ${{ secrets.EDGE_ADDONS_API_KEY }}
|
||||||
|
CLIENT_ID: ${{ vars.EDGE_ADDONS_CLIENT_ID }}
|
||||||
|
PRODUCT_ID: ${{ vars.EDGE_ADDONS_PRODUCT_ID }}
|
||||||
|
RELEASE_VERSION: ${{ needs.resolve.outputs.version }}
|
||||||
|
run: |
|
||||||
|
set -euo pipefail
|
||||||
|
API_ROOT="https://api.addons.microsoftedge.microsoft.com/v1"
|
||||||
|
UPLOAD_HEADERS="$RUNNER_TEMP/edge-upload-headers.txt"
|
||||||
|
UPLOAD_RESPONSE="$RUNNER_TEMP/edge-upload-response.json"
|
||||||
|
STATUS_RESPONSE="$RUNNER_TEMP/edge-status-response.json"
|
||||||
|
PUBLISH_HEADERS="$RUNNER_TEMP/edge-publish-headers.txt"
|
||||||
|
PUBLISH_RESPONSE="$RUNNER_TEMP/edge-publish-response.json"
|
||||||
|
|
||||||
|
operation_id_from_headers() {
|
||||||
|
local headers="$1"
|
||||||
|
local location
|
||||||
|
location="$(awk '
|
||||||
|
tolower($1) == "location:" {
|
||||||
|
sub(/\r$/, "")
|
||||||
|
sub(/^[^:]*:[[:space:]]*/, "")
|
||||||
|
print
|
||||||
|
exit
|
||||||
|
}
|
||||||
|
' "$headers")"
|
||||||
|
location="${location%%\?*}"
|
||||||
|
printf '%s' "${location##*/}"
|
||||||
|
}
|
||||||
|
|
||||||
|
poll_operation() {
|
||||||
|
local operation_name="$1"
|
||||||
|
local status_url="$2"
|
||||||
|
local status="InProgress"
|
||||||
|
local http_status
|
||||||
|
|
||||||
|
for attempt in $(seq 1 30); do
|
||||||
|
sleep 5
|
||||||
|
http_status="$(curl --silent --show-error \
|
||||||
|
--output "$STATUS_RESPONSE" \
|
||||||
|
--write-out '%{http_code}' \
|
||||||
|
--header "Authorization: ApiKey $API_KEY" \
|
||||||
|
--header "X-ClientID: $CLIENT_ID" \
|
||||||
|
"$status_url")"
|
||||||
|
if [ "$http_status" -lt 200 ] || [ "$http_status" -ge 300 ]; then
|
||||||
|
echo "::error::Microsoft Edge Add-ons $operation_name status check failed with HTTP $http_status"
|
||||||
|
jq . "$STATUS_RESPONSE" || cat "$STATUS_RESPONSE"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
status="$(jq -r '.status // empty' "$STATUS_RESPONSE")"
|
||||||
|
echo "$operation_name status check $attempt/30: $status"
|
||||||
|
case "$status" in
|
||||||
|
Succeeded)
|
||||||
|
return 0
|
||||||
|
;;
|
||||||
|
InProgress)
|
||||||
|
;;
|
||||||
|
Failed)
|
||||||
|
echo "::error::Microsoft Edge Add-ons $operation_name failed"
|
||||||
|
jq . "$STATUS_RESPONSE"
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "::error::Microsoft Edge Add-ons returned an unknown $operation_name status: $status"
|
||||||
|
jq . "$STATUS_RESPONSE"
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
|
||||||
|
echo "::error::Microsoft Edge Add-ons $operation_name did not complete after 30 checks"
|
||||||
|
jq . "$STATUS_RESPONSE"
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
HTTP_STATUS="$(curl --silent --show-error \
|
||||||
|
--output "$UPLOAD_RESPONSE" \
|
||||||
|
--dump-header "$UPLOAD_HEADERS" \
|
||||||
|
--write-out '%{http_code}' \
|
||||||
|
--request POST \
|
||||||
|
--header "Authorization: ApiKey $API_KEY" \
|
||||||
|
--header "X-ClientID: $CLIENT_ID" \
|
||||||
|
--header "Content-Type: application/zip" \
|
||||||
|
--data-binary "@$ARCHIVE" \
|
||||||
|
"$API_ROOT/products/$PRODUCT_ID/submissions/draft/package")"
|
||||||
|
if [ "$HTTP_STATUS" != "202" ]; then
|
||||||
|
echo "::error::Microsoft Edge Add-ons upload failed with HTTP $HTTP_STATUS"
|
||||||
|
jq . "$UPLOAD_RESPONSE" || cat "$UPLOAD_RESPONSE"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
UPLOAD_OPERATION_ID="$(operation_id_from_headers "$UPLOAD_HEADERS")"
|
||||||
|
if [ -z "$UPLOAD_OPERATION_ID" ]; then
|
||||||
|
echo "::error::Microsoft Edge Add-ons upload response has no Location operation ID"
|
||||||
|
cat "$UPLOAD_HEADERS"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
poll_operation \
|
||||||
|
"package upload" \
|
||||||
|
"$API_ROOT/products/$PRODUCT_ID/submissions/draft/package/operations/$UPLOAD_OPERATION_ID"
|
||||||
|
|
||||||
|
HTTP_STATUS="$(curl --silent --show-error \
|
||||||
|
--output "$PUBLISH_RESPONSE" \
|
||||||
|
--dump-header "$PUBLISH_HEADERS" \
|
||||||
|
--write-out '%{http_code}' \
|
||||||
|
--request POST \
|
||||||
|
--header "Authorization: ApiKey $API_KEY" \
|
||||||
|
--header "X-ClientID: $CLIENT_ID" \
|
||||||
|
--header "Content-Type: application/json" \
|
||||||
|
--data "{\"notes\":\"Automated BrowserSkill extension v${RELEASE_VERSION} release from GitHub Actions.\"}" \
|
||||||
|
"$API_ROOT/products/$PRODUCT_ID/submissions")"
|
||||||
|
if [ "$HTTP_STATUS" != "202" ]; then
|
||||||
|
echo "::error::Microsoft Edge Add-ons publish failed with HTTP $HTTP_STATUS"
|
||||||
|
jq . "$PUBLISH_RESPONSE" || cat "$PUBLISH_RESPONSE"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
PUBLISH_OPERATION_ID="$(operation_id_from_headers "$PUBLISH_HEADERS")"
|
||||||
|
if [ -z "$PUBLISH_OPERATION_ID" ]; then
|
||||||
|
echo "::error::Microsoft Edge Add-ons publish response has no Location operation ID"
|
||||||
|
cat "$PUBLISH_HEADERS"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
poll_operation \
|
||||||
|
"submission" \
|
||||||
|
"$API_ROOT/products/$PRODUCT_ID/submissions/operations/$PUBLISH_OPERATION_ID"
|
||||||
|
|
||||||
|
echo "Microsoft Edge Add-ons submission created successfully:"
|
||||||
|
jq '{status, message, errorCode, errors}' "$STATUS_RESPONSE"
|
||||||
|
|||||||
@@ -7,6 +7,10 @@ dist/
|
|||||||
.output/
|
.output/
|
||||||
.wxt/
|
.wxt/
|
||||||
.web-extension-id
|
.web-extension-id
|
||||||
|
.pnpm-store/
|
||||||
|
|
||||||
|
# Git worktrees
|
||||||
|
.worktrees/
|
||||||
|
|
||||||
# Logs
|
# Logs
|
||||||
*.log
|
*.log
|
||||||
|
|||||||
@@ -247,6 +247,41 @@ describe("App", () => {
|
|||||||
expect(copyButton.getAttribute("disabled")).not.toBeNull();
|
expect(copyButton.getAttribute("disabled")).not.toBeNull();
|
||||||
expect(screen.getByText("连接后可用")).toBeTruthy();
|
expect(screen.getByText("连接后可用")).toBeTruthy();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("treats protocol drift as still connected and prompts an upgrade", async () => {
|
||||||
|
mockUseConnectionState.mockReturnValue({
|
||||||
|
snapshot: {
|
||||||
|
...baseSnapshot,
|
||||||
|
state: "version_skew",
|
||||||
|
instanceId: "03c3e47f",
|
||||||
|
handshake: {
|
||||||
|
server: "bh",
|
||||||
|
version: mockDaemonVersion,
|
||||||
|
protocol_version: "1.0",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
statusState: "version_skew",
|
||||||
|
setLabel,
|
||||||
|
setConnectionEnabled,
|
||||||
|
});
|
||||||
|
|
||||||
|
render(<App />);
|
||||||
|
|
||||||
|
expect(screen.getByText("已连接")).toBeTruthy();
|
||||||
|
expect(screen.getByText("可升级")).toBeTruthy();
|
||||||
|
expect(screen.queryByText("协议不一致")).toBeNull();
|
||||||
|
expect(screen.queryByText("Action needed")).toBeNull();
|
||||||
|
expect(screen.queryByText("兼容")).toBeNull();
|
||||||
|
const warning = screen.getByText(/协议版本不同,请及时升级/);
|
||||||
|
expect(warning.textContent).toContain("CLI 协议");
|
||||||
|
expect(warning.textContent).toContain("扩展协议");
|
||||||
|
|
||||||
|
openRecordView();
|
||||||
|
const copyButton = screen.getByRole("button", { name: "复制录制指令" });
|
||||||
|
expect(copyButton.getAttribute("disabled")).toBeNull();
|
||||||
|
fireEvent.click(copyButton);
|
||||||
|
expect(navigator.clipboard.writeText).toHaveBeenCalledTimes(1);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("control hints toggle", () => {
|
describe("control hints toggle", () => {
|
||||||
|
|||||||
@@ -79,6 +79,7 @@ export function App() {
|
|||||||
}, [copiedTick]);
|
}, [copiedTick]);
|
||||||
|
|
||||||
const isSkewed = statusState === "version_skew";
|
const isSkewed = statusState === "version_skew";
|
||||||
|
const connectionLive = statusState === "connected" || isSkewed;
|
||||||
const daemonVersion = snapshot.handshake?.version ?? "—";
|
const daemonVersion = snapshot.handshake?.version ?? "—";
|
||||||
const daemonProtocol = snapshot.handshake?.protocol_version ?? "—";
|
const daemonProtocol = snapshot.handshake?.protocol_version ?? "—";
|
||||||
const extensionVersion = snapshot.extensionVersion || "—";
|
const extensionVersion = snapshot.extensionVersion || "—";
|
||||||
@@ -90,7 +91,7 @@ export function App() {
|
|||||||
setCopiedInstanceId(true);
|
setCopiedInstanceId(true);
|
||||||
};
|
};
|
||||||
|
|
||||||
const recordReady = statusState === "connected" && Boolean(snapshot.instanceId);
|
const recordReady = connectionLive && Boolean(snapshot.instanceId);
|
||||||
const recordPurpose = purposeDraft.trim();
|
const recordPurpose = purposeDraft.trim();
|
||||||
const recordStartUrl = startUrlDraft.trim();
|
const recordStartUrl = startUrlDraft.trim();
|
||||||
const recordCommand = snapshot.instanceId
|
const recordCommand = snapshot.instanceId
|
||||||
@@ -206,12 +207,12 @@ export function App() {
|
|||||||
</div>
|
</div>
|
||||||
{isSkewed && (
|
{isSkewed && (
|
||||||
<p
|
<p
|
||||||
className="mt-2 text-xs leading-snug text-amber-600 dark:text-amber-400"
|
className="mt-2 text-xs leading-snug text-muted-foreground"
|
||||||
data-slot="popup-version-skew-warning"
|
data-slot="popup-version-skew-warning"
|
||||||
>
|
>
|
||||||
{t("popup.versionSkewWarning", {
|
{t("popup.versionSkewWarning", {
|
||||||
extensionProtocol: PROTOCOL_VERSION,
|
extensionProtocol: PROTOCOL_VERSION,
|
||||||
daemonProtocol,
|
cliProtocol: daemonProtocol,
|
||||||
})}
|
})}
|
||||||
</p>
|
</p>
|
||||||
)}
|
)}
|
||||||
|
|||||||
+98
-283
@@ -1,327 +1,142 @@
|
|||||||
---
|
---
|
||||||
name: browser-skill
|
name: browser-skill
|
||||||
description: |
|
description: |
|
||||||
Use when the user asks to perform browser automation tasks against their
|
Use when the user asks to automate their logged-in Chromium browser: visit
|
||||||
logged-in browser: visit and read pages, fill forms, scrape data, click
|
and read pages, fill forms, scrape data, click through flows, regression-test
|
||||||
through a flow, regression-test a PR's UI, validate a deployed page.
|
a PR's UI, validate a deployed page, or operate a tab they identify. Requires
|
||||||
Requires the bsk CLI installed and the browser-skill extension loaded.
|
the bsk CLI and browser extension.
|
||||||
---
|
---
|
||||||
|
|
||||||
# browser-skill
|
# browser-skill
|
||||||
|
|
||||||
Drive the user's **real Chromium browser** (with their logins and cookies) through the `bsk` CLI. The extension opens an isolated **Agent Window** for automation; the user's normal windows stay protected unless you explicitly borrow a tab.
|
Drive the user's real Chromium browser through `bsk`. Automation runs in an isolated **Agent
|
||||||
|
Window** with the user's existing logins and cookies. User-window tabs remain protected unless they
|
||||||
|
are explicitly borrowed.
|
||||||
|
|
||||||
## When to use
|
Do not use this skill for tasks with no browser, for extension installation, or when the user only
|
||||||
|
wants instructions. Never extract credentials, cookies, tokens, or other secrets from pages.
|
||||||
|
|
||||||
- Open pages, read titles/text, scrape structured data from sites the user can already access
|
## Required lifecycle
|
||||||
- Fill forms, click through multi-step flows, smoke-test a UI change
|
|
||||||
- Understand pages with `bsk snapshot` first; use `bsk get-html` or `bsk screenshot` only when the snapshot is insufficient
|
|
||||||
- Operate on a specific user tab they point you at (after `bsk tab borrow`)
|
|
||||||
|
|
||||||
## When NOT to use
|
Every browser task owns a bounded session:
|
||||||
|
|
||||||
- Tasks with **no browser** involved (files, APIs, databases only)
|
```text
|
||||||
- Installing or configuring the extension (point the user to setup docs instead)
|
1. bsk session start # retain the printed 4-letter session id
|
||||||
- **Credential harvesting** — never run `bsk evaluate` on banking, SSO, or password-manager pages to extract tokens, cookies, or secrets
|
2. bsk ... --session <id> # pass it to every session-scoped command
|
||||||
- Long-lived control of a user's personal login window — borrow only for the immediate step, then `bsk tab return` or end the session
|
3. bsk session stop <id> # always run on success and error paths
|
||||||
- Replacing the user's manual browsing when they only wanted an explanation
|
|
||||||
|
|
||||||
## Prerequisites
|
|
||||||
|
|
||||||
1. `bsk` on `PATH` (Rust CLI from browser-skill)
|
|
||||||
2. browser-skill **extension** loaded in Chromium and connected (popup shows green)
|
|
||||||
3. Any `bsk` command auto-starts background services as needed; use `bsk doctor` if anything fails
|
|
||||||
|
|
||||||
## Mandatory workflow
|
|
||||||
|
|
||||||
Every automation task **must** follow this lifecycle. Do **not** rely on idle timeouts (default session idle is 5 minutes).
|
|
||||||
|
|
||||||
```
|
|
||||||
1. bsk session start → capture the 4-letter session id printed on stdout
|
|
||||||
2. … every tool command … → always pass --session <id>
|
|
||||||
3. bsk session stop <id> → REQUIRED when done (even on error paths)
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Optional: `bsk session start --browser <instance-id-or-label>` when multiple browsers are connected (`bsk browsers` / error output lists them). Add `--no-focus` to open the Agent Window in the background without stealing focus from the user's current window.
|
Do not rely on the idle timeout for cleanup. Stop the session as soon as the goal is met unless the
|
||||||
|
user explicitly asks to keep it open. Stopping also returns borrowed tabs.
|
||||||
|
|
||||||
Emergency cleanup: `bsk session stop --all` or the Agent Window overlay **Stop all**.
|
Any `bsk` command auto-starts the background services it needs; never manage the daemon by hand.
|
||||||
|
When multiple browsers are connected, use `bsk browsers` and start with
|
||||||
|
`bsk session start --browser <id-or-label>`. Add `--no-focus` to that same start command when the
|
||||||
|
Agent Window does not need to interrupt the user's current work; it is not a flag on other commands.
|
||||||
|
Run `bsk doctor` when startup or transport problems persist after one retry.
|
||||||
|
|
||||||
## Stop when the goal is met
|
## Work toward one observable goal
|
||||||
|
|
||||||
Every task is a **bounded goal**, not open-ended browsing. The goal may come from the user's request, a recorded `trace.json`, or both.
|
- Derive a concrete success condition from the user's request or a supplied trace.
|
||||||
|
- Take the shortest purposeful path: observe, act, then make at most one observation to confirm an
|
||||||
|
ambiguous result.
|
||||||
|
- Once success is visible, do not click, refresh, navigate, switch tabs, or perform extra checks.
|
||||||
|
- If a human-only step appears or two attempts make no progress, request help instead of
|
||||||
|
brute-forcing.
|
||||||
|
|
||||||
1. **Define success first** — one concrete, observable condition derived from the user's words, `purpose`, or the last meaningful step in a trace (e.g. "form submitted", "item added to cart", "playback started").
|
With a trace, follow its semantic target information and values in order, but treat its refs as
|
||||||
2. **Take the shortest path** — snapshot → act → at most one check. Do not wander, re-try unrelated actions, or stack exploratory steps.
|
record-local hints. Stop when its purpose or last meaningful effect is satisfied. A trace guides the
|
||||||
3. **Stop as soon as success is reached** — run `bsk session stop <id>` immediately unless the user explicitly asked to keep the session open (e.g. "don't close yet", "keep browsing").
|
task; it does not expand the user's goal or authorize additional actions.
|
||||||
4. **No post-success work** — once the goal is met, do not click, refresh, navigate, re-search, switch tabs, or "double-check" that it worked. Further verification is a new task.
|
|
||||||
5. **When blocked, pause — do not brute-force** — if the page requires human input (login, captcha, OTP, payment confirmation) or an action fails twice with no progress, call `bsk request-help` instead of retrying blindly. See **Ask the human for help** below.
|
|
||||||
6. **When unsure** — at most one extra `bsk snapshot`. If success looks met, stop. If not, ask the user; do not keep clicking.
|
|
||||||
|
|
||||||
**With a trace:** replay steps in order using `target` role/name/tag and raw `value`/`selection` fields. After the last step (or when its `effect.navigated_to` / success hint is satisfied), apply rules 3–4 immediately. The trace guides execution; it does not extend control beyond the goal.
|
## Observe, act, observe
|
||||||
|
|
||||||
**Without a trace:** the user's request *is* the success condition. Satisfying it ends the task — same stop rules apply.
|
Use this default loop:
|
||||||
|
|
||||||
## Core interaction loop
|
```text
|
||||||
|
|
||||||
Write operations only affect tabs in the **Agent Window** (or tabs you **borrowed** into it).
|
|
||||||
|
|
||||||
```
|
|
||||||
bsk navigate <url> --session <id>
|
bsk navigate <url> --session <id>
|
||||||
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
bsk observe --session <id>
|
||||||
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
bsk click|hover|fill|select|press ... --session <id>
|
||||||
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
bsk observe --session <id> # after navigation or a meaningful DOM change
|
||||||
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
|
||||||
bsk observe --session <id> → again after navigation / DOM change
|
|
||||||
```
|
```
|
||||||
|
|
||||||
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also
|
||||||
|
make them stale. Observe again before the next interaction.
|
||||||
|
|
||||||
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`.
|
||||||
|
The listed items are labels, not usable refs: hover the trigger, observe again, then act on the
|
||||||
|
revealed item's own ref. Do not click the trigger itself unless the user wants the trigger's action.
|
||||||
|
|
||||||
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
Escalate page reading only as needed:
|
||||||
|
|
||||||
## Observation priority
|
1. `bsk observe` for normal semantic understanding, text, controls, and refs.
|
||||||
|
2. `bsk snapshot` when a stricter static accessibility tree is more useful.
|
||||||
|
3. `bsk get-html` for exact markup or hidden metadata that semantic views cannot provide.
|
||||||
|
4. `bsk screenshot` for layout, styling, canvas, images, or requested visual evidence.
|
||||||
|
|
||||||
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
Do not start with raw HTML or screenshots merely to discover ordinary controls. When interaction is
|
||||||
|
needed, obtain a fresh observation before acting on screenshot or HTML findings.
|
||||||
|
|
||||||
1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
## Respect the Agent Window boundary
|
||||||
2. `bsk snapshot` — strict static accessibility tree fallback
|
|
||||||
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
|
|
||||||
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
|
|
||||||
|
|
||||||
Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page.
|
Normal page writes affect only Agent Window tabs. To operate a user tab, first list it with
|
||||||
|
`bsk tab list --scope user --session <id>`, then `bsk tab borrow <tab-id>`. Return it immediately
|
||||||
|
after the relevant step with `bsk tab return <tab-id>`; never invent a tab id or keep a personal tab
|
||||||
|
borrowed across unrelated work.
|
||||||
|
|
||||||
## Sandbox rules
|
## Ask the human when needed
|
||||||
|
|
||||||
| Rule | Detail |
|
Use `bsk request-help` for login, captcha, OTP, payment confirmation, consent, or another step the
|
||||||
|------|--------|
|
user must complete. Give a precise prompt and pass fresh `--target` refs/selectors when concrete
|
||||||
| Agent Window | `bsk tab create`, `bsk navigate`, `bsk click`, etc. work on agent tabs by default |
|
controls can be highlighted. Use completion criteria only when the page has a clear stable success
|
||||||
| User tabs | Read-only until borrowed: `bsk tab list --session <id> --scope user` then `bsk tab borrow <tab-id> --session <id>` |
|
signal.
|
||||||
| Return borrowed tabs | Call `bsk tab return <tab-id> --session <id>` when finished; unreturned tabs are **auto-returned** on `bsk session stop` |
|
|
||||||
| Writes off-agent | Commands that mutate the page fail if the tab is not in the Agent Window — borrow or create a tab first |
|
|
||||||
|
|
||||||
## Global flags
|
The result `outcome` is one of `continued`, `completed`, `cancelled`, `timed_out`, or `disabled`
|
||||||
|
(`navigated` is deprecated — never treat navigation as a completion signal). Resume only after
|
||||||
|
`continued` or `completed`. Treat `cancelled` as rejection, and `timed_out` or `disabled` as a
|
||||||
|
blocker rather than a reason to retry. After control returns, run a fresh `bsk observe` before
|
||||||
|
reasoning about the page or using refs.
|
||||||
|
|
||||||
| Flag | Purpose |
|
## Command inventory
|
||||||
|------|---------|
|
|
||||||
| `--json` | Machine-readable JSON on stdout (errors too) |
|
|
||||||
| `--quiet` | Suppress informational stderr |
|
|
||||||
| `-v` / `-vv` | More verbose logging |
|
|
||||||
|
|
||||||
Auto-update is **on by default** — the background daemon upgrades `bsk` itself when a new release is available (postponed while a session is active). Set `BSK_AUTO_UPDATE=off` to disable it and upgrade manually with `bsk update`.
|
This list of names is complete. Never invent a command outside it; read
|
||||||
|
`bsk <command...> --help` for flags instead of guessing them.
|
||||||
|
|
||||||
Command-specific flags (timeouts, `--tab-id`, `--wait-until`, …): **`bsk <cmd> --help`**
|
```text
|
||||||
|
session start|stop|list browsers status doctor update logs
|
||||||
## CLI command reference (one line each)
|
navigate navigate-back navigate-forward reload wait-for-navigation wait-ms
|
||||||
|
observe snapshot get-html screenshot console network
|
||||||
Details and flags: **`bsk <cmd> --help`**
|
click hover fill select press evaluate
|
||||||
|
tab list|create|close|select|borrow|return window resize emulate
|
||||||
### Diagnostics
|
request-help record start|stop
|
||||||
|
|
||||||
| Command | Summary |
|
|
||||||
|---------|---------|
|
|
||||||
| `bsk status` | Connection health, connected browsers, active sessions |
|
|
||||||
| `bsk doctor` | Deep diagnostics and repair hints |
|
|
||||||
| `bsk browsers` | List connected browser instances (ids, labels, versions) |
|
|
||||||
|
|
||||||
### Session
|
|
||||||
|
|
||||||
| Command | Summary |
|
|
||||||
|---------|---------|
|
|
||||||
| `bsk session start` | Open Agent Window (`--width`/`--height` for initial size); prints **4-letter session id** |
|
|
||||||
| `bsk session start --no-focus` | Open Agent Window in the background without stealing focus |
|
|
||||||
| `bsk session stop <id>` | End session, close Agent Window, auto-return borrowed tabs |
|
|
||||||
| `bsk session stop --all` | Stop every active session |
|
|
||||||
| `bsk session list` | List active sessions |
|
|
||||||
|
|
||||||
### Window (require `--session <id>`)
|
|
||||||
|
|
||||||
| Command | Summary |
|
|
||||||
|---------|---------|
|
|
||||||
| `bsk window resize` | Resize the Agent Window (`--width`, `--height`; 100..=7680 CSS px) |
|
|
||||||
|
|
||||||
### Device emulation — `bsk emulate` (requires `--session <id>`)
|
|
||||||
|
|
||||||
Emulate a mobile device environment on the agent tab via CDP — viewport, User-Agent, and touch — to debug a page's mobile layout and behaviour:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
bsk emulate --session <id> --device iphone-14
|
|
||||||
bsk emulate --session <id> --width 390 --height 844 --dpr 3 --mobile --ua "Mozilla/5.0 (iPhone…" --touch
|
|
||||||
bsk emulate --session <id> --off
|
|
||||||
```
|
```
|
||||||
|
|
||||||
- Presets (`--device`): `iphone-14`, `iphone-14-pro-max`, `iphone-se`, `pixel-7`, `galaxy-s23`, `ipad-mini`, `galaxy-tab-s8`. Manual flags (`--width`/`--height`/`--dpr`/`--mobile`/`--ua`/`--accept-language`/`--touch`/`--max-touch-points`) also work without a preset, or override individual preset fields; `--no-mobile`/`--no-touch` turn a preset's mobile viewport / touch emulation back off.
|
Required flags that are easy to get wrong:
|
||||||
- Repeated runs merge field by field onto the tab's current emulation state — only the flags you pass change. E.g. after `--device iphone-14`, `bsk emulate --session <id> --width 390 --height 844` keeps the preset's dpr (`3`) and mobile viewport. (The extension remembers the state per tab; after an extension reload the next run applies only the fields it carries.)
|
|
||||||
- `--off` clears every override (viewport, UA, touch) and the remembered state, restoring the tab's real environment.
|
|
||||||
- Scope: overrides apply to **one tab only** (CDP per-target) and are **not inherited by new tabs** — re-run `bsk emulate` after opening or switching to another tab (default target is the session's active tab; `--tab-id` overrides).
|
|
||||||
- Emulation covers viewport/UA/touch only; it does not throttle the network, fake geolocation, or synthesise real touch-event streams.
|
|
||||||
|
|
||||||
### Tabs (require `--session <id>`)
|
```text
|
||||||
|
bsk fill <ref> --value <text> bsk select <ref> --value <option-value>
|
||||||
| Command | Summary |
|
bsk screenshot --out <path> bsk emulate --device <preset-id>
|
||||||
|---------|---------|
|
|
||||||
| `bsk tab list` | List tabs (`--scope user\|agent\|all`, default `all`) |
|
|
||||||
| `bsk tab create` | New tab in Agent Window (`--url`, `--no-active`, `--index`) |
|
|
||||||
| `bsk tab close <tab-id>` | Close an agent tab |
|
|
||||||
| `bsk tab select <tab-id>` | Focus an agent tab |
|
|
||||||
| `bsk tab borrow <tab-id>` | Move a user tab into the Agent Window |
|
|
||||||
| `bsk tab return <tab-id>` | Return a borrowed tab to its original window |
|
|
||||||
|
|
||||||
### Observation (require `--session` unless noted)
|
|
||||||
|
|
||||||
| Command | Summary |
|
|
||||||
|---------|---------|
|
|
||||||
| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
|
|
||||||
| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
|
|
||||||
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
|
|
||||||
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
|
|
||||||
|
|
||||||
### Console & network debugging (read-only; require `--session`)
|
|
||||||
|
|
||||||
| Command | Summary |
|
|
||||||
|---------|---------|
|
|
||||||
| `bsk console` | Buffered page console messages, JS exceptions, and browser log entries (`--include-stack` for stack traces) |
|
|
||||||
| `bsk network` | Buffered network responses (status, method, URL, MIME/resource type) and failures (`net::ERR_*` reason) |
|
|
||||||
|
|
||||||
Both capture from the moment the tab is attached and read a bounded per-tab buffer: `--since <seq>` pages from a cursor (`next_since` in the result), `--limit` (default 50, max 200), `--max-text-chars` (default 1000, max 4096), `--tab-id` to target a non-active tab. Both are strictly read-only — they never intercept or modify traffic, and request/response headers, bodies, and timings are not captured.
|
|
||||||
|
|
||||||
### Navigation
|
|
||||||
|
|
||||||
| Command | Summary |
|
|
||||||
|---------|---------|
|
|
||||||
| `bsk navigate <url>` | Go to URL in agent tab (`--wait-until`, `--timeout`) |
|
|
||||||
| `bsk navigate-back` | History back one step |
|
|
||||||
| `bsk navigate-forward` | History forward one step |
|
|
||||||
| `bsk reload` | Reload current tab (`--hard` bypass cache) |
|
|
||||||
|
|
||||||
(`bsk navigate back` / `bsk navigate forward` are equivalent subcommands.)
|
|
||||||
|
|
||||||
### Interaction
|
|
||||||
|
|
||||||
| Command | Summary |
|
|
||||||
|---------|---------|
|
|
||||||
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
|
||||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
|
||||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
|
||||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
|
||||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
|
||||||
|
|
||||||
### Scripting & timing
|
|
||||||
|
|
||||||
| Command | Summary |
|
|
||||||
|---------|---------|
|
|
||||||
| `bsk evaluate <expression>` | Run JS in agent tab (see red lines). JS throw → stderr, **exit 0** (RPC success); use `--json` and check `.ok` to detect JS errors |
|
|
||||||
| `bsk wait-for-navigation` | Block until load/DOM idle/etc. (`--wait-until`, `--timeout`) |
|
|
||||||
| `bsk wait-ms <duration>` | Sleep (`500ms`, `2s`, `1m`; **no** `--session`) |
|
|
||||||
|
|
||||||
### Ask the human for help — `bsk request-help`
|
|
||||||
|
|
||||||
When a step needs a human (captcha, login, OTP) or you want the user to
|
|
||||||
confirm an important action, pause and ask:
|
|
||||||
|
|
||||||
bsk request-help --session <id> --prompt "Solve the captcha, then click Done only after the site accepts it" \
|
|
||||||
--title "Captcha required" --target @e7 --target "#submit" --timeout 5m
|
|
||||||
|
|
||||||
- `--prompt` (required): what the user should do.
|
|
||||||
- `--title` (optional): custom title for the overlay panel. When omitted,
|
|
||||||
the extension shows its default localized title.
|
|
||||||
- `--target` (repeatable): a snapshot ref (`@e7`) or CSS selector
|
|
||||||
(`#submit`) to scroll to and flash-highlight. **Strongly recommended** —
|
|
||||||
whenever the prompt refers to a concrete element (a button to click, a
|
|
||||||
field to fill, a checkbox to toggle), pass its `@eN` ref / selector so the
|
|
||||||
user is guided straight to the right spot instead of hunting for it. For
|
|
||||||
interaction scenarios, always include the relevant target(s); reserve a
|
|
||||||
prompt with no `--target` for cases where there is genuinely no specific
|
|
||||||
element to point at (e.g. "wait for the page to finish loading").
|
|
||||||
- `--timeout` (default `5m`): how long to wait.
|
|
||||||
- `--completion-criteria` (optional): JSON success detector. Use it only
|
|
||||||
when there is a concrete post-help success signal, e.g.
|
|
||||||
`{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`.
|
|
||||||
|
|
||||||
The target tab is brought to the foreground; the page stays interactive
|
|
||||||
while the agent control mask is hidden. The call blocks until the user
|
|
||||||
explicitly acts, the timeout expires, cancellation arrives, or explicit
|
|
||||||
completion criteria match. Page reloads, SPA route changes, and captcha
|
|
||||||
refreshes do not return control by themselves. The result `outcome` is one of:
|
|
||||||
|
|
||||||
- `continued` — the user finished and clicked Done / return control (treat as confirm).
|
|
||||||
- `cancelled` — the user clicked Cancel (treat as reject/abort).
|
|
||||||
- `timed_out` — nobody acted within the timeout.
|
|
||||||
- `completed` — the explicit `--completion-criteria` matched while the user had control.
|
|
||||||
- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal.
|
|
||||||
|
|
||||||
`note` carries any text the user typed back. `resolved_targets` reports
|
|
||||||
which refs/selectors matched a live element.
|
|
||||||
|
|
||||||
`request-help` does not refresh the page model after the user returns
|
|
||||||
control. After a `continued` or `completed` result, issue a separate
|
|
||||||
observation tool call (usually `bsk snapshot --session <id>`) before using
|
|
||||||
new refs or reasoning about the post-help page state.
|
|
||||||
#### Disabling request-help (unattended mode)
|
|
||||||
Set `BSK_REQUEST_HELP=off` on unattended servers: `bsk request-help` then
|
|
||||||
returns immediately with `outcome="disabled"` (no overlay, no waiting,
|
|
||||||
exit 0). Any other value keeps it enabled. If you get `disabled`, do not
|
|
||||||
retry — complete the task autonomously or stop gracefully.
|
|
||||||
|
|
||||||
### Recording — `bsk record`
|
|
||||||
|
|
||||||
Capture the user's own actions in the Agent Window to a **trace bundle**, for later LLM-driven automation. New CLI builds request **trace v3** (page observations + action chain); older extensions may still return **trace v2** (actions only), which the CLI exports as a single `trace.json` without `states/`.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
bsk record start --browser <instance-id-or-label> \
|
|
||||||
[--url https://…] [--purpose "publish a wiki doc"] \
|
|
||||||
[--max-page-tokens 3000] [--redact-values] \
|
|
||||||
[--output trace]
|
|
||||||
# `--url` is optional; default https://example.com/ when omitted (must be http(s)).
|
|
||||||
# Blocks until the user clicks Finish in the recording panel, then writes:
|
|
||||||
# trace/trace.json — action chain (+ state index when v3)
|
|
||||||
# trace/states/ — v3 only: one `sN.txt` observe snapshot per settled page state
|
|
||||||
|
|
||||||
bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable
|
|
||||||
```
|
```
|
||||||
|
|
||||||
- **v3 bundle (preferred):** `--output` is a directory (default `./trace`) containing `trace.json` and `states/`. `trace.json` lists `states[]` (page observation ids) and `steps[]` (each step binds `state` = observe snapshot *before* the action and `result.state` = snapshot *after* settle). Page bodies live in `states/sN.txt` using the same VOM format as `bsk observe`.
|
`select` matches an option's `value` attribute, not its visible label. Device preset ids are
|
||||||
- **v2 fallback:** when the connected extension is older, export may contain only legacy `trace.json` with `pages[]` action context and no `states/` observation files. Update the extension for full v3 bundles.
|
lowercase and hyphenated, such as `iphone-14`.
|
||||||
- Each `states[]` entry is one **settled page observation** (captured after recording start, navigation landing, or action settle — not on a timer).
|
|
||||||
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
|
|
||||||
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
|
||||||
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
|
|
||||||
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
|
||||||
|
|
||||||
## Error handling
|
- `console` and `network` provide bounded, read-only debugging evidence.
|
||||||
|
- `emulate` applies viewport, user-agent, and touch overrides to one tab; new tabs do not inherit
|
||||||
|
them. Use `--off` to restore the real environment.
|
||||||
|
- `evaluate` is a last resort when observe plus normal interactions cannot complete the task. With
|
||||||
|
`--json`, inspect `.ok`: a JavaScript exception may still have CLI exit code 0 because the RPC
|
||||||
|
succeeded. Never evaluate credential surfaces to read storage, cookies, or auth data.
|
||||||
|
- `record` captures a user's actions for later replay. Read `bsk record start --help` before use,
|
||||||
|
and never record banking, SSO, password-manager, or other sensitive pages.
|
||||||
|
|
||||||
### Exit codes (`echo $?` after `bsk …`)
|
## Recover without wandering
|
||||||
|
|
||||||
| Code | Meaning | What to do |
|
- Stale ref: observe again and retry the intended action once.
|
||||||
|------|---------|------------|
|
- Unknown tab or session: list current tabs/sessions; never guess identifiers.
|
||||||
| `0` | Success (including `evaluate` where JS threw but RPC succeeded) | Continue; for `evaluate`, check `--json` `.ok` to distinguish JS success from RPC success |
|
- Timeout: inspect current page state before deciding whether one longer purposeful wait is useful.
|
||||||
| `1` | User error — bad args, unknown session, tab not in Agent Window, stale ref | Fix args; `bsk session list`; re-snapshot |
|
- Unsupported command: continue with available capabilities; suggest updating only when the missing
|
||||||
| `2` | Protocol / transport — service unreachable, IPC failure | `bsk doctor`; check extension connected; retry the command |
|
command is necessary.
|
||||||
| `3` | Browser / CDP execution failed | Retry; simplify selector; check tab still open |
|
- Unrecoverable failure: report the blocker and stop the session in a finally-style path.
|
||||||
| `4` | Timeout | Increase `--timeout`; try `--wait-until domcontentloaded` |
|
|
||||||
| `5` | Version skew (CLI vs extension) | Upgrade/reinstall matching versions |
|
|
||||||
|
|
||||||
Human errors print `error:` + `hint:` on stderr; `--json` includes `code`, `message`, `hint`, `exit_code`.
|
The CLI's current help and error hints are authoritative for flags, parameters, and recovery
|
||||||
|
details.
|
||||||
### When to run diagnostics
|
|
||||||
|
|
||||||
| Situation | Command |
|
|
||||||
|-----------|---------|
|
|
||||||
| Before first task in a session | `bsk status` — extension connected? |
|
|
||||||
| Any failure you cannot fix in one retry | `bsk doctor` |
|
|
||||||
| Multiple browsers / wrong target | `bsk browsers` then `bsk session start --browser <id>` |
|
|
||||||
|
|
||||||
Always **`bsk session stop <id>`** in a `finally`-style path so the Agent Window closes and borrowed tabs return.
|
|
||||||
|
|
||||||
## Red lines
|
|
||||||
|
|
||||||
1. **No token theft** — do not `bsk evaluate` on sensitive sites to read `localStorage`, cookies, or auth headers for exfiltration.
|
|
||||||
2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks.
|
|
||||||
3. **No skip stop** — always `bsk session stop <id>`; never assume idle timeout will clean up.
|
|
||||||
4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open.
|
|
||||||
5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics or conditional surfaces help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual.
|
|
||||||
6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**More detail for any command:** `bsk <cmd> --help`
|
|
||||||
|
|||||||
@@ -351,10 +351,10 @@ fn check_version_compatible(status: Option<&StatusResult>) -> CheckResult {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// `bsk doctor` check: every connected browser should use the same
|
/// `bsk doctor` check: connected browsers should speak a protocol the
|
||||||
/// protocol version as the daemon. Minor protocol drift is accepted by
|
/// daemon accepts. A different protocol string is still a live
|
||||||
/// the daemon but flagged here so the user can see who needs updating
|
/// connection — report Ok so agents keep working. The detail is an
|
||||||
/// (M10.4).
|
/// upgrade reminder, not a blocker.
|
||||||
///
|
///
|
||||||
/// Review M2 (round-1 minor): when no browsers are connected, the
|
/// Review M2 (round-1 minor): when no browsers are connected, the
|
||||||
/// check has nothing to compare against, so it now reports
|
/// check has nothing to compare against, so it now reports
|
||||||
@@ -396,13 +396,12 @@ fn check_browsers_protocol_compatible(status: Option<&StatusResult>) -> CheckRes
|
|||||||
})
|
})
|
||||||
.collect::<Vec<_>>()
|
.collect::<Vec<_>>()
|
||||||
.join(", ");
|
.join(", ");
|
||||||
CheckResult::fail(
|
CheckResult::ok(
|
||||||
name,
|
name,
|
||||||
format!(
|
format!(
|
||||||
"{} browser(s) have protocol minor drift from the daemon: {stale}",
|
"{} browser(s) report a different protocol version (still usable — continue, and upgrade soon): {stale}",
|
||||||
status.version_skew_browsers.len()
|
status.version_skew_browsers.len()
|
||||||
),
|
),
|
||||||
"upgrade the browser-skill extension or bsk CLI so both sides use the same protocol version",
|
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -581,7 +580,7 @@ mod m2_tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn browsers_check_reports_fail_when_skew_present() {
|
fn browsers_check_reports_ok_when_skew_present() {
|
||||||
let status = fake_status(
|
let status = fake_status(
|
||||||
vec![BrowserStatusEntry {
|
vec![BrowserStatusEntry {
|
||||||
instance_id: "alpha".into(),
|
instance_id: "alpha".into(),
|
||||||
@@ -605,9 +604,17 @@ mod m2_tests {
|
|||||||
}],
|
}],
|
||||||
);
|
);
|
||||||
let check = check_browsers_protocol_compatible(Some(&status));
|
let check = check_browsers_protocol_compatible(Some(&status));
|
||||||
assert_eq!(check.status, CheckStatus::Fail);
|
assert_eq!(check.status, CheckStatus::Ok);
|
||||||
assert!(check.detail.contains("alpha"));
|
assert!(check.detail.contains("alpha"));
|
||||||
assert!(check.hint.is_some());
|
assert!(
|
||||||
|
check
|
||||||
|
.detail
|
||||||
|
.contains("still usable — continue, and upgrade soon")
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
check.hint.is_none(),
|
||||||
|
"a protocol-version note must not hint to stop"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -635,7 +642,7 @@ mod m2_tests {
|
|||||||
}],
|
}],
|
||||||
);
|
);
|
||||||
let check = check_browsers_protocol_compatible(Some(&status));
|
let check = check_browsers_protocol_compatible(Some(&status));
|
||||||
assert_eq!(check.status, CheckStatus::Fail);
|
assert_eq!(check.status, CheckStatus::Ok);
|
||||||
assert!(
|
assert!(
|
||||||
check
|
check
|
||||||
.detail
|
.detail
|
||||||
|
|||||||
@@ -65,7 +65,7 @@ fn render_human(s: &StatusResult) {
|
|||||||
let bold = "\x1b[1m";
|
let bold = "\x1b[1m";
|
||||||
let reset = "\x1b[0m";
|
let reset = "\x1b[0m";
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"{bold}{yellow}warning:{reset} {} browser(s) have protocol version drift from the daemon{reset}",
|
"{bold}{yellow}note:{reset} {} browser(s) report a different protocol version; still usable — continue, and run `bsk update` soon{reset}",
|
||||||
s.version_skew_browsers.len()
|
s.version_skew_browsers.len()
|
||||||
);
|
);
|
||||||
for skew in &s.version_skew_browsers {
|
for skew in &s.version_skew_browsers {
|
||||||
@@ -75,7 +75,7 @@ fn render_human(s: &StatusResult) {
|
|||||||
skew.label.clone()
|
skew.label.clone()
|
||||||
};
|
};
|
||||||
eprintln!(
|
eprintln!(
|
||||||
" {} ({}) — protocol ext {} vs daemon {} (app ext v{}, daemon v{}) — please align protocol versions",
|
" {} ({}) — protocol ext {} vs CLI {} (app ext v{}, CLI v{}) — still usable; please upgrade",
|
||||||
skew.instance_id,
|
skew.instance_id,
|
||||||
label,
|
label,
|
||||||
display_protocol(&skew.client_protocol_version),
|
display_protocol(&skew.client_protocol_version),
|
||||||
|
|||||||
@@ -5,142 +5,99 @@ description: Browser automation through six injected domain tools.
|
|||||||
|
|
||||||
# browser-skill for DeepSeek Harness
|
# browser-skill for DeepSeek Harness
|
||||||
|
|
||||||
Drive the user's real Chromium browser through the six structured domain tools provided by this
|
Drive the user's logged-in Chromium through this plugin's structured browser tools. Automation is
|
||||||
plugin. The browser keeps the user's existing logins and cookies while isolating automation in an
|
isolated in an Agent Window; user-window tabs remain protected unless explicitly borrowed.
|
||||||
Agent Window. Every call must include the tool's `action` field; other fields depend on that action.
|
|
||||||
|
|
||||||
## Tool availability
|
Loading this skill reveals six tools for the rest of the conversation:
|
||||||
|
|
||||||
Loading this skill reveals the browser tools for the rest of the conversation. All browser work
|
- `browser_session` owns session lifecycle.
|
||||||
must use the injected tools directly. Do not invoke another process to control the browser. Direct
|
- `browser_page` handles navigation and lifecycle waits.
|
||||||
tool calls preserve session ownership, the live observation window, cancellation, attachments, and
|
- `browser_inspect` reads semantic, visual, console, and network state.
|
||||||
cleanup.
|
- `browser_interact` performs normal page interactions.
|
||||||
|
- `browser_tabs` manages Agent Window tabs and temporary user-tab borrowing.
|
||||||
|
- `browser_assist` handles human help, window size, and device emulation.
|
||||||
|
|
||||||
The six tools are:
|
Every call includes an `action`. Treat each loaded tool schema as authoritative for its actions and
|
||||||
|
parameters; do not guess fields. All browser work must use the injected tools directly so session
|
||||||
- `browser_session`: start, stop, or list plugin-owned sessions.
|
ownership, cancellation, attachments, observation UI, and cleanup remain intact. Do not invoke
|
||||||
- `browser_page`: navigate, back, forward, reload, or wait for navigation.
|
another process to control the browser.
|
||||||
- `browser_inspect`: observe, snapshot, html, screenshot, console, or network.
|
|
||||||
- `browser_interact`: click, hover, fill, select, or press.
|
|
||||||
- `browser_tabs`: list, create, select, close, borrow, or return tabs.
|
|
||||||
- `browser_assist`: resize the Agent Window, emulate a device, or request help from the user.
|
|
||||||
|
|
||||||
## Mandatory workflow
|
## Mandatory workflow
|
||||||
|
|
||||||
Every browser task follows this lifecycle:
|
Every task owns a bounded plugin session:
|
||||||
|
|
||||||
1. Call `browser_session` with `action: "start"` and retain its returned `sessionId`.
|
|
||||||
2. Use that session for every later call. Pass `session` explicitly when more than one exists.
|
|
||||||
3. Call `browser_session` with `action: "stop"` when the goal is reached or an error ends the task.
|
|
||||||
Treat cleanup like a finally path unless the user explicitly asks to keep the session open.
|
|
||||||
|
|
||||||
Conceptual sequence:
|
|
||||||
|
|
||||||
```text
|
```text
|
||||||
browser_session({ action: "start", url: "https://example.com" })
|
browser_session({ action: "start", ... })
|
||||||
browser_inspect({ action: "observe", session: sessionId })
|
... use the returned sessionId for browser work ...
|
||||||
browser_interact({ action: "click", session: sessionId, target: "@e4" })
|
|
||||||
browser_inspect({ action: "observe", session: sessionId })
|
|
||||||
browser_session({ action: "stop", session: sessionId })
|
browser_session({ action: "stop", session: sessionId })
|
||||||
```
|
```
|
||||||
|
|
||||||
The plugin owns only sessions it starts. Never guess or reuse a session id from another program.
|
Pass the session explicitly when more than one exists. Never guess or reuse an id owned by another
|
||||||
When `session` is omitted, most actions use the current owned session, but explicit ids are safer.
|
program. Stop in a finally-style path on success and failure unless the user explicitly asks to keep
|
||||||
|
the session open. Stopping also returns borrowed tabs.
|
||||||
|
|
||||||
## Stop when the goal is met
|
## Work toward one observable goal
|
||||||
|
|
||||||
Browser work is bounded. Define an observable success condition, take the shortest path, and stop
|
- Derive a concrete success condition from the user's request.
|
||||||
as soon as success is visible. Do not refresh or click after success. If an action fails twice
|
- Take the shortest purposeful path: observe, act, then make at most one observation to confirm an
|
||||||
without progress, pause instead of brute-forcing. If state is ambiguous, make at most one fresh
|
ambiguous result.
|
||||||
observation and then ask the user.
|
- Once success is visible, do not click, refresh, navigate, switch tabs, or perform extra checks.
|
||||||
|
- If a human-only step appears or two attempts make no progress, request help instead of
|
||||||
|
brute-forcing.
|
||||||
|
|
||||||
## Observe and interact
|
## Observe, act, observe
|
||||||
|
|
||||||
Use `browser_inspect` with `action: "observe"` as the primary semantic page view. It returns roles,
|
Use `browser_inspect` action `observe` as the primary semantic page view. It returns roles, states,
|
||||||
states, text, and `@eN` refs. Use `action: "snapshot"` when a static accessibility tree is more useful.
|
text, and `@eN` refs. Prefer fresh refs over raw selectors. Refs invalidate after navigation and may
|
||||||
Prefer a fresh ref over a raw CSS selector for interactions.
|
also become stale after large DOM changes, so observe again before the next interaction.
|
||||||
|
|
||||||
Refs invalidate after navigation and may become stale after large DOM changes. Observe again after
|
Use `browser_interact` for click, hover, fill, select, and key actions. An observation marks a
|
||||||
navigation or a meaningful DOM update before using another ref. If an observation identifies a
|
hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`. The listed items are
|
||||||
hover-triggered surface, hover the trigger, observe again, then interact with the revealed control.
|
labels, not usable refs: hover the trigger, observe again, then act on the revealed item's own ref.
|
||||||
|
Do not click the trigger itself unless the user wants the trigger's action.
|
||||||
|
|
||||||
Use `browser_interact` as follows:
|
Escalate reading only as needed:
|
||||||
|
|
||||||
- `action: "click"`: requires `target`; optionally accepts `button` and `clickCount`.
|
1. `observe` for normal understanding and interaction refs.
|
||||||
- `action: "hover"`: requires `target`; optionally accepts `modifiers`, `settleMs`, and `timeoutMs`.
|
2. `snapshot` when a stricter static accessibility tree is more useful.
|
||||||
- `action: "fill"`: requires `target` and `value`; `noClear` appends instead of replacing.
|
3. `html` for exact markup or hidden metadata that semantic views cannot provide.
|
||||||
- `action: "select"`: requires `target` and a non-empty `values` array. Values are option value
|
4. `screenshot` for layout, styling, canvas, images, or requested visual evidence.
|
||||||
attributes, not necessarily the visible labels.
|
|
||||||
- `action: "press"`: requires `key`; optionally focus `target` first and set `holdMs`.
|
|
||||||
|
|
||||||
## Navigate and wait
|
Do not start with raw HTML or screenshots merely to discover ordinary controls. When interaction is
|
||||||
|
needed, obtain a fresh observation before acting on screenshot or HTML findings.
|
||||||
|
|
||||||
Use `browser_page` with these actions:
|
Use `browser_page` for purposeful navigation, history, reload, or a lifecycle wait. Avoid speculative
|
||||||
|
waits when no navigation is expected. After any page change, discard old refs and observe again.
|
||||||
|
|
||||||
- `navigate` requires `url` and optionally accepts `waitUntil` and `timeoutMs`.
|
## Respect the Agent Window boundary
|
||||||
- `back` and `forward` move one history entry and optionally accept lifecycle wait fields.
|
|
||||||
- `reload` refreshes the current tab; set `hard: true` only when bypassing cache is necessary.
|
|
||||||
- `wait` waits for an expected navigation after an interaction that did not already wait.
|
|
||||||
|
|
||||||
Avoid speculative waits when no navigation is expected. After any page change, discard old refs and
|
Use `browser_tabs` to list returned tab ids before selecting, closing, borrowing, or returning tabs.
|
||||||
observe again.
|
Borrow a user tab only for the immediate task, and return it as soon as that step is complete. Never
|
||||||
|
invent a tab id or keep a personal tab borrowed across unrelated work.
|
||||||
|
|
||||||
## Reading priority
|
## Ask the human when needed
|
||||||
|
|
||||||
Escalate page reading only as needed:
|
Use `browser_assist` action `request-help` for login, captcha, OTP, payment confirmation, consent, or
|
||||||
|
another step the user must complete. Give a precise prompt and highlight fresh targets when concrete
|
||||||
|
controls are involved. Use completion criteria only for a clear stable success signal.
|
||||||
|
|
||||||
1. `browser_inspect` with `action: "observe"` for normal understanding and refs.
|
Resume only after the user continues or the criteria complete. Treat cancellation as rejection and
|
||||||
2. The same tool with `action: "snapshot"` for a stricter static tree.
|
timeout as a blocker rather than retrying. Observe again after control returns before reasoning about
|
||||||
3. `action: "html"` for exact markup that semantic views cannot provide; keep `maxBytes` bounded and
|
the new state or using refs.
|
||||||
use only a fresh ref when scoping a subtree.
|
|
||||||
4. `action: "screenshot"` for layout, styling, canvas, or genuinely visual evidence. A fresh `ref`
|
|
||||||
crops to an element; omit it for the visible tab.
|
|
||||||
|
|
||||||
Do not start with raw HTML or a screenshot merely to discover ordinary controls.
|
The same tool can resize the Agent Window or emulate a device when the task requires visual or
|
||||||
|
responsive testing. Emulation is scoped to one tab.
|
||||||
|
|
||||||
## Tabs and the Agent Window boundary
|
## Debug and recover without wandering
|
||||||
|
|
||||||
Use `browser_tabs` with these actions:
|
Use `browser_inspect` console or network actions only for relevant, bounded, read-only diagnostics.
|
||||||
|
Continue from returned sequence cursors instead of rereading the same buffer.
|
||||||
|
|
||||||
- `list` returns visible tabs; `scope: "user"` finds a user tab before borrowing.
|
- Stale ref: observe again and retry the intended action once.
|
||||||
- `create` makes an Agent Window tab and optionally accepts `url`, `active`, and `index`.
|
- Unknown tab: list tabs instead of guessing.
|
||||||
- `select` focuses an Agent Window tab and requires a returned `tabId`.
|
- Unknown session: list owned sessions or start one; never try foreign ids.
|
||||||
- `close` closes an Agent Window tab and requires a returned `tabId`.
|
- Timeout: inspect current state before deciding whether one longer purposeful wait is useful.
|
||||||
- `borrow` moves a user tab into the Agent Window and requires a listed user `tabId`.
|
- Unrecoverable failure: report the blocker and stop the owned session.
|
||||||
- `return` restores a borrowed tab and requires that borrowed `tabId`.
|
|
||||||
|
|
||||||
Return borrowed tabs as soon as the immediate task finishes. Stopping the session is only fallback
|
Arbitrary page-script evaluation and interaction recording are intentionally unsupported. Do not
|
||||||
cleanup. Never invent a tab id.
|
invent tools or route around those limits.
|
||||||
|
|
||||||
## Human help and display controls
|
|
||||||
|
|
||||||
Use `browser_assist` with `action: "request-help"` for login, captcha, OTP, payment confirmation,
|
|
||||||
consent, or another human-only step. `prompt` is required. Optional `targets` highlight fresh refs or
|
|
||||||
selectors. `completionCriteria` can detect explicit stable success through URL, selector, or text
|
|
||||||
conditions. After the user continues or criteria complete, observe again before reasoning about the
|
|
||||||
new state.
|
|
||||||
|
|
||||||
Use `action: "resize"` with `width` and `height` to resize the outer Agent Window. Use
|
|
||||||
`action: "emulate"` with a device preset or explicit width and height; `mobile: true` also requires
|
|
||||||
dimensions. Use `off: true` alone to clear emulation. Emulation is per tab.
|
|
||||||
|
|
||||||
## Read-only diagnostics
|
|
||||||
|
|
||||||
Use `browser_inspect` with `action: "console"` for buffered logs and exceptions, and
|
|
||||||
`action: "network"` for response and failure metadata. Both accept `tabId`, `since`, `limit`, and
|
|
||||||
`maxTextChars`; continue with the returned sequence cursor rather than rereading the same entries.
|
|
||||||
Console additionally accepts `includeStack`.
|
|
||||||
|
|
||||||
Arbitrary page script evaluation and interaction recording are intentionally unsupported. Do not
|
|
||||||
invent tool names or route around that limitation.
|
|
||||||
|
|
||||||
## Error handling and final checklist
|
|
||||||
|
|
||||||
On a stale ref, observe again and retry once. On an unknown tab, list tabs instead of guessing. On
|
|
||||||
an unknown session, list owned sessions or start one; never try foreign ids. On timeout, decide
|
|
||||||
whether one longer purposeful wait is justified, and never immediately repeat a failed history
|
|
||||||
action without first checking page state.
|
|
||||||
|
|
||||||
Before finishing, verify that the observable goal is met or the blocker is clear, borrowed tabs are
|
|
||||||
returned when practical, the owned session is stopped, and no extra page action occurs after
|
|
||||||
success.
|
|
||||||
|
|||||||
@@ -1,10 +1,11 @@
|
|||||||
/**
|
/**
|
||||||
* Screenshot delivery. The canonical tool value stays plain JSON (a PNG file
|
* Screenshot delivery. The canonical tool value stays plain JSON (a screenshot
|
||||||
* path plus pixel metadata); when the host mounts a durable attachment store
|
* file path plus pixel metadata); when the host mounts a durable attachment
|
||||||
* AND the calling route declares image input, the PNG bytes are additionally
|
* store AND the calling route declares image input, the capture bytes are
|
||||||
* committed through `ctx.attachments` so the render step can attach the image
|
* additionally committed through `ctx.attachments` so the render step can
|
||||||
* itself to the tool result. Any uncertainty (no store, unknown route,
|
* attach the image itself to the tool result. Any uncertainty (no store,
|
||||||
* text-only model) falls back to the path-only form.
|
* unknown route, text-only model, unrecognized bytes) falls back to the
|
||||||
|
* path-only form.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import type { Context } from "@deepseek-ai/cordis";
|
import type { Context } from "@deepseek-ai/cordis";
|
||||||
@@ -33,6 +34,54 @@ interface LlmLike {
|
|||||||
): Promise<{ inputModalities?: readonly string[] }>;
|
): Promise<{ inputModalities?: readonly string[] }>;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Sniff the image media type from magic bytes. The capture path can hand back
|
||||||
|
* JPEG (e.g. `chrome.tabs.captureVisibleTab` on some Chromium/Edge builds
|
||||||
|
* returning JPEG regardless of the requested format), so the declared type can
|
||||||
|
* never be trusted from the file extension alone.
|
||||||
|
* @param data - first bytes of the capture.
|
||||||
|
* @returns the detected media type, or undefined when unrecognized.
|
||||||
|
*/
|
||||||
|
function sniffImageMediaType(data: Uint8Array): string | undefined {
|
||||||
|
if (data.length >= 4) {
|
||||||
|
// PNG signature: 89 50 4E 47 (0D 0A 1A 0A)
|
||||||
|
if (data[0] === 0x89 && data[1] === 0x50 && data[2] === 0x4e && data[3] === 0x47) {
|
||||||
|
return "image/png";
|
||||||
|
}
|
||||||
|
// JPEG SOI: FF D8 FF
|
||||||
|
if (data[0] === 0xff && data[1] === 0xd8 && data[2] === 0xff) {
|
||||||
|
return "image/jpeg";
|
||||||
|
}
|
||||||
|
// WebP: "RIFF" (52 49 46 46) + "WEBP" at offset 8
|
||||||
|
if (
|
||||||
|
data[0] === 0x52 &&
|
||||||
|
data[1] === 0x49 &&
|
||||||
|
data[2] === 0x46 &&
|
||||||
|
data[3] === 0x46 &&
|
||||||
|
data.length >= 12 &&
|
||||||
|
data[8] === 0x57 &&
|
||||||
|
data[9] === 0x45 &&
|
||||||
|
data[10] === 0x42 &&
|
||||||
|
data[11] === 0x50
|
||||||
|
) {
|
||||||
|
return "image/webp";
|
||||||
|
}
|
||||||
|
// GIF: "GIF87a" or "GIF89a"
|
||||||
|
if (
|
||||||
|
data[0] === 0x47 &&
|
||||||
|
data[1] === 0x49 &&
|
||||||
|
data[2] === 0x46 &&
|
||||||
|
data[3] === 0x38 &&
|
||||||
|
data.length >= 6 &&
|
||||||
|
(data[4] === 0x37 || data[4] === 0x39) &&
|
||||||
|
data[5] === 0x61
|
||||||
|
) {
|
||||||
|
return "image/gif";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return undefined;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Commit screenshot bytes to the host attachment store when the composition
|
* Commit screenshot bytes to the host attachment store when the composition
|
||||||
* supports durable images on the current model route.
|
* supports durable images on the current model route.
|
||||||
@@ -46,7 +95,14 @@ export async function trySaveScreenshot(
|
|||||||
): Promise<ImageAttachmentRef | undefined> {
|
): Promise<ImageAttachmentRef | undefined> {
|
||||||
const attachments = ctx.get("attachments") as AttachmentLike | undefined;
|
const attachments = ctx.get("attachments") as AttachmentLike | undefined;
|
||||||
if (attachments === undefined) return undefined;
|
if (attachments === undefined) return undefined;
|
||||||
if (!attachments.imageLimits.mediaTypes.includes("image/png")) return undefined;
|
// Declare the real media type from the bytes, never a hard-coded "image/png".
|
||||||
|
// Declaring the wrong type trips the store's IMAGE_TYPE_MISMATCH check and
|
||||||
|
// silently degrades to path-only; sniffing first lets JPEG captures (returned
|
||||||
|
// by some Chromium builds for `captureVisibleTab({ format: "png" })`) still
|
||||||
|
// inline correctly.
|
||||||
|
const mediaType = sniffImageMediaType(data);
|
||||||
|
if (mediaType === undefined) return undefined;
|
||||||
|
if (!attachments.imageLimits.mediaTypes.includes(mediaType)) return undefined;
|
||||||
if (
|
if (
|
||||||
data.byteLength >
|
data.byteLength >
|
||||||
Math.min(attachments.imageLimits.maxImageBytes, attachments.imageLimits.maxMessageImageBytes)
|
Math.min(attachments.imageLimits.maxImageBytes, attachments.imageLimits.maxMessageImageBytes)
|
||||||
@@ -55,9 +111,9 @@ export async function trySaveScreenshot(
|
|||||||
}
|
}
|
||||||
if (!(await isImageCapableRoute(ctx, exec))) return undefined;
|
if (!(await isImageCapableRoute(ctx, exec))) return undefined;
|
||||||
try {
|
try {
|
||||||
return await attachments.saveImage({ data, mediaType: "image/png", name });
|
return await attachments.saveImage({ data, mediaType, name });
|
||||||
} catch {
|
} catch {
|
||||||
// A store hiccup must not fail the tool call; the PNG path still works.
|
// A store hiccup must not fail the tool call; the path-only form still works.
|
||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -776,7 +776,11 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
|
|||||||
additionalProperties: false,
|
additionalProperties: false,
|
||||||
properties: {
|
properties: {
|
||||||
attachmentId: { type: "string", required: true },
|
attachmentId: { type: "string", required: true },
|
||||||
mediaType: { type: "string", required: true, const: "image/png" },
|
mediaType: {
|
||||||
|
type: "string",
|
||||||
|
required: true,
|
||||||
|
enum: ["image/png", "image/jpeg", "image/webp", "image/gif"],
|
||||||
|
},
|
||||||
bytes: { type: "integer", required: true },
|
bytes: { type: "integer", required: true },
|
||||||
width: { type: "integer", required: true },
|
width: { type: "integer", required: true },
|
||||||
height: { type: "integer", required: true },
|
height: { type: "integer", required: true },
|
||||||
@@ -846,7 +850,7 @@ function defineBrowserOperations(deps: ToolDeps, register: DefinitionRegistrar):
|
|||||||
? {
|
? {
|
||||||
image: {
|
image: {
|
||||||
attachmentId: String(ref.attachmentId),
|
attachmentId: String(ref.attachmentId),
|
||||||
mediaType: "image/png" as const,
|
mediaType: ref.mediaType,
|
||||||
bytes: ref.bytes,
|
bytes: ref.bytes,
|
||||||
width: ref.width,
|
width: ref.width,
|
||||||
height: ref.height,
|
height: ref.height,
|
||||||
|
|||||||
@@ -47,7 +47,10 @@ describe("registerBskSkill", () => {
|
|||||||
expect(content).toContain("Mandatory workflow");
|
expect(content).toContain("Mandatory workflow");
|
||||||
expect(content).toContain("Refs invalidate after navigation");
|
expect(content).toContain("Refs invalidate after navigation");
|
||||||
expect(content).toContain("evaluation and interaction recording are intentionally unsupported");
|
expect(content).toContain("evaluation and interaction recording are intentionally unsupported");
|
||||||
expect(content.length).toBeGreaterThan(6_000);
|
// Keep the lazily injected instructions inside a bounded prompt budget,
|
||||||
|
// while the lower bound catches accidental truncation of the guidance.
|
||||||
|
expect(content.length).toBeGreaterThan(3_000);
|
||||||
|
expect(content.length).toBeLessThan(6_000);
|
||||||
expect(skill.source).toBe("bundled");
|
expect(skill.source).toBe("bundled");
|
||||||
// Source frontmatter is registration metadata and must not leak into the body.
|
// Source frontmatter is registration metadata and must not leak into the body.
|
||||||
expect(content.startsWith("---")).toBe(false);
|
expect(content.startsWith("---")).toBe(false);
|
||||||
|
|||||||
@@ -894,6 +894,15 @@ describe("inspect.screenshot", () => {
|
|||||||
return path;
|
return path;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function jpegFile(): string {
|
||||||
|
const dir = mkdtempSync(join(tmpdir(), "bsk-test-"));
|
||||||
|
const path = join(dir, "shot.png");
|
||||||
|
// A JPEG SOI marker (FF D8 FF) even though the file is named ".png" —
|
||||||
|
// mirrors captureVisibleTab returning JPEG on some Chromium builds.
|
||||||
|
writeFileSync(path, Buffer.from([0xff, 0xd8, 0xff, 0xdb, 0, 1, 2, 3, 4]));
|
||||||
|
return path;
|
||||||
|
}
|
||||||
|
|
||||||
it("returns the PNG path when no attachment store is mounted", async () => {
|
it("returns the PNG path when no attachment store is mounted", async () => {
|
||||||
const path = pngFile();
|
const path = pngFile();
|
||||||
const { tools, calls } = setup({
|
const { tools, calls } = setup({
|
||||||
@@ -959,6 +968,50 @@ describe("inspect.screenshot", () => {
|
|||||||
expect(rendered.map((block) => block.type)).toEqual(["text", "image"]);
|
expect(rendered.map((block) => block.type)).toEqual(["text", "image"]);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("declares the real media type when capture bytes are JPEG, not a hard-coded PNG", async () => {
|
||||||
|
const path = jpegFile();
|
||||||
|
const services = {
|
||||||
|
attachments: {
|
||||||
|
imageLimits: {
|
||||||
|
mediaTypes: ["image/png", "image/jpeg", "image/webp", "image/gif"],
|
||||||
|
maxImageBytes: 10_000_000,
|
||||||
|
maxMessageImageBytes: 10_000_000,
|
||||||
|
},
|
||||||
|
saveImage: async (input: { data: Uint8Array; mediaType: string; name?: string }) => ({
|
||||||
|
attachmentId: "att-jpeg",
|
||||||
|
mediaType: input.mediaType,
|
||||||
|
bytes: input.data.byteLength,
|
||||||
|
width: 800,
|
||||||
|
height: 600,
|
||||||
|
name: input.name,
|
||||||
|
}),
|
||||||
|
},
|
||||||
|
llm: {
|
||||||
|
resolveModelInfo: async () => ({ inputModalities: ["text", "image"] }),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
const { tools } = setup(
|
||||||
|
{
|
||||||
|
"session start": START_REPLY("s1"),
|
||||||
|
screenshot: { tab_id: 7, width: 800, height: 600, format: "png", path, byte_size: 9 },
|
||||||
|
},
|
||||||
|
services,
|
||||||
|
);
|
||||||
|
await startSession(tools);
|
||||||
|
const exec = makeExec({
|
||||||
|
agent: {
|
||||||
|
session: { requestHeader: () => ({ config: { provider: "deepseek", model: "vl" } }) },
|
||||||
|
options: {},
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const screenshot = tools.get("inspect.screenshot");
|
||||||
|
const value = (await screenshot?.execute({}, exec)) as {
|
||||||
|
image?: { attachmentId: string; mediaType: string };
|
||||||
|
};
|
||||||
|
// The reference media type must reflect the sniffed JPEG bytes, not "image/png".
|
||||||
|
expect(value.image).toMatchObject({ attachmentId: "att-jpeg", mediaType: "image/jpeg" });
|
||||||
|
});
|
||||||
|
|
||||||
it("stays path-only when the model route is text-only", async () => {
|
it("stays path-only when the model route is text-only", async () => {
|
||||||
const path = pngFile();
|
const path = pngFile();
|
||||||
const services = {
|
const services = {
|
||||||
|
|||||||
@@ -6,27 +6,27 @@
|
|||||||
"disconnected": "Disconnected",
|
"disconnected": "Disconnected",
|
||||||
"connecting": "Connecting…",
|
"connecting": "Connecting…",
|
||||||
"connected": "Connected",
|
"connected": "Connected",
|
||||||
"version_skew": "Protocol mismatch",
|
"version_skew": "Connected",
|
||||||
"disabled": "Connection off"
|
"disabled": "Connection off"
|
||||||
},
|
},
|
||||||
"stateDetail": {
|
"stateDetail": {
|
||||||
"disconnected": "Open BrowserSkill to connect.",
|
"disconnected": "Open BrowserSkill to connect.",
|
||||||
"connecting": "Connecting to BrowserSkill…",
|
"connecting": "Connecting to BrowserSkill…",
|
||||||
"connected": "Ready for automation tasks.",
|
"connected": "Ready for automation tasks.",
|
||||||
"version_skew": "Some features may be unavailable."
|
"version_skew": "Protocol versions differ. Please upgrade."
|
||||||
},
|
},
|
||||||
"stateBadge": {
|
"stateBadge": {
|
||||||
"disconnected": "Standby",
|
"disconnected": "Standby",
|
||||||
"connecting": "Igniting",
|
"connecting": "Igniting",
|
||||||
"connected": "Ready",
|
"connected": "Ready",
|
||||||
"version_skew": "Action needed",
|
"version_skew": "Upgradable",
|
||||||
"disabled": "Off"
|
"disabled": "Off"
|
||||||
},
|
},
|
||||||
"connectionToggleTitle": "BrowserSkill connection",
|
"connectionToggleTitle": "BrowserSkill connection",
|
||||||
"controlHintsToggleTitle": "Control hints",
|
"controlHintsToggleTitle": "Control hints",
|
||||||
"controlHintsToggleHint": "Show the status pill and orange glow while the Agent controls a page.",
|
"controlHintsToggleHint": "Show the status pill and orange glow while the Agent controls a page.",
|
||||||
"controlHintsInfoLabel": "About control hints",
|
"controlHintsInfoLabel": "About control hints",
|
||||||
"versionSkewWarning": "Extension protocol v{{extensionProtocol}}, daemon v{{daemonProtocol}}.",
|
"versionSkewWarning": "Extension protocol v{{extensionProtocol}}, CLI protocol v{{cliProtocol}}. Protocol versions differ — please upgrade.",
|
||||||
"upgradeAvailable": "Upgradable",
|
"upgradeAvailable": "Upgradable",
|
||||||
"launcher": {
|
"launcher": {
|
||||||
"title": "Quick actions"
|
"title": "Quick actions"
|
||||||
|
|||||||
@@ -6,27 +6,27 @@
|
|||||||
"disconnected": "未连接",
|
"disconnected": "未连接",
|
||||||
"connecting": "连接中…",
|
"connecting": "连接中…",
|
||||||
"connected": "已连接",
|
"connected": "已连接",
|
||||||
"version_skew": "协议不一致",
|
"version_skew": "已连接",
|
||||||
"disabled": "连接已关闭"
|
"disabled": "连接已关闭"
|
||||||
},
|
},
|
||||||
"stateDetail": {
|
"stateDetail": {
|
||||||
"disconnected": "请先打开 BrowserSkill。",
|
"disconnected": "请先打开 BrowserSkill。",
|
||||||
"connecting": "正在连接 BrowserSkill…",
|
"connecting": "正在连接 BrowserSkill…",
|
||||||
"connected": "可接收自动化任务。",
|
"connected": "可接收自动化任务。",
|
||||||
"version_skew": "部分能力可能不可用。"
|
"version_skew": "协议版本不同,请及时升级。"
|
||||||
},
|
},
|
||||||
"stateBadge": {
|
"stateBadge": {
|
||||||
"disconnected": "Standby",
|
"disconnected": "Standby",
|
||||||
"connecting": "Igniting",
|
"connecting": "Igniting",
|
||||||
"connected": "Ready",
|
"connected": "Ready",
|
||||||
"version_skew": "Action needed",
|
"version_skew": "可升级",
|
||||||
"disabled": "Off"
|
"disabled": "Off"
|
||||||
},
|
},
|
||||||
"connectionToggleTitle": "BrowserSkill 连接",
|
"connectionToggleTitle": "BrowserSkill 连接",
|
||||||
"controlHintsToggleTitle": "控制提示",
|
"controlHintsToggleTitle": "控制提示",
|
||||||
"controlHintsToggleHint": "Agent 控制页面时显示提示条和橙色闪光。",
|
"controlHintsToggleHint": "Agent 控制页面时显示提示条和橙色闪光。",
|
||||||
"controlHintsInfoLabel": "控制提示说明",
|
"controlHintsInfoLabel": "控制提示说明",
|
||||||
"versionSkewWarning": "扩展协议 v{{extensionProtocol}},daemon 协议 v{{daemonProtocol}}。",
|
"versionSkewWarning": "扩展协议 v{{extensionProtocol}},CLI 协议 v{{cliProtocol}}。协议版本不同,请及时升级。",
|
||||||
"upgradeAvailable": "可升级",
|
"upgradeAvailable": "可升级",
|
||||||
"launcher": {
|
"launcher": {
|
||||||
"title": "快捷功能"
|
"title": "快捷功能"
|
||||||
|
|||||||
+98
-283
@@ -1,327 +1,142 @@
|
|||||||
---
|
---
|
||||||
name: browser-skill
|
name: browser-skill
|
||||||
description: |
|
description: |
|
||||||
Use when the user asks to perform browser automation tasks against their
|
Use when the user asks to automate their logged-in Chromium browser: visit
|
||||||
logged-in browser: visit and read pages, fill forms, scrape data, click
|
and read pages, fill forms, scrape data, click through flows, regression-test
|
||||||
through a flow, regression-test a PR's UI, validate a deployed page.
|
a PR's UI, validate a deployed page, or operate a tab they identify. Requires
|
||||||
Requires the bsk CLI installed and the browser-skill extension loaded.
|
the bsk CLI and browser extension.
|
||||||
---
|
---
|
||||||
|
|
||||||
# browser-skill
|
# browser-skill
|
||||||
|
|
||||||
Drive the user's **real Chromium browser** (with their logins and cookies) through the `bsk` CLI. The extension opens an isolated **Agent Window** for automation; the user's normal windows stay protected unless you explicitly borrow a tab.
|
Drive the user's real Chromium browser through `bsk`. Automation runs in an isolated **Agent
|
||||||
|
Window** with the user's existing logins and cookies. User-window tabs remain protected unless they
|
||||||
|
are explicitly borrowed.
|
||||||
|
|
||||||
## When to use
|
Do not use this skill for tasks with no browser, for extension installation, or when the user only
|
||||||
|
wants instructions. Never extract credentials, cookies, tokens, or other secrets from pages.
|
||||||
|
|
||||||
- Open pages, read titles/text, scrape structured data from sites the user can already access
|
## Required lifecycle
|
||||||
- Fill forms, click through multi-step flows, smoke-test a UI change
|
|
||||||
- Understand pages with `bsk snapshot` first; use `bsk get-html` or `bsk screenshot` only when the snapshot is insufficient
|
|
||||||
- Operate on a specific user tab they point you at (after `bsk tab borrow`)
|
|
||||||
|
|
||||||
## When NOT to use
|
Every browser task owns a bounded session:
|
||||||
|
|
||||||
- Tasks with **no browser** involved (files, APIs, databases only)
|
```text
|
||||||
- Installing or configuring the extension (point the user to setup docs instead)
|
1. bsk session start # retain the printed 4-letter session id
|
||||||
- **Credential harvesting** — never run `bsk evaluate` on banking, SSO, or password-manager pages to extract tokens, cookies, or secrets
|
2. bsk ... --session <id> # pass it to every session-scoped command
|
||||||
- Long-lived control of a user's personal login window — borrow only for the immediate step, then `bsk tab return` or end the session
|
3. bsk session stop <id> # always run on success and error paths
|
||||||
- Replacing the user's manual browsing when they only wanted an explanation
|
|
||||||
|
|
||||||
## Prerequisites
|
|
||||||
|
|
||||||
1. `bsk` on `PATH` (Rust CLI from browser-skill)
|
|
||||||
2. browser-skill **extension** loaded in Chromium and connected (popup shows green)
|
|
||||||
3. Any `bsk` command auto-starts background services as needed; use `bsk doctor` if anything fails
|
|
||||||
|
|
||||||
## Mandatory workflow
|
|
||||||
|
|
||||||
Every automation task **must** follow this lifecycle. Do **not** rely on idle timeouts (default session idle is 5 minutes).
|
|
||||||
|
|
||||||
```
|
|
||||||
1. bsk session start → capture the 4-letter session id printed on stdout
|
|
||||||
2. … every tool command … → always pass --session <id>
|
|
||||||
3. bsk session stop <id> → REQUIRED when done (even on error paths)
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Optional: `bsk session start --browser <instance-id-or-label>` when multiple browsers are connected (`bsk browsers` / error output lists them). Add `--no-focus` to open the Agent Window in the background without stealing focus from the user's current window.
|
Do not rely on the idle timeout for cleanup. Stop the session as soon as the goal is met unless the
|
||||||
|
user explicitly asks to keep it open. Stopping also returns borrowed tabs.
|
||||||
|
|
||||||
Emergency cleanup: `bsk session stop --all` or the Agent Window overlay **Stop all**.
|
Any `bsk` command auto-starts the background services it needs; never manage the daemon by hand.
|
||||||
|
When multiple browsers are connected, use `bsk browsers` and start with
|
||||||
|
`bsk session start --browser <id-or-label>`. Add `--no-focus` to that same start command when the
|
||||||
|
Agent Window does not need to interrupt the user's current work; it is not a flag on other commands.
|
||||||
|
Run `bsk doctor` when startup or transport problems persist after one retry.
|
||||||
|
|
||||||
## Stop when the goal is met
|
## Work toward one observable goal
|
||||||
|
|
||||||
Every task is a **bounded goal**, not open-ended browsing. The goal may come from the user's request, a recorded `trace.json`, or both.
|
- Derive a concrete success condition from the user's request or a supplied trace.
|
||||||
|
- Take the shortest purposeful path: observe, act, then make at most one observation to confirm an
|
||||||
|
ambiguous result.
|
||||||
|
- Once success is visible, do not click, refresh, navigate, switch tabs, or perform extra checks.
|
||||||
|
- If a human-only step appears or two attempts make no progress, request help instead of
|
||||||
|
brute-forcing.
|
||||||
|
|
||||||
1. **Define success first** — one concrete, observable condition derived from the user's words, `purpose`, or the last meaningful step in a trace (e.g. "form submitted", "item added to cart", "playback started").
|
With a trace, follow its semantic target information and values in order, but treat its refs as
|
||||||
2. **Take the shortest path** — snapshot → act → at most one check. Do not wander, re-try unrelated actions, or stack exploratory steps.
|
record-local hints. Stop when its purpose or last meaningful effect is satisfied. A trace guides the
|
||||||
3. **Stop as soon as success is reached** — run `bsk session stop <id>` immediately unless the user explicitly asked to keep the session open (e.g. "don't close yet", "keep browsing").
|
task; it does not expand the user's goal or authorize additional actions.
|
||||||
4. **No post-success work** — once the goal is met, do not click, refresh, navigate, re-search, switch tabs, or "double-check" that it worked. Further verification is a new task.
|
|
||||||
5. **When blocked, pause — do not brute-force** — if the page requires human input (login, captcha, OTP, payment confirmation) or an action fails twice with no progress, call `bsk request-help` instead of retrying blindly. See **Ask the human for help** below.
|
|
||||||
6. **When unsure** — at most one extra `bsk snapshot`. If success looks met, stop. If not, ask the user; do not keep clicking.
|
|
||||||
|
|
||||||
**With a trace:** replay steps in order using `target` role/name/tag and raw `value`/`selection` fields. After the last step (or when its `effect.navigated_to` / success hint is satisfied), apply rules 3–4 immediately. The trace guides execution; it does not extend control beyond the goal.
|
## Observe, act, observe
|
||||||
|
|
||||||
**Without a trace:** the user's request *is* the success condition. Satisfying it ends the task — same stop rules apply.
|
Use this default loop:
|
||||||
|
|
||||||
## Core interaction loop
|
```text
|
||||||
|
|
||||||
Write operations only affect tabs in the **Agent Window** (or tabs you **borrowed** into it).
|
|
||||||
|
|
||||||
```
|
|
||||||
bsk navigate <url> --session <id>
|
bsk navigate <url> --session <id>
|
||||||
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
bsk observe --session <id>
|
||||||
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
bsk click|hover|fill|select|press ... --session <id>
|
||||||
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
bsk observe --session <id> # after navigation or a meaningful DOM change
|
||||||
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
|
||||||
bsk observe --session <id> → again after navigation / DOM change
|
|
||||||
```
|
```
|
||||||
|
|
||||||
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also
|
||||||
|
make them stale. Observe again before the next interaction.
|
||||||
|
|
||||||
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`.
|
||||||
|
The listed items are labels, not usable refs: hover the trigger, observe again, then act on the
|
||||||
|
revealed item's own ref. Do not click the trigger itself unless the user wants the trigger's action.
|
||||||
|
|
||||||
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
Escalate page reading only as needed:
|
||||||
|
|
||||||
## Observation priority
|
1. `bsk observe` for normal semantic understanding, text, controls, and refs.
|
||||||
|
2. `bsk snapshot` when a stricter static accessibility tree is more useful.
|
||||||
|
3. `bsk get-html` for exact markup or hidden metadata that semantic views cannot provide.
|
||||||
|
4. `bsk screenshot` for layout, styling, canvas, images, or requested visual evidence.
|
||||||
|
|
||||||
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
Do not start with raw HTML or screenshots merely to discover ordinary controls. When interaction is
|
||||||
|
needed, obtain a fresh observation before acting on screenshot or HTML findings.
|
||||||
|
|
||||||
1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
## Respect the Agent Window boundary
|
||||||
2. `bsk snapshot` — strict static accessibility tree fallback
|
|
||||||
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
|
|
||||||
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
|
|
||||||
|
|
||||||
Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page.
|
Normal page writes affect only Agent Window tabs. To operate a user tab, first list it with
|
||||||
|
`bsk tab list --scope user --session <id>`, then `bsk tab borrow <tab-id>`. Return it immediately
|
||||||
|
after the relevant step with `bsk tab return <tab-id>`; never invent a tab id or keep a personal tab
|
||||||
|
borrowed across unrelated work.
|
||||||
|
|
||||||
## Sandbox rules
|
## Ask the human when needed
|
||||||
|
|
||||||
| Rule | Detail |
|
Use `bsk request-help` for login, captcha, OTP, payment confirmation, consent, or another step the
|
||||||
|------|--------|
|
user must complete. Give a precise prompt and pass fresh `--target` refs/selectors when concrete
|
||||||
| Agent Window | `bsk tab create`, `bsk navigate`, `bsk click`, etc. work on agent tabs by default |
|
controls can be highlighted. Use completion criteria only when the page has a clear stable success
|
||||||
| User tabs | Read-only until borrowed: `bsk tab list --session <id> --scope user` then `bsk tab borrow <tab-id> --session <id>` |
|
signal.
|
||||||
| Return borrowed tabs | Call `bsk tab return <tab-id> --session <id>` when finished; unreturned tabs are **auto-returned** on `bsk session stop` |
|
|
||||||
| Writes off-agent | Commands that mutate the page fail if the tab is not in the Agent Window — borrow or create a tab first |
|
|
||||||
|
|
||||||
## Global flags
|
The result `outcome` is one of `continued`, `completed`, `cancelled`, `timed_out`, or `disabled`
|
||||||
|
(`navigated` is deprecated — never treat navigation as a completion signal). Resume only after
|
||||||
|
`continued` or `completed`. Treat `cancelled` as rejection, and `timed_out` or `disabled` as a
|
||||||
|
blocker rather than a reason to retry. After control returns, run a fresh `bsk observe` before
|
||||||
|
reasoning about the page or using refs.
|
||||||
|
|
||||||
| Flag | Purpose |
|
## Command inventory
|
||||||
|------|---------|
|
|
||||||
| `--json` | Machine-readable JSON on stdout (errors too) |
|
|
||||||
| `--quiet` | Suppress informational stderr |
|
|
||||||
| `-v` / `-vv` | More verbose logging |
|
|
||||||
|
|
||||||
Auto-update is **on by default** — the background daemon upgrades `bsk` itself when a new release is available (postponed while a session is active). Set `BSK_AUTO_UPDATE=off` to disable it and upgrade manually with `bsk update`.
|
This list of names is complete. Never invent a command outside it; read
|
||||||
|
`bsk <command...> --help` for flags instead of guessing them.
|
||||||
|
|
||||||
Command-specific flags (timeouts, `--tab-id`, `--wait-until`, …): **`bsk <cmd> --help`**
|
```text
|
||||||
|
session start|stop|list browsers status doctor update logs
|
||||||
## CLI command reference (one line each)
|
navigate navigate-back navigate-forward reload wait-for-navigation wait-ms
|
||||||
|
observe snapshot get-html screenshot console network
|
||||||
Details and flags: **`bsk <cmd> --help`**
|
click hover fill select press evaluate
|
||||||
|
tab list|create|close|select|borrow|return window resize emulate
|
||||||
### Diagnostics
|
request-help record start|stop
|
||||||
|
|
||||||
| Command | Summary |
|
|
||||||
|---------|---------|
|
|
||||||
| `bsk status` | Connection health, connected browsers, active sessions |
|
|
||||||
| `bsk doctor` | Deep diagnostics and repair hints |
|
|
||||||
| `bsk browsers` | List connected browser instances (ids, labels, versions) |
|
|
||||||
|
|
||||||
### Session
|
|
||||||
|
|
||||||
| Command | Summary |
|
|
||||||
|---------|---------|
|
|
||||||
| `bsk session start` | Open Agent Window (`--width`/`--height` for initial size); prints **4-letter session id** |
|
|
||||||
| `bsk session start --no-focus` | Open Agent Window in the background without stealing focus |
|
|
||||||
| `bsk session stop <id>` | End session, close Agent Window, auto-return borrowed tabs |
|
|
||||||
| `bsk session stop --all` | Stop every active session |
|
|
||||||
| `bsk session list` | List active sessions |
|
|
||||||
|
|
||||||
### Window (require `--session <id>`)
|
|
||||||
|
|
||||||
| Command | Summary |
|
|
||||||
|---------|---------|
|
|
||||||
| `bsk window resize` | Resize the Agent Window (`--width`, `--height`; 100..=7680 CSS px) |
|
|
||||||
|
|
||||||
### Device emulation — `bsk emulate` (requires `--session <id>`)
|
|
||||||
|
|
||||||
Emulate a mobile device environment on the agent tab via CDP — viewport, User-Agent, and touch — to debug a page's mobile layout and behaviour:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
bsk emulate --session <id> --device iphone-14
|
|
||||||
bsk emulate --session <id> --width 390 --height 844 --dpr 3 --mobile --ua "Mozilla/5.0 (iPhone…" --touch
|
|
||||||
bsk emulate --session <id> --off
|
|
||||||
```
|
```
|
||||||
|
|
||||||
- Presets (`--device`): `iphone-14`, `iphone-14-pro-max`, `iphone-se`, `pixel-7`, `galaxy-s23`, `ipad-mini`, `galaxy-tab-s8`. Manual flags (`--width`/`--height`/`--dpr`/`--mobile`/`--ua`/`--accept-language`/`--touch`/`--max-touch-points`) also work without a preset, or override individual preset fields; `--no-mobile`/`--no-touch` turn a preset's mobile viewport / touch emulation back off.
|
Required flags that are easy to get wrong:
|
||||||
- Repeated runs merge field by field onto the tab's current emulation state — only the flags you pass change. E.g. after `--device iphone-14`, `bsk emulate --session <id> --width 390 --height 844` keeps the preset's dpr (`3`) and mobile viewport. (The extension remembers the state per tab; after an extension reload the next run applies only the fields it carries.)
|
|
||||||
- `--off` clears every override (viewport, UA, touch) and the remembered state, restoring the tab's real environment.
|
|
||||||
- Scope: overrides apply to **one tab only** (CDP per-target) and are **not inherited by new tabs** — re-run `bsk emulate` after opening or switching to another tab (default target is the session's active tab; `--tab-id` overrides).
|
|
||||||
- Emulation covers viewport/UA/touch only; it does not throttle the network, fake geolocation, or synthesise real touch-event streams.
|
|
||||||
|
|
||||||
### Tabs (require `--session <id>`)
|
```text
|
||||||
|
bsk fill <ref> --value <text> bsk select <ref> --value <option-value>
|
||||||
| Command | Summary |
|
bsk screenshot --out <path> bsk emulate --device <preset-id>
|
||||||
|---------|---------|
|
|
||||||
| `bsk tab list` | List tabs (`--scope user\|agent\|all`, default `all`) |
|
|
||||||
| `bsk tab create` | New tab in Agent Window (`--url`, `--no-active`, `--index`) |
|
|
||||||
| `bsk tab close <tab-id>` | Close an agent tab |
|
|
||||||
| `bsk tab select <tab-id>` | Focus an agent tab |
|
|
||||||
| `bsk tab borrow <tab-id>` | Move a user tab into the Agent Window |
|
|
||||||
| `bsk tab return <tab-id>` | Return a borrowed tab to its original window |
|
|
||||||
|
|
||||||
### Observation (require `--session` unless noted)
|
|
||||||
|
|
||||||
| Command | Summary |
|
|
||||||
|---------|---------|
|
|
||||||
| `bsk snapshot` | First-choice static page understanding: accessibility tree with `@eN` element refs |
|
|
||||||
| `bsk observe` | Semantic VOM observation with bounded perception probes for conditional surfaces |
|
|
||||||
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
|
|
||||||
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
|
|
||||||
|
|
||||||
### Console & network debugging (read-only; require `--session`)
|
|
||||||
|
|
||||||
| Command | Summary |
|
|
||||||
|---------|---------|
|
|
||||||
| `bsk console` | Buffered page console messages, JS exceptions, and browser log entries (`--include-stack` for stack traces) |
|
|
||||||
| `bsk network` | Buffered network responses (status, method, URL, MIME/resource type) and failures (`net::ERR_*` reason) |
|
|
||||||
|
|
||||||
Both capture from the moment the tab is attached and read a bounded per-tab buffer: `--since <seq>` pages from a cursor (`next_since` in the result), `--limit` (default 50, max 200), `--max-text-chars` (default 1000, max 4096), `--tab-id` to target a non-active tab. Both are strictly read-only — they never intercept or modify traffic, and request/response headers, bodies, and timings are not captured.
|
|
||||||
|
|
||||||
### Navigation
|
|
||||||
|
|
||||||
| Command | Summary |
|
|
||||||
|---------|---------|
|
|
||||||
| `bsk navigate <url>` | Go to URL in agent tab (`--wait-until`, `--timeout`) |
|
|
||||||
| `bsk navigate-back` | History back one step |
|
|
||||||
| `bsk navigate-forward` | History forward one step |
|
|
||||||
| `bsk reload` | Reload current tab (`--hard` bypass cache) |
|
|
||||||
|
|
||||||
(`bsk navigate back` / `bsk navigate forward` are equivalent subcommands.)
|
|
||||||
|
|
||||||
### Interaction
|
|
||||||
|
|
||||||
| Command | Summary |
|
|
||||||
|---------|---------|
|
|
||||||
| `bsk click <ref-or-selector>` | Click element (`--button`, `--click-count`, `--modifiers`) |
|
|
||||||
| `bsk hover <ref-or-selector>` | Move the mouse to an element and wait for hover UI to settle (`--settle`, `--modifiers`) |
|
|
||||||
| `bsk fill <ref-or-selector> --value <text>` | Clear and type into input |
|
|
||||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
|
||||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
|
||||||
|
|
||||||
### Scripting & timing
|
|
||||||
|
|
||||||
| Command | Summary |
|
|
||||||
|---------|---------|
|
|
||||||
| `bsk evaluate <expression>` | Run JS in agent tab (see red lines). JS throw → stderr, **exit 0** (RPC success); use `--json` and check `.ok` to detect JS errors |
|
|
||||||
| `bsk wait-for-navigation` | Block until load/DOM idle/etc. (`--wait-until`, `--timeout`) |
|
|
||||||
| `bsk wait-ms <duration>` | Sleep (`500ms`, `2s`, `1m`; **no** `--session`) |
|
|
||||||
|
|
||||||
### Ask the human for help — `bsk request-help`
|
|
||||||
|
|
||||||
When a step needs a human (captcha, login, OTP) or you want the user to
|
|
||||||
confirm an important action, pause and ask:
|
|
||||||
|
|
||||||
bsk request-help --session <id> --prompt "Solve the captcha, then click Done only after the site accepts it" \
|
|
||||||
--title "Captcha required" --target @e7 --target "#submit" --timeout 5m
|
|
||||||
|
|
||||||
- `--prompt` (required): what the user should do.
|
|
||||||
- `--title` (optional): custom title for the overlay panel. When omitted,
|
|
||||||
the extension shows its default localized title.
|
|
||||||
- `--target` (repeatable): a snapshot ref (`@e7`) or CSS selector
|
|
||||||
(`#submit`) to scroll to and flash-highlight. **Strongly recommended** —
|
|
||||||
whenever the prompt refers to a concrete element (a button to click, a
|
|
||||||
field to fill, a checkbox to toggle), pass its `@eN` ref / selector so the
|
|
||||||
user is guided straight to the right spot instead of hunting for it. For
|
|
||||||
interaction scenarios, always include the relevant target(s); reserve a
|
|
||||||
prompt with no `--target` for cases where there is genuinely no specific
|
|
||||||
element to point at (e.g. "wait for the page to finish loading").
|
|
||||||
- `--timeout` (default `5m`): how long to wait.
|
|
||||||
- `--completion-criteria` (optional): JSON success detector. Use it only
|
|
||||||
when there is a concrete post-help success signal, e.g.
|
|
||||||
`{"any":[{"url_contains":"/dashboard"},{"selector_exists":"[data-testid='account-menu']"}],"stable_for_ms":1000}`.
|
|
||||||
|
|
||||||
The target tab is brought to the foreground; the page stays interactive
|
|
||||||
while the agent control mask is hidden. The call blocks until the user
|
|
||||||
explicitly acts, the timeout expires, cancellation arrives, or explicit
|
|
||||||
completion criteria match. Page reloads, SPA route changes, and captcha
|
|
||||||
refreshes do not return control by themselves. The result `outcome` is one of:
|
|
||||||
|
|
||||||
- `continued` — the user finished and clicked Done / return control (treat as confirm).
|
|
||||||
- `cancelled` — the user clicked Cancel (treat as reject/abort).
|
|
||||||
- `timed_out` — nobody acted within the timeout.
|
|
||||||
- `completed` — the explicit `--completion-criteria` matched while the user had control.
|
|
||||||
- `navigated` — deprecated legacy outcome. Do not rely on navigation as a completion signal.
|
|
||||||
|
|
||||||
`note` carries any text the user typed back. `resolved_targets` reports
|
|
||||||
which refs/selectors matched a live element.
|
|
||||||
|
|
||||||
`request-help` does not refresh the page model after the user returns
|
|
||||||
control. After a `continued` or `completed` result, issue a separate
|
|
||||||
observation tool call (usually `bsk snapshot --session <id>`) before using
|
|
||||||
new refs or reasoning about the post-help page state.
|
|
||||||
#### Disabling request-help (unattended mode)
|
|
||||||
Set `BSK_REQUEST_HELP=off` on unattended servers: `bsk request-help` then
|
|
||||||
returns immediately with `outcome="disabled"` (no overlay, no waiting,
|
|
||||||
exit 0). Any other value keeps it enabled. If you get `disabled`, do not
|
|
||||||
retry — complete the task autonomously or stop gracefully.
|
|
||||||
|
|
||||||
### Recording — `bsk record`
|
|
||||||
|
|
||||||
Capture the user's own actions in the Agent Window to a **trace bundle**, for later LLM-driven automation. New CLI builds request **trace v3** (page observations + action chain); older extensions may still return **trace v2** (actions only), which the CLI exports as a single `trace.json` without `states/`.
|
|
||||||
|
|
||||||
```bash
|
|
||||||
bsk record start --browser <instance-id-or-label> \
|
|
||||||
[--url https://…] [--purpose "publish a wiki doc"] \
|
|
||||||
[--max-page-tokens 3000] [--redact-values] \
|
|
||||||
[--output trace]
|
|
||||||
# `--url` is optional; default https://example.com/ when omitted (must be http(s)).
|
|
||||||
# Blocks until the user clicks Finish in the recording panel, then writes:
|
|
||||||
# trace/trace.json — action chain (+ state index when v3)
|
|
||||||
# trace/states/ — v3 only: one `sN.txt` observe snapshot per settled page state
|
|
||||||
|
|
||||||
bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable
|
|
||||||
```
|
```
|
||||||
|
|
||||||
- **v3 bundle (preferred):** `--output` is a directory (default `./trace`) containing `trace.json` and `states/`. `trace.json` lists `states[]` (page observation ids) and `steps[]` (each step binds `state` = observe snapshot *before* the action and `result.state` = snapshot *after* settle). Page bodies live in `states/sN.txt` using the same VOM format as `bsk observe`.
|
`select` matches an option's `value` attribute, not its visible label. Device preset ids are
|
||||||
- **v2 fallback:** when the connected extension is older, export may contain only legacy `trace.json` with `pages[]` action context and no `states/` observation files. Update the extension for full v3 bundles.
|
lowercase and hyphenated, such as `iphone-14`.
|
||||||
- Each `states[]` entry is one **settled page observation** (captured after recording start, navigation landing, or action settle — not on a timer).
|
|
||||||
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
|
|
||||||
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
|
||||||
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
|
|
||||||
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
|
||||||
|
|
||||||
## Error handling
|
- `console` and `network` provide bounded, read-only debugging evidence.
|
||||||
|
- `emulate` applies viewport, user-agent, and touch overrides to one tab; new tabs do not inherit
|
||||||
|
them. Use `--off` to restore the real environment.
|
||||||
|
- `evaluate` is a last resort when observe plus normal interactions cannot complete the task. With
|
||||||
|
`--json`, inspect `.ok`: a JavaScript exception may still have CLI exit code 0 because the RPC
|
||||||
|
succeeded. Never evaluate credential surfaces to read storage, cookies, or auth data.
|
||||||
|
- `record` captures a user's actions for later replay. Read `bsk record start --help` before use,
|
||||||
|
and never record banking, SSO, password-manager, or other sensitive pages.
|
||||||
|
|
||||||
### Exit codes (`echo $?` after `bsk …`)
|
## Recover without wandering
|
||||||
|
|
||||||
| Code | Meaning | What to do |
|
- Stale ref: observe again and retry the intended action once.
|
||||||
|------|---------|------------|
|
- Unknown tab or session: list current tabs/sessions; never guess identifiers.
|
||||||
| `0` | Success (including `evaluate` where JS threw but RPC succeeded) | Continue; for `evaluate`, check `--json` `.ok` to distinguish JS success from RPC success |
|
- Timeout: inspect current page state before deciding whether one longer purposeful wait is useful.
|
||||||
| `1` | User error — bad args, unknown session, tab not in Agent Window, stale ref | Fix args; `bsk session list`; re-snapshot |
|
- Unsupported command: continue with available capabilities; suggest updating only when the missing
|
||||||
| `2` | Protocol / transport — service unreachable, IPC failure | `bsk doctor`; check extension connected; retry the command |
|
command is necessary.
|
||||||
| `3` | Browser / CDP execution failed | Retry; simplify selector; check tab still open |
|
- Unrecoverable failure: report the blocker and stop the session in a finally-style path.
|
||||||
| `4` | Timeout | Increase `--timeout`; try `--wait-until domcontentloaded` |
|
|
||||||
| `5` | Version skew (CLI vs extension) | Upgrade/reinstall matching versions |
|
|
||||||
|
|
||||||
Human errors print `error:` + `hint:` on stderr; `--json` includes `code`, `message`, `hint`, `exit_code`.
|
The CLI's current help and error hints are authoritative for flags, parameters, and recovery
|
||||||
|
details.
|
||||||
### When to run diagnostics
|
|
||||||
|
|
||||||
| Situation | Command |
|
|
||||||
|-----------|---------|
|
|
||||||
| Before first task in a session | `bsk status` — extension connected? |
|
|
||||||
| Any failure you cannot fix in one retry | `bsk doctor` |
|
|
||||||
| Multiple browsers / wrong target | `bsk browsers` then `bsk session start --browser <id>` |
|
|
||||||
|
|
||||||
Always **`bsk session stop <id>`** in a `finally`-style path so the Agent Window closes and borrowed tabs return.
|
|
||||||
|
|
||||||
## Red lines
|
|
||||||
|
|
||||||
1. **No token theft** — do not `bsk evaluate` on sensitive sites to read `localStorage`, cookies, or auth headers for exfiltration.
|
|
||||||
2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks.
|
|
||||||
3. **No skip stop** — always `bsk session stop <id>`; never assume idle timeout will clean up.
|
|
||||||
4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open.
|
|
||||||
5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics or conditional surfaces help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual.
|
|
||||||
6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
**More detail for any command:** `bsk <cmd> --help`
|
|
||||||
|
|||||||
Reference in New Issue
Block a user