mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
55
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9a5d45d156 | ||
|
|
11cd12c9cd | ||
|
|
873971f4cb | ||
|
|
a9c7444139 | ||
|
|
59e2c3224e | ||
|
|
d70df753f6 | ||
|
|
66ae555e40 | ||
|
|
fee7c0983c | ||
|
|
51473c01ff | ||
|
|
49744233f7 | ||
|
|
95571f8b70 | ||
|
|
8344e311fa | ||
|
|
63338cd98e | ||
|
|
bd193ceb5d | ||
|
|
5ed828736f | ||
|
|
ee620acf64 | ||
|
|
05a36075b8 | ||
|
|
1bd13d5c3c | ||
|
|
59d6533eff | ||
|
|
536d3a9844 | ||
|
|
641d934337 | ||
|
|
3b21f2f0a7 | ||
|
|
405433ab65 | ||
|
|
58c7f38a05 | ||
|
|
e0c88295c7 | ||
|
|
dc36e38cb9 | ||
|
|
d5de9b0154 | ||
|
|
995061514c | ||
|
|
406bb21b7e | ||
|
|
f06a514bd2 | ||
|
|
1e53ee63ae | ||
|
|
120dc441bb | ||
|
|
710b1219f4 | ||
|
|
177ecd3ebf | ||
|
|
0e6b9c4542 | ||
|
|
506adbe708 | ||
|
|
984d017b0f | ||
|
|
5df93143c5 | ||
|
|
7163f2f9b0 | ||
|
|
e9f0584af3 | ||
|
|
2997f495db | ||
|
|
f96bd72c3c | ||
|
|
329c68054b | ||
|
|
5cd43553ed | ||
|
|
9ad33801c7 | ||
|
|
7aaa5a2d01 | ||
|
|
210b21b637 | ||
|
|
48f64ec0df | ||
|
|
c48a53269e | ||
|
|
3bbd6ab19f | ||
|
|
1a70d2200e | ||
|
|
554861d6b1 | ||
|
|
60fbf30e24 | ||
|
|
a4b6a61d29 | ||
|
|
cad0b5cbc9 |
@@ -325,3 +325,180 @@ jobs:
|
||||
|
||||
echo "Chrome Web Store submission created successfully:"
|
||||
jq '{itemId, state, warningInfo}' "$PUBLISH_RESPONSE"
|
||||
|
||||
publish-edge-addons:
|
||||
needs: [resolve, build-extension]
|
||||
runs-on: ubuntu-latest
|
||||
environment: edge-addons
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Download build artifact
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: extension-zip
|
||||
path: dist
|
||||
|
||||
- name: Validate Microsoft Edge Add-ons configuration
|
||||
shell: bash
|
||||
env:
|
||||
API_KEY: ${{ secrets.EDGE_ADDONS_API_KEY }}
|
||||
CLIENT_ID: ${{ vars.EDGE_ADDONS_CLIENT_ID }}
|
||||
PRODUCT_ID: ${{ vars.EDGE_ADDONS_PRODUCT_ID }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "$API_KEY" ]; then
|
||||
echo "::error::Environment secret EDGE_ADDONS_API_KEY is not configured"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$CLIENT_ID" ]; then
|
||||
echo "::error::Environment variable EDGE_ADDONS_CLIENT_ID is not configured"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$PRODUCT_ID" ]; then
|
||||
echo "::error::Environment variable EDGE_ADDONS_PRODUCT_ID is not configured"
|
||||
exit 1
|
||||
fi
|
||||
if ! [[ "$PRODUCT_ID" =~ ^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$ ]]; then
|
||||
echo "::error::EDGE_ADDONS_PRODUCT_ID must be the Partner Center product GUID, not the public Edge Add-ons extension ID"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ARCHIVE="$(find dist -maxdepth 1 -name '*.zip' -type f -print -quit)"
|
||||
if [ -z "$ARCHIVE" ]; then
|
||||
echo "::error::No extension zip found in the downloaded artifact"
|
||||
exit 1
|
||||
fi
|
||||
echo "ARCHIVE=$ARCHIVE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Upload and publish Microsoft Edge Add-ons extension
|
||||
shell: bash
|
||||
env:
|
||||
API_KEY: ${{ secrets.EDGE_ADDONS_API_KEY }}
|
||||
CLIENT_ID: ${{ vars.EDGE_ADDONS_CLIENT_ID }}
|
||||
PRODUCT_ID: ${{ vars.EDGE_ADDONS_PRODUCT_ID }}
|
||||
RELEASE_VERSION: ${{ needs.resolve.outputs.version }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
API_ROOT="https://api.addons.microsoftedge.microsoft.com/v1"
|
||||
UPLOAD_HEADERS="$RUNNER_TEMP/edge-upload-headers.txt"
|
||||
UPLOAD_RESPONSE="$RUNNER_TEMP/edge-upload-response.json"
|
||||
STATUS_RESPONSE="$RUNNER_TEMP/edge-status-response.json"
|
||||
PUBLISH_HEADERS="$RUNNER_TEMP/edge-publish-headers.txt"
|
||||
PUBLISH_RESPONSE="$RUNNER_TEMP/edge-publish-response.json"
|
||||
|
||||
operation_id_from_headers() {
|
||||
local headers="$1"
|
||||
local location
|
||||
location="$(awk '
|
||||
tolower($1) == "location:" {
|
||||
sub(/\r$/, "")
|
||||
sub(/^[^:]*:[[:space:]]*/, "")
|
||||
print
|
||||
exit
|
||||
}
|
||||
' "$headers")"
|
||||
location="${location%%\?*}"
|
||||
printf '%s' "${location##*/}"
|
||||
}
|
||||
|
||||
poll_operation() {
|
||||
local operation_name="$1"
|
||||
local status_url="$2"
|
||||
local status="InProgress"
|
||||
local http_status
|
||||
|
||||
for attempt in $(seq 1 30); do
|
||||
sleep 5
|
||||
http_status="$(curl --silent --show-error \
|
||||
--output "$STATUS_RESPONSE" \
|
||||
--write-out '%{http_code}' \
|
||||
--header "Authorization: ApiKey $API_KEY" \
|
||||
--header "X-ClientID: $CLIENT_ID" \
|
||||
"$status_url")"
|
||||
if [ "$http_status" -lt 200 ] || [ "$http_status" -ge 300 ]; then
|
||||
echo "::error::Microsoft Edge Add-ons $operation_name status check failed with HTTP $http_status"
|
||||
jq . "$STATUS_RESPONSE" || cat "$STATUS_RESPONSE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
status="$(jq -r '.status // empty' "$STATUS_RESPONSE")"
|
||||
echo "$operation_name status check $attempt/30: $status"
|
||||
case "$status" in
|
||||
Succeeded)
|
||||
return 0
|
||||
;;
|
||||
InProgress)
|
||||
;;
|
||||
Failed)
|
||||
echo "::error::Microsoft Edge Add-ons $operation_name failed"
|
||||
jq . "$STATUS_RESPONSE"
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
echo "::error::Microsoft Edge Add-ons returned an unknown $operation_name status: $status"
|
||||
jq . "$STATUS_RESPONSE"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
echo "::error::Microsoft Edge Add-ons $operation_name did not complete after 30 checks"
|
||||
jq . "$STATUS_RESPONSE"
|
||||
exit 1
|
||||
}
|
||||
|
||||
HTTP_STATUS="$(curl --silent --show-error \
|
||||
--output "$UPLOAD_RESPONSE" \
|
||||
--dump-header "$UPLOAD_HEADERS" \
|
||||
--write-out '%{http_code}' \
|
||||
--request POST \
|
||||
--header "Authorization: ApiKey $API_KEY" \
|
||||
--header "X-ClientID: $CLIENT_ID" \
|
||||
--header "Content-Type: application/zip" \
|
||||
--data-binary "@$ARCHIVE" \
|
||||
"$API_ROOT/products/$PRODUCT_ID/submissions/draft/package")"
|
||||
if [ "$HTTP_STATUS" != "202" ]; then
|
||||
echo "::error::Microsoft Edge Add-ons upload failed with HTTP $HTTP_STATUS"
|
||||
jq . "$UPLOAD_RESPONSE" || cat "$UPLOAD_RESPONSE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
UPLOAD_OPERATION_ID="$(operation_id_from_headers "$UPLOAD_HEADERS")"
|
||||
if [ -z "$UPLOAD_OPERATION_ID" ]; then
|
||||
echo "::error::Microsoft Edge Add-ons upload response has no Location operation ID"
|
||||
cat "$UPLOAD_HEADERS"
|
||||
exit 1
|
||||
fi
|
||||
poll_operation \
|
||||
"package upload" \
|
||||
"$API_ROOT/products/$PRODUCT_ID/submissions/draft/package/operations/$UPLOAD_OPERATION_ID"
|
||||
|
||||
HTTP_STATUS="$(curl --silent --show-error \
|
||||
--output "$PUBLISH_RESPONSE" \
|
||||
--dump-header "$PUBLISH_HEADERS" \
|
||||
--write-out '%{http_code}' \
|
||||
--request POST \
|
||||
--header "Authorization: ApiKey $API_KEY" \
|
||||
--header "X-ClientID: $CLIENT_ID" \
|
||||
--header "Content-Type: application/json" \
|
||||
--data "{\"notes\":\"Automated BrowserSkill extension v${RELEASE_VERSION} release from GitHub Actions.\"}" \
|
||||
"$API_ROOT/products/$PRODUCT_ID/submissions")"
|
||||
if [ "$HTTP_STATUS" != "202" ]; then
|
||||
echo "::error::Microsoft Edge Add-ons publish failed with HTTP $HTTP_STATUS"
|
||||
jq . "$PUBLISH_RESPONSE" || cat "$PUBLISH_RESPONSE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PUBLISH_OPERATION_ID="$(operation_id_from_headers "$PUBLISH_HEADERS")"
|
||||
if [ -z "$PUBLISH_OPERATION_ID" ]; then
|
||||
echo "::error::Microsoft Edge Add-ons publish response has no Location operation ID"
|
||||
cat "$PUBLISH_HEADERS"
|
||||
exit 1
|
||||
fi
|
||||
poll_operation \
|
||||
"submission" \
|
||||
"$API_ROOT/products/$PRODUCT_ID/submissions/operations/$PUBLISH_OPERATION_ID"
|
||||
|
||||
echo "Microsoft Edge Add-ons submission created successfully:"
|
||||
jq '{status, message, errorCode, errors}' "$STATUS_RESPONSE"
|
||||
|
||||
@@ -7,10 +7,18 @@ dist/
|
||||
.output/
|
||||
.wxt/
|
||||
.web-extension-id
|
||||
.pnpm-store/
|
||||
|
||||
# Git worktrees
|
||||
.worktrees/
|
||||
|
||||
# Logs
|
||||
*.log
|
||||
|
||||
# Local browser evaluation artifacts
|
||||
evals/browser/results/
|
||||
evals/browser/agents.local.json
|
||||
|
||||
# macOS
|
||||
.DS_Store
|
||||
|
||||
|
||||
Generated
+1
@@ -137,6 +137,7 @@ name = "bsk"
|
||||
version = "0.1.11"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"base64",
|
||||
"bsk-protocol",
|
||||
"clap",
|
||||
"console",
|
||||
|
||||
@@ -36,6 +36,7 @@ flate2 = "1.1.9"
|
||||
tar = "0.4.46"
|
||||
zip = { version = "7.2.0", default-features = false, features = ["deflate"] }
|
||||
sha2 = "0.11.0"
|
||||
base64 = "0.22"
|
||||
tempfile = "3"
|
||||
nix = { version = "0.29", features = ["signal", "process"] }
|
||||
rand = "0.8"
|
||||
|
||||
@@ -208,6 +208,7 @@ The repository is a Cargo + pnpm workspace:
|
||||
- `apps/extension` — browser extension
|
||||
- `packages/ui` and `packages/i18n` — shared extension UI support
|
||||
- `packages/dsh-plugin-browserskill` — DeepSeek Harness plugin (`@wxg-prc-cpg/browser-skill-dsh-plugin`)
|
||||
- [`evals/browser`](evals/browser/README.md) — deterministic local pages and agent-neutral browser capability evaluation
|
||||
|
||||
## License
|
||||
|
||||
|
||||
@@ -178,6 +178,7 @@ Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;
|
||||
- `apps/extension` — 浏览器扩展
|
||||
- `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持
|
||||
- `packages/dsh-plugin-browserskill` — DeepSeek Harness 插件(`@wxg-prc-cpg/browser-skill-dsh-plugin`)
|
||||
- [`evals/browser`](evals/browser/README.zh-CN.md) — 确定性本地页面与 Agent 无关的浏览器能力测试台
|
||||
|
||||
## 许可证
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# BrowserSkill — Privacy Policy
|
||||
|
||||
**Last updated:** May 25, 2026
|
||||
**Last updated:** August 20, 2026
|
||||
|
||||
This Privacy Policy describes how the **BrowserSkill** browser extension (the "Extension") handles information when you install and use it. BrowserSkill is published as part of the open-source [BrowserSkill](https://github.com/Tencent/BrowserSkill) project. The source code is publicly auditable.
|
||||
|
||||
@@ -14,7 +14,7 @@ BrowserSkill is a local automation bridge that lets AI coding agents (such as Cu
|
||||
|
||||
## 2. Single Purpose
|
||||
|
||||
The Extension's single purpose is to expose browser automation primitives (navigation, DOM observation, screenshots, clicks, form filling, tab management) to a locally running BrowserSkill daemon over a WebSocket connection on `127.0.0.1`, so that an AI agent invoked by the user can interact with web pages on the user's behalf.
|
||||
The Extension's single purpose is to expose browser automation primitives (navigation, DOM observation, screenshots, clicks, form filling, task-scoped file transfer, and tab management) to a locally running BrowserSkill daemon over a WebSocket connection on `127.0.0.1`, so that an AI agent invoked by the user can interact with web pages on the user's behalf.
|
||||
|
||||
## 3. Data the Extension Accesses
|
||||
|
||||
@@ -26,6 +26,7 @@ Depending on the commands the user (via their AI agent) sends to the local daemo
|
||||
| **User input simulated by the agent** | Mouse clicks, keystrokes, and form values that the AI agent dispatches through the Chrome DevTools Protocol (CDP). | Required to perform automation actions the user has asked the agent to do. |
|
||||
| **Tab and window metadata** | Tab IDs, URLs, titles, window IDs of the Agent Window and any tabs the user explicitly authorizes. | Required to target automation commands at the correct tab/window. |
|
||||
| **Local extension storage** | A randomly generated 8-character instance ID and an optional user-supplied label. | Used so the local daemon can recognize this browser instance across reconnects. No personal data is stored. |
|
||||
| **File transfers requested by the agent** | Local files explicitly supplied to `bsk upload`, and the file created by a single `bsk download` action. | Required to attach a task file to a web page or return a browser-generated download to the invoking local agent. |
|
||||
| **OS notifications** | Permission to display a system notification when the agent requests to "borrow" one of the user's existing tabs. | Required to obtain explicit, per-tab user consent before the agent touches any pre-existing tab. |
|
||||
|
||||
## 4. Data the Extension Does **Not** Collect
|
||||
@@ -34,7 +35,7 @@ BrowserSkill does **not**:
|
||||
|
||||
- Send any data to remote servers, the Extension's authors, or any third party.
|
||||
- Call any LLM, AI, or cloud API. The Extension contains no API keys, model identifiers, or remote endpoints.
|
||||
- Read or transmit cookies, browsing history, bookmarks, downloads, saved passwords, or autofill data.
|
||||
- Read or transmit cookies, browsing history, bookmarks, saved passwords, or autofill data. It observes only the download initiated by an active `bsk download` call, not download history generally.
|
||||
- Use webcam, microphone, geolocation, or any device sensor.
|
||||
- Include analytics, telemetry, crash reporting, advertising SDKs, or fingerprinting code.
|
||||
- Track users across websites or across sessions.
|
||||
@@ -50,17 +51,19 @@ The Extension requests the following Chrome permissions. Each is used solely for
|
||||
- **`alarms`** — Periodically wake the service worker to keep the local WebSocket connection alive.
|
||||
- **`idle`** — Detect when the device returns from idle/locked so the Extension can promptly re-establish the local WebSocket connection after the machine wakes. No idle data is stored or transmitted.
|
||||
- **`notifications`** — Show a system notification to obtain user approval before the agent borrows a user-owned tab.
|
||||
- **`downloads`** — Correlate and route the one browser download initiated by an active `bsk download` command. If that claimed transaction fails, BrowserSkill cancels an in-progress file or removes its completed temporary browser file. It is not used to enumerate download history or alter unclaimed downloads.
|
||||
- **`storage`** — Persist a random instance ID and optional label in `chrome.storage.local`.
|
||||
- **Host permission `<all_urls>`** — Inject a small status overlay (showing "Agent Active") on pages controlled by the agent, and enable automation across whatever sites the user directs the agent to. The Extension does **not** read or transmit page content from sites the agent is not actively driving.
|
||||
|
||||
## 6. Where Data Goes
|
||||
|
||||
All Extension activity stays on the user's local device. The only network traffic the Extension generates is a WebSocket connection to `ws://127.0.0.1:52800` (loopback only). What the AI agent connected to that local daemon does with the data afterwards (for example, sending a screenshot to an LLM provider) is governed by the privacy policy of that agent or LLM provider, **not** by this policy. BrowserSkill is not a party to those communications.
|
||||
All Extension activity stays on the user's local device. The only network traffic the Extension generates is a WebSocket connection to the local bsk daemon on `127.0.0.1` (loopback only; default port **52800**, configurable in the extension popup). What the AI agent connected to that local daemon does with the data afterwards (for example, sending a screenshot to an LLM provider) is governed by the privacy policy of that agent or LLM provider, **not** by this policy. BrowserSkill is not a party to those communications.
|
||||
|
||||
## 7. Data Retention
|
||||
|
||||
- The instance ID and optional label persist in `chrome.storage.local` until the user uninstalls the Extension or clears extension storage.
|
||||
- Page content, screenshots, DOM snapshots, and other observed data are returned to the local daemon in response to commands and are **not retained by the Extension**. They live only as long as the agent's tool call.
|
||||
- Upload and download bytes are staged by the local daemon in a private, session-scoped directory. Download staging is removed after it is copied to the requested destination. Upload staging is retained until the session ends so a later form submission can still read the attached file. Remaining staging is removed when the session ends or disconnects, or when the daemon next starts after a crash.
|
||||
|
||||
## 8. User Control
|
||||
|
||||
@@ -77,7 +80,7 @@ The Extension is a developer tool and is not directed at children under 13. It d
|
||||
|
||||
## 10. Security
|
||||
|
||||
Because the Extension communicates only with `127.0.0.1`, no data is exposed to the network. Users should still avoid running BrowserSkill in untrusted environments, since any local process able to bind to `127.0.0.1:52800` could send commands to the Extension. Run BrowserSkill only on machines you control.
|
||||
Because the Extension communicates only with `127.0.0.1`, no data is exposed to the network. Users should still avoid running BrowserSkill in untrusted environments, since any local process able to bind to the configured loopback port could send commands to the Extension. Run BrowserSkill only on machines you control.
|
||||
|
||||
## 11. Open Source and Auditability
|
||||
|
||||
|
||||
@@ -54,7 +54,7 @@ BrowserSkill **不会**:
|
||||
|
||||
## 6. 数据流向
|
||||
|
||||
本扩展的所有活动都停留在用户本地设备上。本扩展产生的唯一网络流量是与 `ws://127.0.0.1:52800`(仅回环地址)的 WebSocket 连接。连接到该本地守护进程的 AI 助手在拿到数据之后如何处理(例如将截图发送给某个 LLM 服务),由该助手或 LLM 提供商自身的隐私政策约束,**不在本政策范围内**。BrowserSkill 不参与那些通信。
|
||||
本扩展的所有活动都停留在用户本地设备上。本扩展产生的唯一网络流量是与本地 bsk 守护进程在 `127.0.0.1`(仅回环地址;默认端口 **52800**,可在扩展弹窗中配置)上的 WebSocket 连接。连接到该本地守护进程的 AI 助手在拿到数据之后如何处理(例如将截图发送给某个 LLM 服务),由该助手或 LLM 提供商自身的隐私政策约束,**不在本政策范围内**。BrowserSkill 不参与那些通信。
|
||||
|
||||
## 7. 数据保留
|
||||
|
||||
@@ -76,7 +76,7 @@ BrowserSkill **不会**:
|
||||
|
||||
## 10. 安全性
|
||||
|
||||
由于本扩展仅与 `127.0.0.1` 通信,因此不会向网络暴露任何数据。但用户仍应避免在不可信的环境中运行 BrowserSkill —— 任何能够绑定到 `127.0.0.1:52800` 的本地进程理论上都可以向本扩展发送指令。请仅在您本人控制的机器上运行 BrowserSkill。
|
||||
由于本扩展仅与 `127.0.0.1` 通信,因此不会向网络暴露任何数据。但用户仍应避免在不可信的环境中运行 BrowserSkill —— 任何能够绑定到所配置回环端口的本地进程理论上都可以向本扩展发送指令。请仅在您本人控制的机器上运行 BrowserSkill。
|
||||
|
||||
## 11. 开源与可审计性
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ describe("BorrowConfirmationOverlay", () => {
|
||||
id: "req-1",
|
||||
isActiveTab: true,
|
||||
tabTitle: "Example",
|
||||
timeoutMs: 5000,
|
||||
timeoutMs: 60_000,
|
||||
onAllow,
|
||||
onDeny,
|
||||
},
|
||||
@@ -32,7 +32,7 @@ describe("BorrowConfirmationOverlay", () => {
|
||||
);
|
||||
|
||||
expect(screen.getByText("允许借用标签页?")).toBeTruthy();
|
||||
for (let i = 0; i < 5; i++) {
|
||||
for (let i = 0; i < 60; i++) {
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
});
|
||||
@@ -66,7 +66,7 @@ describe("BorrowConfirmationOverlay", () => {
|
||||
id: "req-fallback",
|
||||
isActiveTab: true,
|
||||
tabTitle: "Example",
|
||||
timeoutMs: 5000,
|
||||
timeoutMs: 60_000,
|
||||
onAllow,
|
||||
onDeny,
|
||||
},
|
||||
@@ -74,7 +74,7 @@ describe("BorrowConfirmationOverlay", () => {
|
||||
/>,
|
||||
);
|
||||
|
||||
for (let i = 0; i < 5; i++) {
|
||||
for (let i = 0; i < 60; i++) {
|
||||
await act(async () => {
|
||||
await vi.advanceTimersByTimeAsync(1000);
|
||||
});
|
||||
|
||||
@@ -4,7 +4,9 @@ import { ConnectionController } from "@/lib/connection-controller";
|
||||
import { startHeartbeat } from "@/lib/heartbeat";
|
||||
import {
|
||||
getConnectionEnabled,
|
||||
getDaemonPort,
|
||||
setConnectionEnabled as persistConnectionEnabled,
|
||||
STORAGE_KEYS,
|
||||
setLabel,
|
||||
} from "@/lib/instance-id";
|
||||
import { startKeepalive } from "@/lib/keepalive";
|
||||
@@ -41,6 +43,7 @@ import {
|
||||
attachRecordStepListener,
|
||||
type RecordRuntimeDeps,
|
||||
} from "@/tools/record";
|
||||
import { resolveDaemonWsUrl } from "@/transport/daemon-endpoint";
|
||||
import { detectBrowserMeta } from "@/transport/handshake";
|
||||
import type { Transport } from "@/transport/transport";
|
||||
import { WSTransport } from "@/transport/ws-transport";
|
||||
@@ -54,6 +57,27 @@ export default defineBackground(() => {
|
||||
let overlayGeneration = 0;
|
||||
const controlModes = new Map<string, OverlayMode>();
|
||||
|
||||
async function applyDaemonPort(port: number): Promise<void> {
|
||||
const url = resolveDaemonWsUrl(port);
|
||||
if (!transport.setUrl(url)) return;
|
||||
await transport.disconnect();
|
||||
if (!controller.isConnectionEnabled) return;
|
||||
try {
|
||||
await transport.connect();
|
||||
} catch (err) {
|
||||
console.debug("[browser-skill] reconnect after port change failed", err);
|
||||
}
|
||||
}
|
||||
|
||||
if (typeof chrome !== "undefined" && chrome.storage?.onChanged) {
|
||||
chrome.storage.onChanged.addListener((changes, areaName) => {
|
||||
if (areaName !== "local") return;
|
||||
const change = changes[STORAGE_KEYS.DAEMON_PORT];
|
||||
if (!change || typeof change.newValue !== "number") return;
|
||||
void applyDaemonPort(change.newValue);
|
||||
});
|
||||
}
|
||||
|
||||
function setControlMode(sessionId: string, mode: OverlayMode): void {
|
||||
if (controlModes.get(sessionId) === mode) return;
|
||||
controlModes.set(sessionId, mode);
|
||||
@@ -279,6 +303,8 @@ export default defineBackground(() => {
|
||||
|
||||
void (async () => {
|
||||
const connectionEnabled = await getConnectionEnabled();
|
||||
const port = await getDaemonPort();
|
||||
transport.setUrl(resolveDaemonWsUrl(port));
|
||||
await controller.attach(transport, detectBrowserMeta(), connectionEnabled, {
|
||||
beforeDisconnect: async () => {
|
||||
const report = await cleanupAfterDisconnect();
|
||||
@@ -345,11 +371,6 @@ export default defineBackground(() => {
|
||||
if (msg && typeof msg === "object" && "kind" in msg) {
|
||||
if (msg.kind === "set_label") {
|
||||
void setLabel(msg.value).then(() => controller.refreshLabel());
|
||||
} else if (msg.kind === "set_port") {
|
||||
// Placeholder for the future custom-port UI; warn loudly so
|
||||
// any reintroduced popup control is caught instead of
|
||||
// silently doing nothing (review M4/M5 C2).
|
||||
console.warn("[browser-skill] set_port is not wired yet; ignoring", msg.value);
|
||||
} else if (msg.kind === "set_connection_enabled") {
|
||||
void controller
|
||||
.setConnectionEnabled(msg.value)
|
||||
|
||||
@@ -2,6 +2,7 @@ import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/re
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { SnapshotInfo } from "@/lib/connection-controller";
|
||||
import { STORAGE_KEYS } from "@/lib/instance-id";
|
||||
import { DEFAULT_DAEMON_PORT } from "@/transport/daemon-endpoint";
|
||||
import { EXTENSION_VERSION } from "@/transport/handshake";
|
||||
import { App } from "./App";
|
||||
import { useConnectionState } from "./use-connection-state";
|
||||
@@ -66,9 +67,32 @@ describe("App", () => {
|
||||
render(<App />);
|
||||
|
||||
expect(screen.getByText("未连接")).toBeTruthy();
|
||||
expect(screen.getByText("无法连接,请确认 daemon 已启动且端口一致。")).toBeTruthy();
|
||||
expect(screen.queryByText("请先打开 BrowserSkill。")).toBeNull();
|
||||
});
|
||||
|
||||
it("shows the connection switch off and hides transport errors when disconnected", () => {
|
||||
mockUseConnectionState.mockReturnValue({
|
||||
snapshot: {
|
||||
...baseSnapshot,
|
||||
lastError: "[WSTransport] disconnect during connect",
|
||||
},
|
||||
statusState: "disconnected",
|
||||
setLabel,
|
||||
setConnectionEnabled,
|
||||
});
|
||||
|
||||
render(<App />);
|
||||
|
||||
expect(screen.getByText("未连接")).toBeTruthy();
|
||||
expect(screen.getByText("无法连接,请确认 daemon 已启动且端口一致。")).toBeTruthy();
|
||||
expect(screen.queryByText("端口不匹配")).toBeNull();
|
||||
expect(
|
||||
screen.getByRole("switch", { name: "BrowserSkill 连接" }).getAttribute("aria-checked"),
|
||||
).toBe("false");
|
||||
expect(screen.queryByText("[WSTransport] disconnect during connect")).toBeNull();
|
||||
});
|
||||
|
||||
it("does not render record UI on the main view", () => {
|
||||
render(<App />);
|
||||
|
||||
@@ -136,6 +160,13 @@ describe("App", () => {
|
||||
});
|
||||
|
||||
it("renders the connection toggle with switch semantics", () => {
|
||||
mockUseConnectionState.mockReturnValue({
|
||||
snapshot: { ...baseSnapshot, state: "connected" },
|
||||
statusState: "connected",
|
||||
setLabel,
|
||||
setConnectionEnabled,
|
||||
});
|
||||
|
||||
render(<App />);
|
||||
|
||||
const toggle = screen.getByRole("switch", { name: "BrowserSkill 连接" });
|
||||
@@ -143,6 +174,13 @@ describe("App", () => {
|
||||
});
|
||||
|
||||
it("calls setConnectionEnabled(false) when the toggle is turned off", () => {
|
||||
mockUseConnectionState.mockReturnValue({
|
||||
snapshot: { ...baseSnapshot, state: "connected" },
|
||||
statusState: "connected",
|
||||
setLabel,
|
||||
setConnectionEnabled,
|
||||
});
|
||||
|
||||
render(<App />);
|
||||
|
||||
fireEvent.click(screen.getByRole("switch", { name: "BrowserSkill 连接" }));
|
||||
@@ -160,6 +198,7 @@ describe("App", () => {
|
||||
render(<App />);
|
||||
|
||||
expect(screen.getByText("连接已关闭")).toBeTruthy();
|
||||
expect(screen.queryByText("无法连接,请确认 daemon 已启动且端口一致。")).toBeNull();
|
||||
expect(
|
||||
screen.getByRole("switch", { name: "BrowserSkill 连接" }).getAttribute("aria-checked"),
|
||||
).toBe("false");
|
||||
@@ -247,6 +286,41 @@ describe("App", () => {
|
||||
expect(copyButton.getAttribute("disabled")).not.toBeNull();
|
||||
expect(screen.getByText("连接后可用")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("treats protocol drift as still connected and prompts an upgrade", async () => {
|
||||
mockUseConnectionState.mockReturnValue({
|
||||
snapshot: {
|
||||
...baseSnapshot,
|
||||
state: "version_skew",
|
||||
instanceId: "03c3e47f",
|
||||
handshake: {
|
||||
server: "bh",
|
||||
version: mockDaemonVersion,
|
||||
protocol_version: "1.0",
|
||||
},
|
||||
},
|
||||
statusState: "version_skew",
|
||||
setLabel,
|
||||
setConnectionEnabled,
|
||||
});
|
||||
|
||||
render(<App />);
|
||||
|
||||
expect(screen.getByText("已连接")).toBeTruthy();
|
||||
expect(screen.getByText("可升级")).toBeTruthy();
|
||||
expect(screen.queryByText("协议不一致")).toBeNull();
|
||||
expect(screen.queryByText("Action needed")).toBeNull();
|
||||
expect(screen.queryByText("兼容")).toBeNull();
|
||||
const warning = screen.getByText(/协议版本不同,请及时升级/);
|
||||
expect(warning.textContent).toContain("CLI 协议");
|
||||
expect(warning.textContent).toContain("扩展协议");
|
||||
|
||||
openRecordView();
|
||||
const copyButton = screen.getByRole("button", { name: "复制录制指令" });
|
||||
expect(copyButton.getAttribute("disabled")).toBeNull();
|
||||
fireEvent.click(copyButton);
|
||||
expect(navigator.clipboard.writeText).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("control hints toggle", () => {
|
||||
@@ -319,14 +393,20 @@ describe("control hints toggle", () => {
|
||||
|
||||
const info = await screen.findByRole("button", { name: "控制提示说明" });
|
||||
expect(info).toBeTruthy();
|
||||
const tooltip = screen.getByRole("tooltip");
|
||||
expect(tooltip.textContent).toBe("Agent 控制页面时显示提示条和橙色闪光。");
|
||||
const tooltip = screen.getByText("Agent 控制页面时显示提示条和橙色闪光。");
|
||||
expect(tooltip.getAttribute("role")).toBe("tooltip");
|
||||
// Hidden until the info button is hovered or focused.
|
||||
expect(tooltip.className).toContain("opacity-0");
|
||||
});
|
||||
|
||||
it("uses the same switch component and size for both settings rows", async () => {
|
||||
stubChromeStorage();
|
||||
mockUseConnectionState.mockReturnValue({
|
||||
snapshot: { ...baseSnapshot, state: "connected" },
|
||||
statusState: "connected",
|
||||
setLabel: vi.fn(),
|
||||
setConnectionEnabled: vi.fn(),
|
||||
});
|
||||
|
||||
render(<App />);
|
||||
|
||||
@@ -339,3 +419,109 @@ describe("control hints toggle", () => {
|
||||
expect(hintsToggle.className).toBe(connectionToggle.className);
|
||||
});
|
||||
});
|
||||
|
||||
describe("daemon port input", () => {
|
||||
function stubChromeStorage(initial: Record<string, unknown> = {}) {
|
||||
const store = { ...initial };
|
||||
vi.stubGlobal("chrome", {
|
||||
runtime: { lastError: undefined },
|
||||
storage: {
|
||||
local: {
|
||||
get: (keys: string | string[], cb: (items: Record<string, unknown>) => void) => {
|
||||
const items: Record<string, unknown> = {};
|
||||
for (const k of Array.isArray(keys) ? keys : [keys]) {
|
||||
if (k in store) items[k] = store[k];
|
||||
}
|
||||
cb(items);
|
||||
},
|
||||
set: (items: Record<string, unknown>, cb?: () => void) => {
|
||||
Object.assign(store, items);
|
||||
cb?.();
|
||||
},
|
||||
},
|
||||
onChanged: {
|
||||
addListener: vi.fn(),
|
||||
removeListener: vi.fn(),
|
||||
},
|
||||
},
|
||||
});
|
||||
return store;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it("prefills the port from storage", async () => {
|
||||
stubChromeStorage({ [STORAGE_KEYS.DAEMON_PORT]: 53200 });
|
||||
|
||||
render(<App />);
|
||||
|
||||
const input = await screen.findByRole("textbox", { name: "连接端口" });
|
||||
await waitFor(() => expect((input as HTMLInputElement).value).toBe("53200"));
|
||||
});
|
||||
|
||||
it("persists a valid port on blur", async () => {
|
||||
const store = stubChromeStorage();
|
||||
|
||||
render(<App />);
|
||||
|
||||
const input = await screen.findByRole("textbox", { name: "连接端口" });
|
||||
fireEvent.change(input, { target: { value: "53200" } });
|
||||
fireEvent.blur(input);
|
||||
|
||||
expect(store[STORAGE_KEYS.DAEMON_PORT]).toBe(53200);
|
||||
expect((input as HTMLInputElement).value).toBe("53200");
|
||||
});
|
||||
|
||||
it("persists a valid port when Enter blurs the field", async () => {
|
||||
const store = stubChromeStorage();
|
||||
|
||||
render(<App />);
|
||||
|
||||
const input = await screen.findByRole("textbox", { name: "连接端口" });
|
||||
fireEvent.change(input, { target: { value: "53200" } });
|
||||
fireEvent.keyDown(input, { key: "Enter" });
|
||||
|
||||
expect(store[STORAGE_KEYS.DAEMON_PORT]).toBe(53200);
|
||||
});
|
||||
|
||||
it("shows an error and does not write invalid ports", async () => {
|
||||
const store = stubChromeStorage();
|
||||
|
||||
render(<App />);
|
||||
|
||||
const input = await screen.findByRole("textbox", { name: "连接端口" });
|
||||
fireEvent.change(input, { target: { value: "abc" } });
|
||||
fireEvent.blur(input);
|
||||
|
||||
expect(screen.getByText("请输入 1 到 65535 之间的端口号。")).toBeTruthy();
|
||||
expect(store[STORAGE_KEYS.DAEMON_PORT]).toBeUndefined();
|
||||
});
|
||||
|
||||
it("stores the default port when the field is cleared", async () => {
|
||||
const store = stubChromeStorage({ [STORAGE_KEYS.DAEMON_PORT]: 53200 });
|
||||
|
||||
render(<App />);
|
||||
|
||||
const input = await screen.findByRole("textbox", { name: "连接端口" });
|
||||
fireEvent.change(input, { target: { value: "" } });
|
||||
fireEvent.blur(input);
|
||||
|
||||
expect(store[STORAGE_KEYS.DAEMON_PORT]).toBe(DEFAULT_DAEMON_PORT);
|
||||
expect((input as HTMLInputElement).value).toBe(String(DEFAULT_DAEMON_PORT));
|
||||
});
|
||||
|
||||
it("keeps the port hint copy in an accessible info tooltip", async () => {
|
||||
stubChromeStorage();
|
||||
|
||||
render(<App />);
|
||||
|
||||
const info = await screen.findByRole("button", { name: "连接端口说明" });
|
||||
expect(info).toBeTruthy();
|
||||
const tooltip = screen.getByText("扩展通过此端口连接本机 daemon。非必要请勿修改。");
|
||||
expect(tooltip.getAttribute("role")).toBe("tooltip");
|
||||
expect(tooltip.className).toContain("opacity-0");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -15,6 +15,7 @@ import { POPUP_FEATURES, type PopupView } from "./features";
|
||||
import { Switch } from "./switch";
|
||||
import { type PopupStatusState, useConnectionState } from "./use-connection-state";
|
||||
import { useControlHintsHidden } from "./use-control-hints-hidden";
|
||||
import { useDaemonPort } from "./use-daemon-port";
|
||||
|
||||
const STATE_LABEL_KEYS = {
|
||||
disconnected: "popup.stateLabel.disconnected",
|
||||
@@ -41,6 +42,12 @@ export function App() {
|
||||
const { t } = useTranslation("extension");
|
||||
const { snapshot, statusState, setConnectionEnabled } = useConnectionState();
|
||||
const [controlHintsHidden, setControlHintsHidden] = useControlHintsHidden();
|
||||
const {
|
||||
draft: daemonPortDraft,
|
||||
setDraft: setDaemonPortDraft,
|
||||
commit: commitDaemonPort,
|
||||
invalid: daemonPortInvalid,
|
||||
} = useDaemonPort();
|
||||
const [view, setView] = useState<PopupView>("main");
|
||||
const [copiedInstanceId, setCopiedInstanceId] = useState(false);
|
||||
const [purposeDraft, setPurposeDraft] = useState("");
|
||||
@@ -79,6 +86,8 @@ export function App() {
|
||||
}, [copiedTick]);
|
||||
|
||||
const isSkewed = statusState === "version_skew";
|
||||
const isDisconnected = statusState === "disconnected";
|
||||
const connectionLive = statusState === "connected" || isSkewed;
|
||||
const daemonVersion = snapshot.handshake?.version ?? "—";
|
||||
const daemonProtocol = snapshot.handshake?.protocol_version ?? "—";
|
||||
const extensionVersion = snapshot.extensionVersion || "—";
|
||||
@@ -90,7 +99,7 @@ export function App() {
|
||||
setCopiedInstanceId(true);
|
||||
};
|
||||
|
||||
const recordReady = statusState === "connected" && Boolean(snapshot.instanceId);
|
||||
const recordReady = connectionLive && Boolean(snapshot.instanceId);
|
||||
const recordPurpose = purposeDraft.trim();
|
||||
const recordStartUrl = startUrlDraft.trim();
|
||||
const recordCommand = snapshot.instanceId
|
||||
@@ -197,21 +206,29 @@ export function App() {
|
||||
{t(STATE_BADGE_KEYS[statusState])}
|
||||
</Badge>
|
||||
<Switch
|
||||
checked={snapshot.connectionEnabled}
|
||||
checked={snapshot.connectionEnabled && !isDisconnected}
|
||||
onCheckedChange={setConnectionEnabled}
|
||||
aria-label={t("popup.connectionToggleTitle")}
|
||||
data-slot="popup-connection-toggle"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
{isDisconnected && (
|
||||
<p
|
||||
className="mt-2 text-xs leading-snug text-muted-foreground"
|
||||
data-slot="popup-daemon-unreachable"
|
||||
>
|
||||
{t("popup.daemonUnreachable")}
|
||||
</p>
|
||||
)}
|
||||
{isSkewed && (
|
||||
<p
|
||||
className="mt-2 text-xs leading-snug text-amber-600 dark:text-amber-400"
|
||||
className="mt-2 text-xs leading-snug text-muted-foreground"
|
||||
data-slot="popup-version-skew-warning"
|
||||
>
|
||||
{t("popup.versionSkewWarning", {
|
||||
extensionProtocol: PROTOCOL_VERSION,
|
||||
daemonProtocol,
|
||||
cliProtocol: daemonProtocol,
|
||||
})}
|
||||
</p>
|
||||
)}
|
||||
@@ -252,7 +269,66 @@ export function App() {
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{snapshot.lastError && (
|
||||
<section
|
||||
className="rounded-xl border border-border/80 bg-card/60 px-3 py-2.5"
|
||||
data-slot="popup-daemon-port-card"
|
||||
>
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<span className="flex min-w-0 items-center gap-1">
|
||||
<Label
|
||||
htmlFor="bh-daemon-port"
|
||||
className="truncate text-sm font-medium leading-normal"
|
||||
>
|
||||
{t("popup.daemonPortLabel")}
|
||||
</Label>
|
||||
<span className="group relative inline-flex shrink-0">
|
||||
<button
|
||||
type="button"
|
||||
aria-label={t("popup.daemonPortInfoLabel")}
|
||||
data-slot="popup-daemon-port-info"
|
||||
className="flex size-4 items-center justify-center rounded-full text-muted-foreground/70 transition-colors hover:text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring/50"
|
||||
>
|
||||
<RiInformationLine className="size-3.5" aria-hidden />
|
||||
</button>
|
||||
<span
|
||||
role="tooltip"
|
||||
className="pointer-events-none absolute bottom-full left-0 z-10 mb-1.5 w-56 whitespace-normal rounded-md bg-foreground/65 px-2 py-1 text-[10px] font-medium leading-snug text-background opacity-0 shadow-md backdrop-blur-sm transition-opacity group-hover:opacity-100 group-focus-within:opacity-100"
|
||||
>
|
||||
{t("popup.daemonPortHint")}
|
||||
</span>
|
||||
</span>
|
||||
</span>
|
||||
<Input
|
||||
id="bh-daemon-port"
|
||||
type="text"
|
||||
inputMode="numeric"
|
||||
value={daemonPortDraft}
|
||||
onChange={(event: ChangeEvent<HTMLInputElement>) =>
|
||||
setDaemonPortDraft(event.target.value)
|
||||
}
|
||||
onBlur={commitDaemonPort}
|
||||
onKeyDown={(event) => {
|
||||
if (event.key === "Enter") {
|
||||
event.preventDefault();
|
||||
commitDaemonPort();
|
||||
}
|
||||
}}
|
||||
className="mt-0 h-7.5 w-19 shrink-0 rounded-md px-2 py-0 text-center text-sm leading-none shadow-none"
|
||||
aria-invalid={daemonPortInvalid || undefined}
|
||||
data-slot="popup-daemon-port-input"
|
||||
/>
|
||||
</div>
|
||||
{daemonPortInvalid && (
|
||||
<p
|
||||
className="mt-1.5 text-[11px] leading-snug text-destructive"
|
||||
data-slot="popup-daemon-port-error"
|
||||
>
|
||||
{t("popup.daemonPortInvalid")}
|
||||
</p>
|
||||
)}
|
||||
</section>
|
||||
|
||||
{snapshot.lastError && !isDisconnected && (
|
||||
<div
|
||||
className="rounded-lg border border-destructive/25 bg-destructive/10 px-3 py-2 text-xs leading-snug text-destructive"
|
||||
data-slot="popup-error"
|
||||
|
||||
@@ -40,13 +40,8 @@ const FALLBACK_SNAPSHOT: SnapshotInfo = {
|
||||
/**
|
||||
* Live snapshot of the background's `ConnectionController` for popup UI.
|
||||
*
|
||||
* Posts `set_label` mutations back to the background; the incoming
|
||||
* snapshot stream reflects the canonical state. The `set_port` channel
|
||||
* is defined by the popup-bridge but is not currently routed to the
|
||||
* background — wiring it requires `ConnectionController` to swap its
|
||||
* Transport URL and persist the chosen port. Tracked for a follow-up
|
||||
* milestone; the popup intentionally does not expose a control until
|
||||
* the underlying support lands (review M4/M5 C2).
|
||||
* Posts `set_label` and `set_connection_enabled` mutations back to the
|
||||
* background; the incoming snapshot stream reflects the canonical state.
|
||||
*/
|
||||
export function useConnectionState(): {
|
||||
snapshot: SnapshotInfo;
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
import { useCallback, useEffect, useRef, useState } from "react";
|
||||
import { getDaemonPort, STORAGE_KEYS, setDaemonPort } from "@/lib/instance-id";
|
||||
import { parseDaemonPortInput } from "@/transport/daemon-endpoint";
|
||||
|
||||
/**
|
||||
* Popup-side daemon port preference stored in `chrome.storage.local`.
|
||||
* Commits on blur, Enter, and pagehide.
|
||||
*/
|
||||
export function useDaemonPort(): {
|
||||
draft: string;
|
||||
setDraft: (value: string) => void;
|
||||
commit: () => void;
|
||||
invalid: boolean;
|
||||
} {
|
||||
const [draft, setDraftState] = useState("");
|
||||
const [invalid, setInvalid] = useState(false);
|
||||
const draftRef = useRef(draft);
|
||||
draftRef.current = draft;
|
||||
|
||||
useEffect(() => {
|
||||
if (typeof chrome === "undefined" || !chrome.storage?.local) return undefined;
|
||||
let cancelled = false;
|
||||
getDaemonPort()
|
||||
.then((port) => {
|
||||
if (!cancelled) setDraftState(String(port));
|
||||
})
|
||||
.catch((err) => {
|
||||
console.debug("[browser-skill] daemon port read failed", err);
|
||||
});
|
||||
const onChanged = (changes: Record<string, chrome.storage.StorageChange>, areaName: string) => {
|
||||
if (areaName !== "local") return;
|
||||
const change = changes[STORAGE_KEYS.DAEMON_PORT];
|
||||
if (change && typeof change.newValue === "number") {
|
||||
setDraftState(String(change.newValue));
|
||||
setInvalid(false);
|
||||
}
|
||||
};
|
||||
chrome.storage.onChanged.addListener(onChanged);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
chrome.storage.onChanged.removeListener(onChanged);
|
||||
};
|
||||
}, []);
|
||||
|
||||
const commit = useCallback(() => {
|
||||
const parsed = parseDaemonPortInput(draftRef.current);
|
||||
if (parsed === null) {
|
||||
setInvalid(true);
|
||||
return;
|
||||
}
|
||||
setInvalid(false);
|
||||
setDraftState(String(parsed));
|
||||
if (typeof chrome === "undefined" || !chrome.storage?.local) return;
|
||||
setDaemonPort(parsed).catch((err) => {
|
||||
console.debug("[browser-skill] daemon port write failed", err);
|
||||
});
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
const onPageHide = () => commit();
|
||||
window.addEventListener("pagehide", onPageHide);
|
||||
return () => window.removeEventListener("pagehide", onPageHide);
|
||||
}, [commit]);
|
||||
|
||||
const setDraft = useCallback((value: string) => {
|
||||
setDraftState(value);
|
||||
setInvalid(false);
|
||||
}, []);
|
||||
|
||||
return { draft, setDraft, commit, invalid };
|
||||
}
|
||||
@@ -1,12 +1,15 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { DEFAULT_DAEMON_PORT } from "@/transport/daemon-endpoint";
|
||||
import {
|
||||
getConnectionEnabled,
|
||||
getControlHintsHidden,
|
||||
getDaemonPort,
|
||||
getLabel,
|
||||
getOrCreateInstanceId,
|
||||
STORAGE_KEYS,
|
||||
setConnectionEnabled,
|
||||
setControlHintsHidden,
|
||||
setDaemonPort,
|
||||
setLabel,
|
||||
} from "../instance-id";
|
||||
|
||||
@@ -117,4 +120,21 @@ describe("instance-id", () => {
|
||||
expect(store[STORAGE_KEYS.CONTROL_HINTS_HIDDEN]).toBe(true);
|
||||
expect(await getControlHintsHidden(backend)).toBe(true);
|
||||
});
|
||||
|
||||
it("getDaemonPort returns default when storage is empty", async () => {
|
||||
const { backend } = fakeStorage();
|
||||
expect(await getDaemonPort(backend)).toBe(DEFAULT_DAEMON_PORT);
|
||||
});
|
||||
|
||||
it("getDaemonPort normalizes invalid stored values to default", async () => {
|
||||
const { backend } = fakeStorage({ [STORAGE_KEYS.DAEMON_PORT]: 0 });
|
||||
expect(await getDaemonPort(backend)).toBe(DEFAULT_DAEMON_PORT);
|
||||
});
|
||||
|
||||
it("setDaemonPort persists the value retrievable by getDaemonPort", async () => {
|
||||
const { backend, store } = fakeStorage();
|
||||
await setDaemonPort(53200, backend);
|
||||
expect(store[STORAGE_KEYS.DAEMON_PORT]).toBe(53200);
|
||||
expect(await getDaemonPort(backend)).toBe(53200);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
import { normalizeDaemonPort } from "@/transport/daemon-endpoint";
|
||||
|
||||
const STORAGE_KEY = "bsk_instance_id";
|
||||
const LABEL_STORAGE_KEY = "bh_label";
|
||||
const CONNECTION_ENABLED_KEY = "bh_connection_enabled";
|
||||
const CONTROL_HINTS_HIDDEN_KEY = "bsk_control_hints_hidden";
|
||||
const DAEMON_PORT_KEY = "bsk_daemon_port";
|
||||
|
||||
export interface StorageBackend {
|
||||
get(keys: string | string[]): Promise<Record<string, unknown>>;
|
||||
@@ -127,9 +130,23 @@ export async function setControlHintsHidden(
|
||||
await storage.set({ [CONTROL_HINTS_HIDDEN_KEY]: hidden });
|
||||
}
|
||||
|
||||
/** Defaults to {@link DEFAULT_DAEMON_PORT} when unset or invalid. */
|
||||
export async function getDaemonPort(storage: StorageBackend = defaultStorage()): Promise<number> {
|
||||
const items = await storage.get(DAEMON_PORT_KEY);
|
||||
return normalizeDaemonPort(items[DAEMON_PORT_KEY]);
|
||||
}
|
||||
|
||||
export async function setDaemonPort(
|
||||
port: number,
|
||||
storage: StorageBackend = defaultStorage(),
|
||||
): Promise<void> {
|
||||
await storage.set({ [DAEMON_PORT_KEY]: port });
|
||||
}
|
||||
|
||||
export const STORAGE_KEYS = {
|
||||
INSTANCE_ID: STORAGE_KEY,
|
||||
LABEL: LABEL_STORAGE_KEY,
|
||||
CONNECTION_ENABLED: CONNECTION_ENABLED_KEY,
|
||||
CONTROL_HINTS_HIDDEN: CONTROL_HINTS_HIDDEN_KEY,
|
||||
DAEMON_PORT: DAEMON_PORT_KEY,
|
||||
} as const;
|
||||
|
||||
@@ -5,19 +5,14 @@ import type { SnapshotInfo } from "./connection-controller";
|
||||
* - Background pushes `{ kind: "snapshot", data: SnapshotInfo }`.
|
||||
* - Popup sends `{ kind: "set_label" }`, `{ kind: "set_connection_enabled" }`, etc.
|
||||
*
|
||||
* NOTE(review M4/M5 C2): the `set_port` variant is defined as a
|
||||
* placeholder so the future custom-port UI does not have to re-design
|
||||
* the bridge, but the popup does NOT render a control that emits it
|
||||
* yet and the background does NOT route it. The wiring requires
|
||||
* ConnectionController to dispose its current Transport and persist
|
||||
* the port in chrome.storage; tracked as a follow-up.
|
||||
* Daemon port preference is persisted via `chrome.storage.local` instead
|
||||
* of this bridge (see `use-daemon-port.ts`).
|
||||
*/
|
||||
|
||||
export const POPUP_PORT_NAME = "popup";
|
||||
|
||||
export type PopupOutbound =
|
||||
| { kind: "set_label"; value: string }
|
||||
| { kind: "set_port"; value: number }
|
||||
| { kind: "set_connection_enabled"; value: boolean };
|
||||
|
||||
export type PopupInbound = { kind: "snapshot"; data: SnapshotInfo };
|
||||
|
||||
@@ -127,6 +127,38 @@ describe("requestBorrowConfirmation", () => {
|
||||
expect(message.requestId.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("proactively focuses the user window before sending the overlay message", async () => {
|
||||
tabs.get.mockResolvedValueOnce({ id: 42, title: "Tab" });
|
||||
windows.getLastFocused.mockResolvedValueOnce(
|
||||
userWindowWithActiveTab({ windowId: 77, tabId: 7, url: "https://app.example/" }),
|
||||
);
|
||||
tabs.sendMessage.mockResolvedValueOnce({ type: "borrow-response", allowed: true });
|
||||
|
||||
const pending = requestBorrowConfirmation(42, {
|
||||
deps: { tabs, windows, notifications },
|
||||
});
|
||||
await vi.runAllTimersAsync();
|
||||
await pending;
|
||||
|
||||
expect(windows.update).toHaveBeenCalledWith(77, { focused: true });
|
||||
});
|
||||
|
||||
it("still proceeds when proactive window focus fails", async () => {
|
||||
tabs.get.mockResolvedValueOnce({ id: 42, title: "Tab" });
|
||||
windows.getLastFocused.mockResolvedValueOnce(
|
||||
userWindowWithActiveTab({ windowId: 77, tabId: 7, url: "https://app.example/" }),
|
||||
);
|
||||
windows.update.mockRejectedValueOnce(new Error("window gone"));
|
||||
tabs.sendMessage.mockResolvedValueOnce({ type: "borrow-response", allowed: true });
|
||||
|
||||
const pending = requestBorrowConfirmation(42, {
|
||||
deps: { tabs, windows, notifications },
|
||||
});
|
||||
await vi.runAllTimersAsync();
|
||||
|
||||
expect(await pending).toBe(true);
|
||||
});
|
||||
|
||||
it("sends confirmation to the borrow target when it is the active tab", async () => {
|
||||
tabs.get.mockResolvedValueOnce({ id: 42, title: "Active Tab" });
|
||||
windows.getLastFocused.mockResolvedValueOnce(
|
||||
|
||||
@@ -232,6 +232,107 @@ describe("ToolDispatcher", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("bypasses and restores the control overlay for an upload trigger click", async () => {
|
||||
const sendMessage = vi.fn(async () => undefined);
|
||||
vi.stubGlobal("chrome", {
|
||||
tabs: {
|
||||
get: vi.fn(async () => ({ id: 7, windowId: 4242, active: true })),
|
||||
query: vi.fn(async () => [{ id: 7, windowId: 4242, active: true }]),
|
||||
sendMessage,
|
||||
},
|
||||
});
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const ctx = await sessions.start("aa11");
|
||||
ctx.refStore.set("e1", 123, { tabId: 7 });
|
||||
const send = vi.fn(async <T>(_tabId: number, method: string, params?: object) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } } as T;
|
||||
}
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] } as T;
|
||||
}
|
||||
if (method === "DOM.resolveNode") {
|
||||
return { object: { objectId: "trigger-object" } } as T;
|
||||
}
|
||||
if (method === "DOM.describeNode") return { node: { backendNodeId: 456 } } as T;
|
||||
if (method === "Runtime.callFunctionOn") {
|
||||
const declaration = (params as { functionDeclaration?: string }).functionDeclaration ?? "";
|
||||
if (declaration.includes("count: state.inputs.length")) {
|
||||
return { result: { value: { count: 1, multiple: false } } } as T;
|
||||
}
|
||||
if (declaration.includes("inputs[0]")) {
|
||||
return { result: { objectId: "input-object" } } as T;
|
||||
}
|
||||
return { result: { value: true } } as T;
|
||||
}
|
||||
if (method === "Runtime.evaluate") {
|
||||
const expression = (params as { expression?: string }).expression ?? "";
|
||||
if (expression.includes("overlayDetails")) {
|
||||
return { result: { value: { hitIndex: 0 } } } as T;
|
||||
}
|
||||
if (expression.includes("overlayHostPresent")) {
|
||||
return {
|
||||
result: {
|
||||
value: { overlayHostPresent: true, overlayHostConnected: true },
|
||||
},
|
||||
} as T;
|
||||
}
|
||||
if (expression.includes("count:")) {
|
||||
return { result: { value: { count: 1, multiple: false } } } as T;
|
||||
}
|
||||
if (expression.includes("?.inputs[0]")) {
|
||||
return { result: { objectId: "input-object" } } as T;
|
||||
}
|
||||
return { result: { value: true } } as T;
|
||||
}
|
||||
return {} as T;
|
||||
});
|
||||
const cdp = {
|
||||
send,
|
||||
detachSession: vi.fn(async () => {}),
|
||||
ensureNetworkCapture: vi.fn(async () => {}),
|
||||
networkEntriesSince: vi.fn(() => ({
|
||||
tab_id: 7,
|
||||
entries: [],
|
||||
next_since: 0,
|
||||
truncated: false,
|
||||
})),
|
||||
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
setUserAgentOverride: vi.fn(async () => {}),
|
||||
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||
dispatcher.start();
|
||||
|
||||
deliver(
|
||||
makeRequest("tool.upload", {
|
||||
session_id: "aa11",
|
||||
ref: "@e1",
|
||||
files: [{ transfer_id: "tr_1", name: "test.png", staged_path: "/stage/test.png" }],
|
||||
}),
|
||||
);
|
||||
await flushMicrotasks();
|
||||
await vi.waitFor(() => expect(sent).toHaveLength(1));
|
||||
|
||||
expect(sent[0]).toMatchObject({ result: { tab_id: 7, file_names: ["test.png"] } });
|
||||
expect(sendMessage).toHaveBeenNthCalledWith(1, 7, {
|
||||
type: "bh-automation-bypass",
|
||||
enabled: true,
|
||||
});
|
||||
expect(sendMessage).toHaveBeenNthCalledWith(2, 7, {
|
||||
type: "bh-automation-bypass",
|
||||
enabled: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("detaches CDP state before stopping a session", async () => {
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
|
||||
@@ -0,0 +1,809 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { type DownloadsApi, handleDownload } from "../download";
|
||||
import { captureBrowserDownload } from "../download-capture";
|
||||
import { uploadThroughActivatedFileInput } from "../file-input-transaction";
|
||||
import type { ResolvedActionTarget } from "../interaction";
|
||||
import type { CdpRunner } from "../shared";
|
||||
import { handleUpload } from "../upload";
|
||||
|
||||
function sessions() {
|
||||
return new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 100),
|
||||
remove: vi.fn(async () => {}),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function tabsApi() {
|
||||
return {
|
||||
get: vi.fn(
|
||||
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
};
|
||||
}
|
||||
|
||||
function fakeEvent<T extends (...args: never[]) => unknown>() {
|
||||
const listeners = new Set<T>();
|
||||
return {
|
||||
addListener: (listener: T) => listeners.add(listener),
|
||||
removeListener: (listener: T) => listeners.delete(listener),
|
||||
emit: (...args: Parameters<T>) => {
|
||||
for (const listener of listeners) listener(...args);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function actionTarget(frameId?: string): ResolvedActionTarget {
|
||||
return {
|
||||
tab: { tabId: 4, windowId: 100, active: true },
|
||||
backendNodeId: 123,
|
||||
cdpTarget: { tabId: 4 },
|
||||
...(frameId ? { frameId } : {}),
|
||||
usedRef: "e3",
|
||||
};
|
||||
}
|
||||
|
||||
function uploadCdp(
|
||||
options: {
|
||||
inputCount?: number;
|
||||
multiple?: boolean;
|
||||
chooser?: { frameId?: string; backendNodeId?: number; mode?: string };
|
||||
pendingResolve?: boolean;
|
||||
} = {},
|
||||
) {
|
||||
const calls: Array<{ method: string; params?: object }> = [];
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
calls.push({ method, params });
|
||||
if (method === "Page.setInterceptFileChooserDialog") return {};
|
||||
if (method === "Page.getLayoutMetrics")
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||
if (method === "DOM.resolveNode") {
|
||||
if (options.pendingResolve) return new Promise<never>(() => {});
|
||||
return { object: { objectId: "trigger-object" } };
|
||||
}
|
||||
if (method === "Runtime.callFunctionOn") {
|
||||
const declaration = (params as { functionDeclaration?: string }).functionDeclaration ?? "";
|
||||
if (declaration.includes("Object.defineProperty")) return { result: { value: true } };
|
||||
if (declaration.includes("count: state.inputs.length")) {
|
||||
return {
|
||||
result: {
|
||||
value: {
|
||||
count: options.inputCount ?? 1,
|
||||
multiple: options.multiple ?? true,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
if (declaration.includes("inputs[0]")) {
|
||||
return { result: { objectId: "input-object" } };
|
||||
}
|
||||
return { result: { value: true } };
|
||||
}
|
||||
if (method === "DOM.describeNode") return { node: { backendNodeId: 456 } };
|
||||
if (
|
||||
method === "Input.dispatchMouseEvent" &&
|
||||
(params as { type?: string }).type === "mousePressed" &&
|
||||
options.chooser
|
||||
) {
|
||||
cdpEvent?.({ tabId: 4 }, "Page.fileChooserOpened", options.chooser);
|
||||
}
|
||||
return {};
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as unknown as CdpRunner["send"],
|
||||
onEvent: (handler) => {
|
||||
cdpEvent = handler;
|
||||
return { dispose: vi.fn() };
|
||||
},
|
||||
};
|
||||
return {
|
||||
calls,
|
||||
emitChooser: (event: { frameId?: string; backendNodeId?: number; mode?: string }) =>
|
||||
cdpEvent?.({ tabId: 4 }, "Page.fileChooserOpened", event),
|
||||
cdp,
|
||||
};
|
||||
}
|
||||
|
||||
describe("file transfer tools", () => {
|
||||
it("captures the file input activated by the requested click and injects only staged paths", async () => {
|
||||
const manager = sessions();
|
||||
const ctx = await manager.start("s1");
|
||||
ctx.refStore.set("e3", 123, { tabId: 4 });
|
||||
const { cdp, calls } = uploadCdp();
|
||||
|
||||
const result = await handleUpload(
|
||||
manager,
|
||||
{
|
||||
session_id: "s1",
|
||||
ref: "@e3",
|
||||
files: [
|
||||
{ transfer_id: "tr_1", name: "one.png", staged_path: "/private/stage/one" },
|
||||
{ transfer_id: "tr_2", name: "two.png", staged_path: "/private/stage/two" },
|
||||
],
|
||||
},
|
||||
{ cdp, tabsApi: tabsApi() },
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({ tab_id: 4, file_names: ["one.png", "two.png"] });
|
||||
expect(calls[0]).toEqual({
|
||||
method: "Page.setInterceptFileChooserDialog",
|
||||
params: { enabled: true },
|
||||
});
|
||||
expect(calls).toContainEqual({
|
||||
method: "Page.setInterceptFileChooserDialog",
|
||||
params: { enabled: false, cancel: true },
|
||||
});
|
||||
expect(calls).toContainEqual({
|
||||
method: "DOM.setFileInputFiles",
|
||||
params: { files: ["/private/stage/one", "/private/stage/two"], backendNodeId: 456 },
|
||||
});
|
||||
});
|
||||
|
||||
it("fails immediately when the trigger does not activate a file input", async () => {
|
||||
const { cdp } = uploadCdp({ inputCount: 0 });
|
||||
const result = await uploadThroughActivatedFileInput({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 100,
|
||||
trigger: async () => ({ tab_id: 4, x: 10, y: 10 }),
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "unsupported",
|
||||
data: { reason: "file_input_not_activated", phase: "resolve_input" },
|
||||
});
|
||||
});
|
||||
|
||||
it("fails before clicking when chooser interception is unavailable", async () => {
|
||||
const trigger = vi.fn(async () => ({ tab_id: 4, x: 10, y: 10 }));
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "Page.setInterceptFileChooserDialog") {
|
||||
throw new Error("method unavailable");
|
||||
}
|
||||
return {};
|
||||
}) as CdpRunner["send"],
|
||||
};
|
||||
|
||||
const result = await uploadThroughActivatedFileInput({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 100,
|
||||
trigger,
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { effect_state: "none", phase: "arm_interception" },
|
||||
});
|
||||
expect(trigger).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("uses an exact chooser event as an independent input-location signal", async () => {
|
||||
const { cdp, calls, emitChooser } = uploadCdp({ inputCount: 0 });
|
||||
const result = await uploadThroughActivatedFileInput({
|
||||
cdp,
|
||||
actionTarget: actionTarget("f1"),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 100,
|
||||
trigger: async () => {
|
||||
emitChooser({ frameId: "f1", backendNodeId: 789, mode: "selectSingle" });
|
||||
return { tab_id: 4, x: 10, y: 10 };
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({ multiple: false });
|
||||
expect(calls).toContainEqual({
|
||||
method: "DOM.setFileInputFiles",
|
||||
params: { files: ["/private/stage/one"], backendNodeId: 789 },
|
||||
});
|
||||
});
|
||||
|
||||
it("reports a File System Access picker without waiting for a timeout", async () => {
|
||||
const { cdp, emitChooser } = uploadCdp({
|
||||
inputCount: 0,
|
||||
});
|
||||
const result = await uploadThroughActivatedFileInput({
|
||||
cdp,
|
||||
actionTarget: actionTarget("f1"),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 100,
|
||||
trigger: async () => {
|
||||
emitChooser({ frameId: "f1", mode: "selectSingle" });
|
||||
return { tab_id: 4, x: 10, y: 10 };
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "unsupported",
|
||||
message: "upload trigger invoked a non-input file picker",
|
||||
data: { reason: "file_input_not_activated", phase: "resolve_input" },
|
||||
});
|
||||
});
|
||||
|
||||
it("bounds a stuck file-input probe", async () => {
|
||||
const { cdp } = uploadCdp({ pendingResolve: true });
|
||||
|
||||
const result = await uploadThroughActivatedFileInput({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 5,
|
||||
trigger: vi.fn(),
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "timeout",
|
||||
data: { reason: "file_input_probe_failed", phase: "arm_input_probe" },
|
||||
});
|
||||
});
|
||||
|
||||
it("does not dispatch browser work when upload is already cancelled", async () => {
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
const { cdp, calls } = uploadCdp();
|
||||
|
||||
const result = await uploadThroughActivatedFileInput({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 100,
|
||||
signal: controller.signal,
|
||||
trigger: vi.fn(),
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "cancelled",
|
||||
data: { effect_state: "none", phase: "arm_interception" },
|
||||
});
|
||||
expect(calls).not.toContainEqual({
|
||||
method: "Page.setInterceptFileChooserDialog",
|
||||
params: { enabled: true },
|
||||
});
|
||||
expect(calls.some((call) => call.method === "DOM.setFileInputFiles")).toBe(false);
|
||||
});
|
||||
|
||||
it("stops before file assignment when cancellation interrupts input resolution", async () => {
|
||||
const controller = new AbortController();
|
||||
const fixture = uploadCdp();
|
||||
const originalSend = fixture.cdp.send;
|
||||
let markProbeStarted!: () => void;
|
||||
const probeStarted = new Promise<void>((resolve) => {
|
||||
markProbeStarted = resolve;
|
||||
});
|
||||
fixture.cdp.send = vi.fn((tabId: number, method: string, params?: object) => {
|
||||
const declaration = (params as { functionDeclaration?: string } | undefined)
|
||||
?.functionDeclaration;
|
||||
if (
|
||||
method === "Runtime.callFunctionOn" &&
|
||||
declaration?.includes("count: state.inputs.length")
|
||||
) {
|
||||
markProbeStarted();
|
||||
return new Promise<never>(() => {});
|
||||
}
|
||||
return originalSend(tabId, method, params);
|
||||
}) as CdpRunner["send"];
|
||||
|
||||
const transaction = uploadThroughActivatedFileInput({
|
||||
cdp: fixture.cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 1_000,
|
||||
signal: controller.signal,
|
||||
trigger: async () => ({ tab_id: 4, x: 10, y: 10 }),
|
||||
});
|
||||
await probeStarted;
|
||||
controller.abort();
|
||||
const result = await transaction;
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "cancelled",
|
||||
data: { effect_state: "none", phase: "resolve_input" },
|
||||
});
|
||||
expect(fixture.calls.some((call) => call.method === "DOM.setFileInputFiles")).toBe(false);
|
||||
});
|
||||
|
||||
it("disarms chooser interception when its enable response times out", async () => {
|
||||
const calls: Array<{ method: string; params?: object }> = [];
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
calls.push({ method, params });
|
||||
if (
|
||||
method === "Page.setInterceptFileChooserDialog" &&
|
||||
(params as { enabled?: boolean }).enabled
|
||||
) {
|
||||
return new Promise<never>(() => {});
|
||||
}
|
||||
return {};
|
||||
}) as CdpRunner["send"],
|
||||
};
|
||||
|
||||
const result = await uploadThroughActivatedFileInput({
|
||||
cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 5,
|
||||
trigger: vi.fn(),
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "timeout",
|
||||
data: { effect_state: "none", phase: "arm_interception" },
|
||||
});
|
||||
expect(calls).toContainEqual({
|
||||
method: "Page.setInterceptFileChooserDialog",
|
||||
params: { enabled: false, cancel: true },
|
||||
});
|
||||
});
|
||||
|
||||
it("marks a timed-out file assignment unknown and detaches browser state", async () => {
|
||||
const fixture = uploadCdp();
|
||||
const originalSend = fixture.cdp.send;
|
||||
const detach = vi.fn(async () => {});
|
||||
fixture.cdp.detach = detach;
|
||||
fixture.cdp.send = vi.fn((tabId: number, method: string, params?: object) => {
|
||||
if (method === "DOM.setFileInputFiles") return new Promise<never>(() => {});
|
||||
return originalSend(tabId, method, params);
|
||||
}) as CdpRunner["send"];
|
||||
|
||||
const result = await uploadThroughActivatedFileInput({
|
||||
cdp: fixture.cdp,
|
||||
actionTarget: actionTarget(),
|
||||
files: ["/private/stage/one"],
|
||||
timeoutMs: 10,
|
||||
trigger: async () => ({ tab_id: 4, x: 10, y: 10 }),
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "timeout",
|
||||
data: { effect_state: "unknown", phase: "set_files" },
|
||||
});
|
||||
expect(detach).toHaveBeenCalledWith(4);
|
||||
});
|
||||
|
||||
it("routes one exact-target download through a browser-relative capability", async () => {
|
||||
const manager = sessions();
|
||||
const ctx = await manager.start("s1");
|
||||
ctx.refStore.set("e3", 123, { tabId: 4 });
|
||||
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||
const onDeterminingFilename =
|
||||
fakeEvent<
|
||||
(
|
||||
item: chrome.downloads.DownloadItem,
|
||||
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||
) => void | true
|
||||
>();
|
||||
const initial = {
|
||||
id: 9,
|
||||
url: "https://example.test/result.zip",
|
||||
finalUrl: "https://example.test/result.zip",
|
||||
filename: "result.zip",
|
||||
state: "in_progress",
|
||||
fileSize: -1,
|
||||
totalBytes: 12,
|
||||
mime: "application/zip",
|
||||
danger: "safe",
|
||||
} as chrome.downloads.DownloadItem;
|
||||
const completed = {
|
||||
...initial,
|
||||
filename: "/profile/Downloads/BrowserSkill/tr_1/result.zip",
|
||||
state: "complete",
|
||||
fileSize: 12,
|
||||
} as chrome.downloads.DownloadItem;
|
||||
const downloads: DownloadsApi = {
|
||||
onCreated,
|
||||
onChanged,
|
||||
onDeterminingFilename,
|
||||
search: vi.fn(async () => [completed]),
|
||||
cancel: vi.fn(async () => {}),
|
||||
removeFile: vi.fn(async () => {}),
|
||||
};
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
let suggested: chrome.downloads.DownloadFilenameSuggestion | undefined;
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
if (method === "Page.getLayoutMetrics")
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||
if (
|
||||
method === "Input.dispatchMouseEvent" &&
|
||||
(params as { type?: string }).type === "mousePressed"
|
||||
) {
|
||||
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
|
||||
url: initial.url,
|
||||
suggestedFilename: "result.zip",
|
||||
});
|
||||
await new Promise<void>((resolve) => {
|
||||
onDeterminingFilename.emit(initial, (value) => {
|
||||
suggested = value;
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
onCreated.emit(completed);
|
||||
}
|
||||
return {};
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as unknown as CdpRunner["send"],
|
||||
onEvent: (handler) => {
|
||||
cdpEvent = handler;
|
||||
return { dispose: vi.fn() };
|
||||
},
|
||||
};
|
||||
|
||||
const result = await handleDownload(
|
||||
manager,
|
||||
{ session_id: "s1", ref: "@e3", browser_relative_dir: "BrowserSkill/tr_1" },
|
||||
{ cdp, tabsApi: tabsApi(), downloads },
|
||||
);
|
||||
|
||||
expect(suggested).toEqual({
|
||||
filename: "BrowserSkill/tr_1/result.zip",
|
||||
conflictAction: "overwrite",
|
||||
});
|
||||
expect(result).toMatchObject({
|
||||
tab_id: 4,
|
||||
suggested_filename: "result.zip",
|
||||
byte_size: 12,
|
||||
browser_path: completed.filename,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not claim a download without an intent from the exact target", async () => {
|
||||
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||
const onDeterminingFilename =
|
||||
fakeEvent<
|
||||
(
|
||||
item: chrome.downloads.DownloadItem,
|
||||
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||
) => void | true
|
||||
>();
|
||||
const downloads: DownloadsApi = {
|
||||
onCreated,
|
||||
onChanged,
|
||||
onDeterminingFilename,
|
||||
search: vi.fn(async () => []),
|
||||
cancel: vi.fn(async () => {}),
|
||||
removeFile: vi.fn(async () => {}),
|
||||
};
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({})) as unknown as CdpRunner["send"],
|
||||
onEvent: (handler) => {
|
||||
cdpEvent = handler;
|
||||
return { dispose: vi.fn() };
|
||||
},
|
||||
};
|
||||
const unrelated = {
|
||||
id: 17,
|
||||
url: "https://example.test/unrelated.zip",
|
||||
finalUrl: "https://example.test/unrelated.zip",
|
||||
filename: "unrelated.zip",
|
||||
state: "in_progress",
|
||||
} as chrome.downloads.DownloadItem;
|
||||
let defaultSuggestionCalled = false;
|
||||
|
||||
const result = await captureBrowserDownload({
|
||||
cdp,
|
||||
target: { tabId: 4, sessionId: "expected-child" },
|
||||
downloads,
|
||||
browserRelativeDir: "BrowserSkill/tr_1",
|
||||
timeoutMs: 5,
|
||||
trigger: async () => {
|
||||
cdpEvent?.({ tabId: 4, sessionId: "other-child" }, "Page.downloadWillBegin", {
|
||||
url: unrelated.url,
|
||||
suggestedFilename: unrelated.filename,
|
||||
});
|
||||
onDeterminingFilename.emit(unrelated, (suggestion) => {
|
||||
defaultSuggestionCalled = suggestion === undefined;
|
||||
});
|
||||
return { tab_id: 4, x: 10, y: 10 };
|
||||
},
|
||||
});
|
||||
|
||||
expect(defaultSuggestionCalled).toBe(true);
|
||||
expect(downloads.cancel).not.toHaveBeenCalled();
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "download_capture_failed" },
|
||||
});
|
||||
});
|
||||
|
||||
it("correlates a filename candidate that arrives before the CDP intent", async () => {
|
||||
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||
const onDeterminingFilename =
|
||||
fakeEvent<
|
||||
(
|
||||
item: chrome.downloads.DownloadItem,
|
||||
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||
) => void | true
|
||||
>();
|
||||
const initial = {
|
||||
id: 21,
|
||||
url: "https://example.test/candidate-first.bin",
|
||||
finalUrl: "https://example.test/candidate-first.bin",
|
||||
filename: "candidate-first.bin",
|
||||
state: "in_progress",
|
||||
fileSize: -1,
|
||||
totalBytes: 4,
|
||||
bytesReceived: 0,
|
||||
} as chrome.downloads.DownloadItem;
|
||||
const complete = {
|
||||
...initial,
|
||||
state: "complete",
|
||||
fileSize: 4,
|
||||
} as chrome.downloads.DownloadItem;
|
||||
const downloads: DownloadsApi = {
|
||||
onCreated,
|
||||
onChanged,
|
||||
onDeterminingFilename,
|
||||
search: vi.fn(async () => [complete]),
|
||||
cancel: vi.fn(async () => {}),
|
||||
removeFile: vi.fn(async () => {}),
|
||||
};
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||
onEvent: (handler) => {
|
||||
cdpEvent = handler;
|
||||
return { dispose: vi.fn() };
|
||||
},
|
||||
};
|
||||
let suggestion: chrome.downloads.DownloadFilenameSuggestion | undefined;
|
||||
|
||||
const result = await captureBrowserDownload({
|
||||
cdp,
|
||||
target: { tabId: 4 },
|
||||
downloads,
|
||||
browserRelativeDir: "BrowserSkill/tr_21",
|
||||
timeoutMs: 1_000,
|
||||
trigger: async () => {
|
||||
const suggested = new Promise<void>((resolve) => {
|
||||
onDeterminingFilename.emit(initial, (value) => {
|
||||
suggestion = value;
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
|
||||
url: initial.url,
|
||||
suggestedFilename: initial.filename,
|
||||
});
|
||||
await suggested;
|
||||
onCreated.emit(complete);
|
||||
return { tab_id: 4, x: 10, y: 10 };
|
||||
},
|
||||
});
|
||||
|
||||
expect(suggestion).toEqual({
|
||||
filename: "BrowserSkill/tr_21/candidate-first.bin",
|
||||
conflictAction: "overwrite",
|
||||
});
|
||||
expect(result).toMatchObject({ item: { id: 21, state: "complete" } });
|
||||
});
|
||||
|
||||
it("removes a completed download when the final size exceeds the limit", async () => {
|
||||
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||
const onDeterminingFilename =
|
||||
fakeEvent<
|
||||
(
|
||||
item: chrome.downloads.DownloadItem,
|
||||
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||
) => void | true
|
||||
>();
|
||||
const initial = {
|
||||
id: 22,
|
||||
url: "https://example.test/oversized.bin",
|
||||
finalUrl: "https://example.test/oversized.bin",
|
||||
filename: "oversized.bin",
|
||||
state: "in_progress",
|
||||
fileSize: -1,
|
||||
totalBytes: -1,
|
||||
bytesReceived: 0,
|
||||
} as chrome.downloads.DownloadItem;
|
||||
const complete = {
|
||||
...initial,
|
||||
filename: "/profile/Downloads/BrowserSkill/tr_22/oversized.bin",
|
||||
state: "complete",
|
||||
fileSize: 8,
|
||||
totalBytes: 8,
|
||||
bytesReceived: 8,
|
||||
} as chrome.downloads.DownloadItem;
|
||||
const downloads: DownloadsApi = {
|
||||
onCreated,
|
||||
onChanged,
|
||||
onDeterminingFilename,
|
||||
search: vi.fn(async () => [complete]),
|
||||
cancel: vi.fn(async () => {}),
|
||||
removeFile: vi.fn(async () => {}),
|
||||
};
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||
onEvent: (handler) => {
|
||||
cdpEvent = handler;
|
||||
return { dispose: vi.fn() };
|
||||
},
|
||||
};
|
||||
|
||||
const result = await captureBrowserDownload({
|
||||
cdp,
|
||||
target: { tabId: 4 },
|
||||
downloads,
|
||||
browserRelativeDir: "BrowserSkill/tr_22",
|
||||
maxByteSize: 4,
|
||||
timeoutMs: 1_000,
|
||||
trigger: async () => {
|
||||
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
|
||||
url: initial.url,
|
||||
suggestedFilename: initial.filename,
|
||||
});
|
||||
await new Promise<void>((resolve) => {
|
||||
onDeterminingFilename.emit(initial, () => resolve());
|
||||
});
|
||||
onCreated.emit(initial);
|
||||
onChanged.emit({ id: initial.id, state: { current: "complete" } });
|
||||
return { tab_id: 4, x: 10, y: 10 };
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "download_capture_failed", effect_state: "committed" },
|
||||
});
|
||||
expect(downloads.removeFile).toHaveBeenCalledWith(initial.id);
|
||||
expect(downloads.cancel).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
["cancel resolves", false],
|
||||
["cancel reports that completion won", true],
|
||||
])("reconciles a download that completes while %s", async (_label, cancelThrows) => {
|
||||
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||
const onDeterminingFilename =
|
||||
fakeEvent<
|
||||
(
|
||||
item: chrome.downloads.DownloadItem,
|
||||
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||
) => void | true
|
||||
>();
|
||||
const initial = {
|
||||
id: 23,
|
||||
url: "https://example.test/racing.bin",
|
||||
finalUrl: "https://example.test/racing.bin",
|
||||
filename: "racing.bin",
|
||||
state: "in_progress",
|
||||
fileSize: -1,
|
||||
totalBytes: -1,
|
||||
bytesReceived: 0,
|
||||
} as chrome.downloads.DownloadItem;
|
||||
const complete = {
|
||||
...initial,
|
||||
filename: "/profile/Downloads/BrowserSkill/tr_23/racing.bin",
|
||||
state: "complete",
|
||||
fileSize: 4,
|
||||
totalBytes: 4,
|
||||
bytesReceived: 4,
|
||||
} as chrome.downloads.DownloadItem;
|
||||
const downloads: DownloadsApi = {
|
||||
onCreated,
|
||||
onChanged,
|
||||
onDeterminingFilename,
|
||||
search: vi.fn().mockResolvedValueOnce([initial]).mockResolvedValueOnce([complete]),
|
||||
cancel: vi.fn(async () => {
|
||||
if (cancelThrows) throw new Error("download already complete");
|
||||
}),
|
||||
removeFile: vi.fn(async () => {}),
|
||||
};
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||
onEvent: (handler) => {
|
||||
cdpEvent = handler;
|
||||
return { dispose: vi.fn() };
|
||||
},
|
||||
};
|
||||
|
||||
const result = await captureBrowserDownload({
|
||||
cdp,
|
||||
target: { tabId: 4 },
|
||||
downloads,
|
||||
browserRelativeDir: "BrowserSkill/tr_23",
|
||||
timeoutMs: 80,
|
||||
trigger: async () => {
|
||||
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
|
||||
url: initial.url,
|
||||
suggestedFilename: initial.filename,
|
||||
});
|
||||
await new Promise<void>((resolve) => {
|
||||
onDeterminingFilename.emit(initial, () => resolve());
|
||||
});
|
||||
onCreated.emit(initial);
|
||||
return { tab_id: 4, x: 10, y: 10 };
|
||||
},
|
||||
});
|
||||
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "download_capture_failed", effect_state: "committed" },
|
||||
});
|
||||
expect(downloads.cancel).toHaveBeenCalledWith(initial.id);
|
||||
expect(downloads.removeFile).toHaveBeenCalledWith(initial.id);
|
||||
expect(result).not.toMatchObject({ data: { cleanup_state: "failed" } });
|
||||
});
|
||||
|
||||
it("rejects ambiguous attribution without cancelling either unclaimed download", async () => {
|
||||
const onCreated = fakeEvent<(item: chrome.downloads.DownloadItem) => void>();
|
||||
const onChanged = fakeEvent<(delta: chrome.downloads.DownloadDelta) => void>();
|
||||
const onDeterminingFilename =
|
||||
fakeEvent<
|
||||
(
|
||||
item: chrome.downloads.DownloadItem,
|
||||
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||
) => void | true
|
||||
>();
|
||||
const downloads: DownloadsApi = {
|
||||
onCreated,
|
||||
onChanged,
|
||||
onDeterminingFilename,
|
||||
search: vi.fn(async () => []),
|
||||
cancel: vi.fn(async () => {}),
|
||||
removeFile: vi.fn(async () => {}),
|
||||
};
|
||||
let cdpEvent: Parameters<NonNullable<CdpRunner["onEvent"]>>[0] | undefined;
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||
onEvent: (handler) => {
|
||||
cdpEvent = handler;
|
||||
return { dispose: vi.fn() };
|
||||
},
|
||||
};
|
||||
const defaults: number[] = [];
|
||||
const candidate = (id: number) =>
|
||||
({
|
||||
id,
|
||||
url: "https://example.test/same.bin",
|
||||
finalUrl: "https://example.test/same.bin",
|
||||
filename: "same.bin",
|
||||
state: "in_progress",
|
||||
}) as chrome.downloads.DownloadItem;
|
||||
|
||||
const result = await captureBrowserDownload({
|
||||
cdp,
|
||||
target: { tabId: 4 },
|
||||
downloads,
|
||||
browserRelativeDir: "BrowserSkill/tr_ambiguous",
|
||||
timeoutMs: 100,
|
||||
trigger: async () => {
|
||||
cdpEvent?.({ tabId: 4 }, "Page.downloadWillBegin", {
|
||||
url: "https://example.test/same.bin",
|
||||
suggestedFilename: "same.bin",
|
||||
});
|
||||
for (const id of [31, 32]) {
|
||||
onDeterminingFilename.emit(candidate(id), (value) => {
|
||||
if (value === undefined) defaults.push(id);
|
||||
});
|
||||
}
|
||||
return { tab_id: 4, x: 10, y: 10 };
|
||||
},
|
||||
});
|
||||
|
||||
expect(defaults.sort()).toEqual([31, 32]);
|
||||
expect(downloads.cancel).not.toHaveBeenCalled();
|
||||
expect(result).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { effect_state: "unknown", phase: "attribution" },
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1904,36 +1904,41 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 20,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "button",
|
||||
attrs: {},
|
||||
rect: { x: 20, y: 20, w: 120, h: 40 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
const scene = buildVomScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "button",
|
||||
attrs: {},
|
||||
rect: { x: 20, y: 20, w: 120, h: 40 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 20, triggerAction: "hover", subItems: ["Shoes", "Bags"] },
|
||||
@@ -1957,46 +1962,51 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 21,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
const scene = buildVomScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(renderVom(scene).text).toContain('@e1 button "image" [hover first: My profile]');
|
||||
});
|
||||
@@ -2017,48 +2027,53 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 21,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] },
|
||||
{ triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] },
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
const scene = buildVomScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
surfaceProbes: [
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["My profile"] },
|
||||
{ triggerBackendNodeId: 20, triggerAction: "hover", subItems: ["Sign out"] },
|
||||
{ triggerBackendNodeId: 21, triggerAction: "hover", subItems: ["Settings"] },
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 21, triggerAction: "hover", subItems: ["My profile"] },
|
||||
@@ -2081,54 +2096,59 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 21,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{
|
||||
triggerBackendNodeId: 20,
|
||||
triggerPoint: { x: 915, y: 25 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile", "Sign out"],
|
||||
},
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
},
|
||||
],
|
||||
});
|
||||
const scene = buildVomScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 20,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar" },
|
||||
rect: { x: 900, y: 10, w: 30, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
},
|
||||
{
|
||||
backendNodeId: 21,
|
||||
parentBackendNodeId: 20,
|
||||
tag: "div",
|
||||
attrs: { class: "tg-avatar__inner" },
|
||||
rect: { x: 902, y: 12, w: 26, h: 26 },
|
||||
paintOrder: 2,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
cursor: "pointer",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
surfaceProbes: [
|
||||
{
|
||||
triggerBackendNodeId: 20,
|
||||
triggerPoint: { x: 915, y: 25 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile", "Sign out"],
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(scene.surfaces).toEqual([
|
||||
{ triggerId: 20, triggerAction: "hover", subItems: ["My profile", "Sign out"] },
|
||||
@@ -2154,41 +2174,46 @@ describe("buildVomScene", () => {
|
||||
backendDOMNodeId: 42,
|
||||
},
|
||||
];
|
||||
const scene = buildVomScene(axNodes, {
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
surfaceProbes: [
|
||||
{
|
||||
triggerBackendNodeId: 999,
|
||||
triggerPoint: { x: 900, y: 20 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile"],
|
||||
},
|
||||
],
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 42,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "a",
|
||||
attrs: {},
|
||||
rect: { x: 860, y: 120, w: 40, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
});
|
||||
const scene = buildVomScene(
|
||||
axNodes,
|
||||
{
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
nodes: [
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: null,
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 1000, h: 800 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 42,
|
||||
parentBackendNodeId: 10,
|
||||
tag: "a",
|
||||
attrs: {},
|
||||
rect: { x: 860, y: 120, w: 40, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
surfaceProbes: [
|
||||
{
|
||||
triggerBackendNodeId: 999,
|
||||
triggerPoint: { x: 900, y: 20 },
|
||||
triggerAction: "hover",
|
||||
subItems: ["My profile"],
|
||||
},
|
||||
],
|
||||
},
|
||||
);
|
||||
|
||||
expect(scene.surfaces).toBeUndefined();
|
||||
expect(renderVom(scene).text).not.toContain("[hover first:");
|
||||
@@ -2606,7 +2631,106 @@ describe("handleSnapshot", () => {
|
||||
);
|
||||
});
|
||||
|
||||
it("enables conditional surface probing for semantic observe", async () => {
|
||||
it("hovers the page only after the accessibility tree has been captured", async () => {
|
||||
// Hovering can open menus and reflow the page. Probing between the DOM and
|
||||
// AX captures would leave the two halves of one observation describing the
|
||||
// page on either side of that change.
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const root: CdpAxNode = {
|
||||
nodeId: "1",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
name: { type: "computedString", value: "Example" },
|
||||
backendDOMNodeId: 100,
|
||||
childIds: ["2"],
|
||||
};
|
||||
const button: CdpAxNode = {
|
||||
nodeId: "2",
|
||||
parentId: "1",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Products" },
|
||||
backendDOMNodeId: 200,
|
||||
};
|
||||
const strings = ["body", "button", "position", "static", "pointer-events", "auto", "cursor"];
|
||||
const i = (s: string) => strings.indexOf(s);
|
||||
const methodOrder: string[] = [];
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
methodOrder.push(method);
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [root, button] };
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [i("body"), i("button")],
|
||||
backendNodeId: [100, 200],
|
||||
attributes: [[], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
[i("static"), i("auto"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[20, 20, 120, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (method === "Input.dispatchMouseEvent") return {};
|
||||
if (method === "Runtime.evaluate") {
|
||||
const expression = (params as { expression?: string } | undefined)?.expression ?? "";
|
||||
// Report the button as a `:hover` rule target so a real hover fires.
|
||||
if (expression.includes("document.styleSheets")) {
|
||||
return { result: { value: [{ x: 80, y: 40 }] } };
|
||||
}
|
||||
return { result: { value: [] } };
|
||||
}
|
||||
throw new Error(`unexpected CDP method ${method}`);
|
||||
});
|
||||
|
||||
const res = await handleObserve(
|
||||
sm,
|
||||
{ session_id: "aa11", probe_hover: true },
|
||||
{
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) =>
|
||||
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
const firstHover = methodOrder.indexOf("Input.dispatchMouseEvent");
|
||||
expect(firstHover).toBeGreaterThan(-1);
|
||||
expect(methodOrder.lastIndexOf("DOMSnapshot.captureSnapshot")).toBeLessThan(firstHover);
|
||||
expect(methodOrder.lastIndexOf("Accessibility.getFullAXTree")).toBeLessThan(firstHover);
|
||||
});
|
||||
|
||||
it("runs conditional surface probing only when observe opts in", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const root: CdpAxNode = {
|
||||
@@ -2662,30 +2786,46 @@ describe("handleSnapshot", () => {
|
||||
if (method === "Runtime.evaluate") return { result: { value: [] } };
|
||||
throw new Error(`unexpected CDP method ${method}`);
|
||||
});
|
||||
const res = await handleObserve(
|
||||
const deps = {
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
};
|
||||
|
||||
const withoutProbe = await handleObserve(
|
||||
sm,
|
||||
{ session_id: "aa11", debug_surfaces: true },
|
||||
{
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) =>
|
||||
({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
},
|
||||
deps,
|
||||
);
|
||||
if ("code" in withoutProbe) {
|
||||
throw new Error(`unexpected error: ${JSON.stringify(withoutProbe)}`);
|
||||
}
|
||||
expect(withoutProbe.hover_probe).toBeUndefined();
|
||||
expect(send).not.toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
expect.objectContaining({ returnByValue: true }),
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.debug).toEqual({ surface_probes: [] });
|
||||
const withProbe = await handleObserve(
|
||||
sm,
|
||||
{ session_id: "aa11", debug_surfaces: true, probe_hover: true },
|
||||
deps,
|
||||
);
|
||||
if ("code" in withProbe) throw new Error(`unexpected error: ${JSON.stringify(withProbe)}`);
|
||||
expect(withProbe.debug).toEqual({ surface_probes: [] });
|
||||
expect(withProbe.hover_probe).toEqual({ performed: true, revealed_content: false });
|
||||
expect(send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
|
||||
@@ -24,7 +24,7 @@ export interface BorrowResponseMessage {
|
||||
allowed: boolean;
|
||||
}
|
||||
|
||||
export const CONFIRMATION_TIMEOUT_MS = 5000;
|
||||
export const CONFIRMATION_TIMEOUT_MS = 60_000;
|
||||
const EXIT_ANIMATION_MS = 150;
|
||||
/** Matches BorrowConfirmationOverlay progress ring/bar transition (duration-1000). */
|
||||
export const PROGRESS_TRANSITION_MS = 1000;
|
||||
@@ -480,6 +480,13 @@ export async function requestBorrowConfirmation(
|
||||
settle(false);
|
||||
}, BACKGROUND_TIMEOUT_MS);
|
||||
|
||||
// Bring the user window to the front so the overlay is visible even
|
||||
// when the Agent Window has stolen focus. This is fire-and-forget: if
|
||||
// it fails the overlay / OS notification still give the user a path.
|
||||
void windowsApi.update(notificationAnchor.windowId, { focused: true }).catch((err) => {
|
||||
console.debug("[bsk borrow] proactive focus of user window failed", err);
|
||||
});
|
||||
|
||||
// Surface the OS notification *before* messaging any candidate so the
|
||||
// user has a parallel signal even if every content script is missing.
|
||||
// The Allow / Deny buttons let the user authorize explicitly when the
|
||||
|
||||
@@ -4,6 +4,7 @@ import type { Transport } from "@/transport/transport";
|
||||
import type {
|
||||
ClickParams,
|
||||
ConsoleParams,
|
||||
DownloadParams,
|
||||
EmulateParams,
|
||||
EvaluateParams,
|
||||
FillParams,
|
||||
@@ -28,10 +29,12 @@ import type {
|
||||
ScreenshotParams,
|
||||
SelectParams,
|
||||
SnapshotParams,
|
||||
UploadParams,
|
||||
WaitForNavigationParams,
|
||||
} from "@/transport/types";
|
||||
import { isRequestFrame } from "@/transport/types";
|
||||
import { handleConsole } from "./console";
|
||||
import { handleDownload } from "./download";
|
||||
import { type EmulateCdpRunner, handleEmulate } from "./emulate";
|
||||
import { handleEvaluate } from "./evaluate";
|
||||
import { handleRequestHelp } from "./human-loop";
|
||||
@@ -79,6 +82,7 @@ import {
|
||||
type TabReturnParams,
|
||||
type TabSelectParams,
|
||||
} from "./tabs";
|
||||
import { handleUpload } from "./upload";
|
||||
import { handleWaitForNavigation } from "./waits";
|
||||
import { handleWindowResize, type WindowResizeParams } from "./window";
|
||||
|
||||
@@ -383,7 +387,11 @@ export class ToolDispatcher {
|
||||
? {
|
||||
cdp: this.cdp,
|
||||
tabsApi: chromeTabsCaptureApi,
|
||||
conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope),
|
||||
// Active hover probing is opt-in. A held hover latch still
|
||||
// suppresses it, because probing would move the cursor off
|
||||
// the element the caller is deliberately holding.
|
||||
conditionalSurfaceProbe:
|
||||
params.probe_hover === true && !this.hasHoverLatchForScope(hoverScope),
|
||||
hoverProbeBypassOverlay: bypassOverlay,
|
||||
}
|
||||
: undefined,
|
||||
@@ -513,6 +521,40 @@ export class ToolDispatcher {
|
||||
),
|
||||
signal,
|
||||
);
|
||||
case "tool.upload":
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as UploadParams,
|
||||
() =>
|
||||
this.cdp
|
||||
? handleUpload(this.sessions, req.params as UploadParams, {
|
||||
cdp: this.cdp,
|
||||
tabsApi: chromeTabsApi,
|
||||
signal,
|
||||
bypassOverlay,
|
||||
})
|
||||
: Promise.resolve({
|
||||
code: "unsupported",
|
||||
message: "upload requires CDP",
|
||||
} satisfies RpcError),
|
||||
signal,
|
||||
);
|
||||
case "tool.download":
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as DownloadParams,
|
||||
() =>
|
||||
this.cdp
|
||||
? handleDownload(this.sessions, req.params as DownloadParams, {
|
||||
cdp: this.cdp,
|
||||
tabsApi: chromeTabsApi,
|
||||
signal,
|
||||
bypassOverlay,
|
||||
})
|
||||
: Promise.resolve({
|
||||
code: "unsupported",
|
||||
message: "download requires CDP",
|
||||
} satisfies RpcError),
|
||||
signal,
|
||||
);
|
||||
case "tool.evaluate":
|
||||
return handleEvaluate(
|
||||
this.sessions,
|
||||
@@ -719,6 +761,8 @@ function sessionIdForBrowserControlMethod(req: RequestFrame): string | null {
|
||||
case "tool.fill":
|
||||
case "tool.press":
|
||||
case "tool.select":
|
||||
case "tool.upload":
|
||||
case "tool.download":
|
||||
case "tool.evaluate":
|
||||
case "tool.observe":
|
||||
case "tool.request_help":
|
||||
|
||||
@@ -0,0 +1,375 @@
|
||||
// Order-independent coordinator for one browser download. CDP supplies the
|
||||
// exact target/frame intent while chrome.downloads supplies the download id
|
||||
// and filename routing hook; neither event is assumed to arrive first.
|
||||
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { ClickResult, RpcError, TransferEffectState } from "@/transport/types";
|
||||
import { transferError } from "./errors";
|
||||
import { type CdpRunner, isRpcError } from "./shared";
|
||||
|
||||
const CORRELATION_GRACE_MS = 750;
|
||||
const UNIQUE_SETTLE_MS = 50;
|
||||
const CLEANUP_RECONCILE_MS = 1_000;
|
||||
const CLEANUP_POLL_MS = 25;
|
||||
const SIZE_POLL_MS = 250;
|
||||
|
||||
type DeterminingFilenameListener = (
|
||||
item: chrome.downloads.DownloadItem,
|
||||
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void,
|
||||
) => void | true;
|
||||
|
||||
interface ListenerEvent<T> {
|
||||
addListener(listener: T): void;
|
||||
removeListener(listener: T): void;
|
||||
}
|
||||
|
||||
export interface DownloadsApi {
|
||||
onCreated: ListenerEvent<(item: chrome.downloads.DownloadItem) => void>;
|
||||
onChanged: ListenerEvent<(delta: chrome.downloads.DownloadDelta) => void>;
|
||||
onDeterminingFilename: ListenerEvent<DeterminingFilenameListener>;
|
||||
search(query: chrome.downloads.DownloadQuery): Promise<chrome.downloads.DownloadItem[]>;
|
||||
cancel(downloadId: number): Promise<void>;
|
||||
removeFile(downloadId: number): Promise<void>;
|
||||
}
|
||||
|
||||
export const chromeDownloadsApi: DownloadsApi = {
|
||||
get onCreated() {
|
||||
return chrome.downloads.onCreated;
|
||||
},
|
||||
get onChanged() {
|
||||
return chrome.downloads.onChanged;
|
||||
},
|
||||
get onDeterminingFilename() {
|
||||
return chrome.downloads.onDeterminingFilename;
|
||||
},
|
||||
search: (query) => chrome.downloads.search(query),
|
||||
cancel: (id) => chrome.downloads.cancel(id),
|
||||
removeFile: (id) => chrome.downloads.removeFile(id),
|
||||
};
|
||||
|
||||
export interface DownloadCaptureOptions {
|
||||
cdp: CdpRunner;
|
||||
target: CdpTarget;
|
||||
expectedFrameId?: string;
|
||||
downloads: DownloadsApi;
|
||||
browserRelativeDir: string;
|
||||
maxByteSize?: number;
|
||||
timeoutMs: number;
|
||||
signal?: AbortSignal;
|
||||
trigger(): Promise<ClickResult | RpcError>;
|
||||
}
|
||||
|
||||
export interface DownloadCaptureResult {
|
||||
click: ClickResult;
|
||||
item: chrome.downloads.DownloadItem;
|
||||
}
|
||||
|
||||
interface DownloadIntent {
|
||||
url: string;
|
||||
suggestedFilename: string;
|
||||
frameId?: string;
|
||||
}
|
||||
|
||||
interface DownloadCandidate {
|
||||
item: chrome.downloads.DownloadItem;
|
||||
suggest: (suggestion?: chrome.downloads.DownloadFilenameSuggestion) => void;
|
||||
suggested: boolean;
|
||||
graceTimer: ReturnType<typeof setTimeout>;
|
||||
}
|
||||
|
||||
function safeBasename(filename: string): string {
|
||||
const basename = filename.split(/[\\/]/).pop()?.trim();
|
||||
return basename && basename !== "." && basename !== ".." ? basename : "download";
|
||||
}
|
||||
|
||||
function sameTarget(source: { tabId?: number; sessionId?: string }, target: CdpTarget): boolean {
|
||||
return source.tabId === target.tabId && source.sessionId === target.sessionId;
|
||||
}
|
||||
|
||||
function matchesIntent(item: chrome.downloads.DownloadItem, intent: DownloadIntent): boolean {
|
||||
const urlMatches = item.url === intent.url || item.finalUrl === intent.url;
|
||||
return urlMatches && safeBasename(item.filename) === safeBasename(intent.suggestedFilename);
|
||||
}
|
||||
|
||||
function knownSize(item: chrome.downloads.DownloadItem): number | undefined {
|
||||
if (item.fileSize >= 0) return item.fileSize;
|
||||
if (item.totalBytes >= 0) return item.totalBytes;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function captureError(
|
||||
message: string,
|
||||
effectState: TransferEffectState,
|
||||
phase: string,
|
||||
cleanupFailed = false,
|
||||
): RpcError {
|
||||
return transferError("cdp_failed", "download_capture_failed", message, {
|
||||
effectState,
|
||||
phase,
|
||||
...(cleanupFailed ? { cleanupState: "failed" } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
async function cleanupClaimedDownload(downloads: DownloadsApi, downloadId: number): Promise<void> {
|
||||
const lookup = async () => (await downloads.search({ id: downloadId }))[0];
|
||||
let item = await lookup();
|
||||
if (!item || item.state === "interrupted") return;
|
||||
if (item.state === "complete") {
|
||||
await downloads.removeFile(downloadId);
|
||||
return;
|
||||
}
|
||||
|
||||
let cancelError: unknown;
|
||||
try {
|
||||
await downloads.cancel(downloadId);
|
||||
} catch (error) {
|
||||
cancelError = error;
|
||||
}
|
||||
|
||||
const deadline = Date.now() + CLEANUP_RECONCILE_MS;
|
||||
while (true) {
|
||||
item = await lookup();
|
||||
if (!item || item.state === "interrupted") return;
|
||||
if (item.state === "complete") {
|
||||
await downloads.removeFile(downloadId);
|
||||
return;
|
||||
}
|
||||
if (Date.now() >= deadline) {
|
||||
if (cancelError instanceof Error) throw cancelError;
|
||||
throw new Error("download remained in progress after cancellation");
|
||||
}
|
||||
await new Promise((resolve) =>
|
||||
setTimeout(resolve, Math.min(CLEANUP_POLL_MS, Math.max(1, deadline - Date.now()))),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function captureBrowserDownload(
|
||||
options: DownloadCaptureOptions,
|
||||
): Promise<DownloadCaptureResult | RpcError> {
|
||||
let click: ClickResult | undefined;
|
||||
let intent: DownloadIntent | undefined;
|
||||
let capturedId: number | undefined;
|
||||
let settled = false;
|
||||
let succeeded = false;
|
||||
let failureResult: RpcError | undefined;
|
||||
let uniquenessTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
let operationTimer: ReturnType<typeof setTimeout> | undefined;
|
||||
let sizePoll: ReturnType<typeof setInterval> | undefined;
|
||||
const candidates = new Map<number, DownloadCandidate>();
|
||||
const createdItems = new Map<number, chrome.downloads.DownloadItem>();
|
||||
|
||||
let resolveCompletion!: (item: chrome.downloads.DownloadItem) => void;
|
||||
let rejectCompletion!: (error: Error) => void;
|
||||
const completion = new Promise<chrome.downloads.DownloadItem>((resolve, reject) => {
|
||||
resolveCompletion = resolve;
|
||||
rejectCompletion = reject;
|
||||
});
|
||||
const fail = (error: Error) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
rejectCompletion(error);
|
||||
};
|
||||
const complete = (item: chrome.downloads.DownloadItem) => {
|
||||
if (settled) return;
|
||||
const size = knownSize(item);
|
||||
if (size !== undefined && options.maxByteSize !== undefined && size > options.maxByteSize) {
|
||||
fail(new Error(`download exceeds transfer limit ${options.maxByteSize}`));
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
resolveCompletion(item);
|
||||
};
|
||||
const suggestDefault = (candidate: DownloadCandidate) => {
|
||||
if (candidate.suggested) return;
|
||||
candidate.suggested = true;
|
||||
candidate.suggest();
|
||||
};
|
||||
const matchingCandidates = (): DownloadCandidate[] => {
|
||||
const currentIntent = intent;
|
||||
return currentIntent
|
||||
? [...candidates.values()].filter(
|
||||
(candidate) => !candidate.suggested && matchesIntent(candidate.item, currentIntent),
|
||||
)
|
||||
: [];
|
||||
};
|
||||
|
||||
const claimUnique = () => {
|
||||
uniquenessTimer = undefined;
|
||||
if (settled || capturedId !== undefined || !intent) return;
|
||||
const matches = matchingCandidates();
|
||||
if (matches.length !== 1) {
|
||||
if (matches.length > 1) {
|
||||
for (const candidate of matches) suggestDefault(candidate);
|
||||
fail(new Error("download attribution is ambiguous"));
|
||||
}
|
||||
return;
|
||||
}
|
||||
const candidate = matches[0];
|
||||
candidate.suggested = true;
|
||||
clearTimeout(candidate.graceTimer);
|
||||
capturedId = candidate.item.id;
|
||||
candidate.suggest({
|
||||
filename: `${options.browserRelativeDir}/${safeBasename(intent.suggestedFilename)}`,
|
||||
conflictAction: "overwrite",
|
||||
});
|
||||
const size = knownSize(candidate.item);
|
||||
if (size !== undefined && options.maxByteSize !== undefined && size > options.maxByteSize) {
|
||||
fail(new Error(`download exceeds transfer limit ${options.maxByteSize}`));
|
||||
return;
|
||||
}
|
||||
const created = createdItems.get(candidate.item.id);
|
||||
if (created?.state === "interrupted") {
|
||||
fail(new Error(created.error ?? "download interrupted"));
|
||||
} else if (created?.state === "complete") {
|
||||
complete(created);
|
||||
}
|
||||
};
|
||||
const reconcile = () => {
|
||||
if (settled || capturedId !== undefined || !intent) return;
|
||||
const matches = matchingCandidates();
|
||||
if (matches.length > 1) {
|
||||
for (const candidate of matches) suggestDefault(candidate);
|
||||
fail(new Error("download attribution is ambiguous"));
|
||||
return;
|
||||
}
|
||||
if (matches.length === 1 && !uniquenessTimer) {
|
||||
uniquenessTimer = setTimeout(claimUnique, UNIQUE_SETTLE_MS);
|
||||
}
|
||||
};
|
||||
|
||||
const determiningListener: DeterminingFilenameListener = (item, suggest) => {
|
||||
const candidate: DownloadCandidate = {
|
||||
item,
|
||||
suggest,
|
||||
suggested: false,
|
||||
graceTimer: setTimeout(() => {
|
||||
suggestDefault(candidate);
|
||||
candidates.delete(item.id);
|
||||
if (intent && matchesIntent(item, intent) && capturedId === undefined) {
|
||||
fail(new Error("download correlation grace elapsed before unique attribution"));
|
||||
}
|
||||
}, CORRELATION_GRACE_MS),
|
||||
};
|
||||
candidates.set(item.id, candidate);
|
||||
reconcile();
|
||||
return true;
|
||||
};
|
||||
const createdListener = (item: chrome.downloads.DownloadItem) => {
|
||||
createdItems.set(item.id, item);
|
||||
if (capturedId !== item.id) return;
|
||||
if (item.state === "interrupted") {
|
||||
fail(new Error(item.error ?? "download interrupted"));
|
||||
} else if (item.state === "complete") {
|
||||
complete(item);
|
||||
}
|
||||
};
|
||||
const changedListener = async (delta: chrome.downloads.DownloadDelta) => {
|
||||
if (capturedId === undefined || delta.id !== capturedId || settled) return;
|
||||
if (delta.state?.current === "interrupted" || delta.error?.current) {
|
||||
fail(new Error(delta.error?.current ?? "download interrupted"));
|
||||
return;
|
||||
}
|
||||
if (delta.state?.current === "complete") {
|
||||
try {
|
||||
const [item] = await options.downloads.search({ id: delta.id });
|
||||
if (item) complete(item);
|
||||
else fail(new Error("completed download disappeared"));
|
||||
} catch (err) {
|
||||
fail(err instanceof Error ? err : new Error(String(err)));
|
||||
}
|
||||
}
|
||||
};
|
||||
const onAbort = () => fail(new DOMException("aborted", "AbortError"));
|
||||
const cdpSubscription = options.cdp.onEvent?.((source, method, raw) => {
|
||||
if (method !== "Page.downloadWillBegin" || !sameTarget(source, options.target)) return;
|
||||
const event = raw as { url?: unknown; suggestedFilename?: unknown; frameId?: unknown };
|
||||
if (typeof event.url !== "string" || typeof event.suggestedFilename !== "string") return;
|
||||
if (options.expectedFrameId && event.frameId !== options.expectedFrameId) {
|
||||
fail(new Error("download originated from a different frame"));
|
||||
return;
|
||||
}
|
||||
if (intent) {
|
||||
fail(new Error("download trigger produced more than one browser download intent"));
|
||||
return;
|
||||
}
|
||||
intent = {
|
||||
url: event.url,
|
||||
suggestedFilename: event.suggestedFilename,
|
||||
...(typeof event.frameId === "string" ? { frameId: event.frameId } : {}),
|
||||
};
|
||||
reconcile();
|
||||
});
|
||||
if (!cdpSubscription) {
|
||||
return captureError("CDP download intent subscription unavailable", "none", "arm");
|
||||
}
|
||||
|
||||
options.downloads.onDeterminingFilename.addListener(determiningListener);
|
||||
options.downloads.onCreated.addListener(createdListener);
|
||||
options.downloads.onChanged.addListener(changedListener);
|
||||
options.signal?.addEventListener("abort", onAbort, { once: true });
|
||||
operationTimer = setTimeout(
|
||||
() => fail(new Error("download did not complete before timeout")),
|
||||
options.timeoutMs,
|
||||
);
|
||||
sizePoll = setInterval(() => {
|
||||
if (capturedId === undefined || settled || options.maxByteSize === undefined) return;
|
||||
void options.downloads
|
||||
.search({ id: capturedId })
|
||||
.then(([item]) => {
|
||||
if (!item || settled) return;
|
||||
if (item.bytesReceived > (options.maxByteSize as number)) {
|
||||
fail(new Error(`download exceeds transfer limit ${options.maxByteSize}`));
|
||||
}
|
||||
})
|
||||
.catch((err) => fail(err instanceof Error ? err : new Error(String(err))));
|
||||
}, SIZE_POLL_MS);
|
||||
|
||||
try {
|
||||
const triggered = await options.trigger();
|
||||
if (isRpcError(triggered)) {
|
||||
void completion.catch(() => undefined);
|
||||
const effect: TransferEffectState =
|
||||
capturedId !== undefined ? "committed" : intent ? "unknown" : "none";
|
||||
failureResult = {
|
||||
...triggered,
|
||||
data: { ...triggered.data, effect_state: effect, phase: "trigger" },
|
||||
};
|
||||
return failureResult;
|
||||
}
|
||||
click = triggered;
|
||||
const item = await completion;
|
||||
succeeded = true;
|
||||
return { click, item };
|
||||
} catch (err) {
|
||||
const effect: TransferEffectState =
|
||||
capturedId !== undefined ? "committed" : click ? "unknown" : "none";
|
||||
failureResult = captureError(
|
||||
err instanceof Error ? err.message : String(err),
|
||||
effect,
|
||||
capturedId !== undefined ? "download" : "attribution",
|
||||
);
|
||||
return failureResult;
|
||||
} finally {
|
||||
settled = true;
|
||||
if (operationTimer) clearTimeout(operationTimer);
|
||||
if (uniquenessTimer) clearTimeout(uniquenessTimer);
|
||||
if (sizePoll) clearInterval(sizePoll);
|
||||
options.signal?.removeEventListener("abort", onAbort);
|
||||
options.downloads.onDeterminingFilename.removeListener(determiningListener);
|
||||
options.downloads.onCreated.removeListener(createdListener);
|
||||
options.downloads.onChanged.removeListener(changedListener);
|
||||
cdpSubscription.dispose();
|
||||
for (const candidate of candidates.values()) {
|
||||
clearTimeout(candidate.graceTimer);
|
||||
if (candidate.item.id !== capturedId) suggestDefault(candidate);
|
||||
}
|
||||
if (!succeeded && capturedId !== undefined) {
|
||||
try {
|
||||
await cleanupClaimedDownload(options.downloads, capturedId);
|
||||
} catch {
|
||||
if (failureResult?.data) failureResult.data.cleanup_state = "failed";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,64 @@
|
||||
// Download orchestration: validate session/tab ownership, then delegate one
|
||||
// browser-global chrome.downloads transaction to download-capture.ts.
|
||||
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import type { DownloadParams, DownloadResult, RpcError } from "@/transport/types";
|
||||
import { captureBrowserDownload, chromeDownloadsApi, type DownloadsApi } from "./download-capture";
|
||||
import { clickResolvedTarget, type InteractionDeps, resolveActionTarget } from "./interaction";
|
||||
import { enforceAgentWindow, isRpcError, lookupSession, resolveTargetTab } from "./shared";
|
||||
|
||||
let downloadActive = false;
|
||||
|
||||
export type { DownloadsApi } from "./download-capture";
|
||||
|
||||
export interface DownloadDeps extends InteractionDeps {
|
||||
downloads?: DownloadsApi;
|
||||
}
|
||||
|
||||
export async function handleDownload(
|
||||
manager: SessionManager,
|
||||
params: DownloadParams,
|
||||
deps: DownloadDeps,
|
||||
): Promise<DownloadResult | RpcError> {
|
||||
if (downloadActive) return { code: "invalid_params", message: "another bsk download is active" };
|
||||
downloadActive = true;
|
||||
try {
|
||||
const ctx = lookupSession(manager, params, "download");
|
||||
if (isRpcError(ctx)) return ctx;
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
const denied = enforceAgentWindow(ctx, target, "download");
|
||||
if (denied) return denied;
|
||||
if (!params.browser_relative_dir) {
|
||||
return { code: "invalid_params", message: "download requires a daemon capability directory" };
|
||||
}
|
||||
const address = await resolveActionTarget(deps.cdp, ctx, target, params, "download");
|
||||
if (isRpcError(address)) return address;
|
||||
|
||||
const capture = await captureBrowserDownload({
|
||||
cdp: deps.cdp,
|
||||
target: address.cdpTarget,
|
||||
downloads: deps.downloads ?? chromeDownloadsApi,
|
||||
browserRelativeDir: params.browser_relative_dir,
|
||||
maxByteSize: params.max_byte_size,
|
||||
timeoutMs: params.timeout_ms ?? 120_000,
|
||||
signal: deps.signal,
|
||||
expectedFrameId: address.frameId,
|
||||
trigger: () => clickResolvedTarget(ctx, address, {}, deps),
|
||||
});
|
||||
if (isRpcError(capture)) return capture;
|
||||
const { click, item } = capture;
|
||||
return {
|
||||
tab_id: target.tabId,
|
||||
used_ref: click.used_ref,
|
||||
used_selector: click.used_selector,
|
||||
suggested_filename: item.filename.split(/[\\/]/).pop() ?? "download",
|
||||
byte_size: item.fileSize >= 0 ? item.fileSize : item.totalBytes,
|
||||
mime: item.mime || undefined,
|
||||
danger: item.danger,
|
||||
browser_path: item.filename,
|
||||
};
|
||||
} finally {
|
||||
downloadActive = false;
|
||||
}
|
||||
}
|
||||
@@ -2,10 +2,23 @@
|
||||
// rendering. Extension handlers attach reasons here; human-facing copy
|
||||
// lives in bsk-cli `render_error.rs`.
|
||||
|
||||
import type { ErrorCode, RpcError, RpcErrorData, RpcErrorReason } from "@/transport/types";
|
||||
import type {
|
||||
ErrorCode,
|
||||
RpcError,
|
||||
RpcErrorData,
|
||||
RpcErrorReason,
|
||||
TransferCleanupState,
|
||||
TransferEffectState,
|
||||
} from "@/transport/types";
|
||||
|
||||
export type { RpcErrorData, RpcErrorReason };
|
||||
|
||||
export interface TransferErrorOptions {
|
||||
effectState: TransferEffectState;
|
||||
phase: string;
|
||||
cleanupState?: TransferCleanupState;
|
||||
}
|
||||
|
||||
export function rpcError(
|
||||
code: ErrorCode,
|
||||
reason: RpcErrorReason,
|
||||
@@ -15,3 +28,16 @@ export function rpcError(
|
||||
const data: RpcErrorData = { reason, ...extra };
|
||||
return { code, message, data };
|
||||
}
|
||||
|
||||
export function transferError(
|
||||
code: ErrorCode,
|
||||
reason: RpcErrorReason,
|
||||
message: string,
|
||||
options: TransferErrorOptions,
|
||||
): RpcError {
|
||||
return rpcError(code, reason, message, {
|
||||
effect_state: options.effectState,
|
||||
phase: options.phase,
|
||||
...(options.cleanupState ? { cleanup_state: options.cleanupState } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,528 @@
|
||||
// One upload transaction with an explicit browser-side commit boundary.
|
||||
// Chrome interception prevents a native chooser from escaping automation;
|
||||
// chooser events and a frame-scoped DOM probe are independent input-location
|
||||
// signals, so event delivery is not required for standard file inputs.
|
||||
|
||||
import type { ClickResult, RpcError, TransferEffectState } from "@/transport/types";
|
||||
import { transferError } from "./errors";
|
||||
import type { ResolvedActionTarget } from "./interaction";
|
||||
import { type CdpRunner, isRpcError, sendToCdpTarget } from "./shared";
|
||||
|
||||
const CLEANUP_TIMEOUT_MS = 1_000;
|
||||
const DETACH_TIMEOUT_MS = 500;
|
||||
const ACTIVATION_GRACE_MS = 1_000;
|
||||
const PROBE_INTERVAL_MS = 20;
|
||||
|
||||
type UploadPhase =
|
||||
| "arm_interception"
|
||||
| "arm_input_probe"
|
||||
| "trigger"
|
||||
| "resolve_input"
|
||||
| "set_files"
|
||||
| "cleanup";
|
||||
|
||||
interface RuntimeReply {
|
||||
result?: { value?: unknown; objectId?: string };
|
||||
exceptionDetails?: { text?: string; exception?: { description?: string } };
|
||||
}
|
||||
|
||||
interface ChooserEvent {
|
||||
frameId?: string;
|
||||
backendNodeId?: number;
|
||||
mode?: "selectSingle" | "selectMultiple";
|
||||
}
|
||||
|
||||
export interface FileInputTransactionOptions {
|
||||
cdp: CdpRunner;
|
||||
actionTarget: ResolvedActionTarget;
|
||||
files: string[];
|
||||
timeoutMs: number;
|
||||
signal?: AbortSignal;
|
||||
trigger(): Promise<ClickResult | RpcError>;
|
||||
}
|
||||
|
||||
export interface FileInputTransactionResult {
|
||||
click: ClickResult;
|
||||
multiple: boolean;
|
||||
}
|
||||
|
||||
class BoundedWaitError extends Error {
|
||||
constructor(
|
||||
readonly kind: "timeout" | "aborted",
|
||||
message: string,
|
||||
) {
|
||||
super(message);
|
||||
}
|
||||
}
|
||||
|
||||
function remainingMs(deadline: number): number {
|
||||
return Math.max(0, deadline - Date.now());
|
||||
}
|
||||
|
||||
async function waitBounded<T>(
|
||||
operation: () => Promise<T>,
|
||||
deadline: number,
|
||||
signal: AbortSignal | undefined,
|
||||
timeoutMessage: string,
|
||||
): Promise<T> {
|
||||
const remaining = remainingMs(deadline);
|
||||
if (signal?.aborted) throw new BoundedWaitError("aborted", "upload transaction aborted");
|
||||
if (remaining === 0) throw new BoundedWaitError("timeout", timeoutMessage);
|
||||
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
let onAbort: (() => void) | undefined;
|
||||
const boundary = new Promise<never>((_resolve, reject) => {
|
||||
timer = setTimeout(() => reject(new BoundedWaitError("timeout", timeoutMessage)), remaining);
|
||||
if (signal) {
|
||||
onAbort = () => reject(new BoundedWaitError("aborted", "upload transaction aborted"));
|
||||
signal.addEventListener("abort", onAbort, { once: true });
|
||||
}
|
||||
});
|
||||
try {
|
||||
const work = Promise.resolve().then(() => {
|
||||
if (signal?.aborted) throw new BoundedWaitError("aborted", "upload transaction aborted");
|
||||
if (remainingMs(deadline) === 0) throw new BoundedWaitError("timeout", timeoutMessage);
|
||||
return operation();
|
||||
});
|
||||
return await Promise.race([work, boundary]);
|
||||
} finally {
|
||||
if (timer) clearTimeout(timer);
|
||||
if (signal && onAbort) signal.removeEventListener("abort", onAbort);
|
||||
}
|
||||
}
|
||||
|
||||
function runtimeError(reply: RuntimeReply, fallback: string): Error | null {
|
||||
if (!reply.exceptionDetails) return null;
|
||||
return new Error(
|
||||
reply.exceptionDetails.exception?.description ?? reply.exceptionDetails.text ?? fallback,
|
||||
);
|
||||
}
|
||||
|
||||
function transferFailure(
|
||||
code: RpcError["code"],
|
||||
reason: "file_input_probe_failed" | "file_input_not_activated" | "set_file_input_failed",
|
||||
message: string,
|
||||
effectState: TransferEffectState,
|
||||
phase: UploadPhase,
|
||||
): RpcError {
|
||||
return transferError(code, reason, message, { effectState, phase });
|
||||
}
|
||||
|
||||
function enrichFailure(
|
||||
error: RpcError,
|
||||
effectState: TransferEffectState,
|
||||
phase: UploadPhase,
|
||||
): RpcError {
|
||||
return {
|
||||
...error,
|
||||
data: { ...error.data, effect_state: effectState, phase },
|
||||
};
|
||||
}
|
||||
|
||||
function sameTarget(
|
||||
source: { tabId?: number; sessionId?: string },
|
||||
target: { tabId: number; sessionId?: string },
|
||||
): boolean {
|
||||
return source.tabId === target.tabId && source.sessionId === target.sessionId;
|
||||
}
|
||||
|
||||
async function delay(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
await waitBounded(
|
||||
() => new Promise((resolve) => setTimeout(resolve, ms)),
|
||||
Date.now() + ms + 1,
|
||||
signal,
|
||||
"upload activation probe timed out",
|
||||
);
|
||||
}
|
||||
|
||||
function waitErrorCode(error: unknown): RpcError["code"] {
|
||||
if (!(error instanceof BoundedWaitError)) return "cdp_failed";
|
||||
return error.kind === "aborted" ? "cancelled" : "timeout";
|
||||
}
|
||||
|
||||
async function callOnTrigger<T = RuntimeReply>(
|
||||
options: FileInputTransactionOptions,
|
||||
objectId: string,
|
||||
functionDeclaration: string,
|
||||
args: unknown[],
|
||||
returnByValue: boolean,
|
||||
): Promise<T> {
|
||||
return sendToCdpTarget<T>(options.cdp, options.actionTarget.cdpTarget, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration,
|
||||
arguments: args.map((value) => ({ value })),
|
||||
returnByValue,
|
||||
awaitPromise: false,
|
||||
});
|
||||
}
|
||||
|
||||
export async function uploadThroughActivatedFileInput(
|
||||
options: FileInputTransactionOptions,
|
||||
): Promise<FileInputTransactionResult | RpcError> {
|
||||
const deadline = Date.now() + options.timeoutMs;
|
||||
const target = options.actionTarget.cdpTarget;
|
||||
const objectGroup = `bsk-upload-${crypto.randomUUID()}`;
|
||||
const stateKey = `__bskUpload_${crypto.randomUUID().replaceAll("-", "")}`;
|
||||
const chooserEvents: ChooserEvent[] = [];
|
||||
let interceptionState: "not_started" | "possibly_active" | "active" = "not_started";
|
||||
let probeState: "not_started" | "possibly_active" | "active" = "not_started";
|
||||
let triggerObjectId: string | undefined;
|
||||
let outcome: FileInputTransactionResult | RpcError = transferFailure(
|
||||
"protocol_error",
|
||||
"file_input_probe_failed",
|
||||
"upload transaction ended without an outcome",
|
||||
"none",
|
||||
"cleanup",
|
||||
);
|
||||
|
||||
const chooserSubscription = options.cdp.onEvent?.((source, method, raw) => {
|
||||
if (method !== "Page.fileChooserOpened" || !sameTarget(source, target)) return;
|
||||
const event = raw as ChooserEvent;
|
||||
chooserEvents.push(event);
|
||||
});
|
||||
|
||||
try {
|
||||
try {
|
||||
interceptionState = "possibly_active";
|
||||
await waitBounded(
|
||||
() =>
|
||||
sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", {
|
||||
enabled: true,
|
||||
}),
|
||||
deadline,
|
||||
options.signal,
|
||||
"arming native file chooser interception timed out",
|
||||
);
|
||||
interceptionState = "active";
|
||||
} catch (err) {
|
||||
outcome = transferFailure(
|
||||
waitErrorCode(err),
|
||||
"file_input_probe_failed",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
"none",
|
||||
"arm_interception",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
|
||||
try {
|
||||
const resolved = await waitBounded(
|
||||
() =>
|
||||
sendToCdpTarget<{ object?: { objectId?: string } }>(
|
||||
options.cdp,
|
||||
target,
|
||||
"DOM.resolveNode",
|
||||
{
|
||||
backendNodeId: options.actionTarget.backendNodeId,
|
||||
objectGroup,
|
||||
},
|
||||
),
|
||||
deadline,
|
||||
options.signal,
|
||||
"resolving upload trigger timed out",
|
||||
);
|
||||
triggerObjectId = resolved.object?.objectId;
|
||||
if (!triggerObjectId) throw new Error("DOM.resolveNode returned no trigger objectId");
|
||||
|
||||
probeState = "possibly_active";
|
||||
const armed = await waitBounded(
|
||||
() =>
|
||||
callOnTrigger<RuntimeReply>(
|
||||
options,
|
||||
triggerObjectId as string,
|
||||
`function(key) {
|
||||
const doc = this.ownerDocument;
|
||||
const owner = doc.defaultView;
|
||||
if (!owner) return false;
|
||||
const state = { inputs: [], listener: null };
|
||||
Object.defineProperty(owner, key, { value: state, configurable: true });
|
||||
state.listener = event => {
|
||||
const path = typeof event.composedPath === "function" ? event.composedPath() : [];
|
||||
const candidate = path[0] || event.target;
|
||||
if (candidate && candidate.nodeType === 1 &&
|
||||
candidate.localName === "input" && candidate.type === "file") {
|
||||
if (!state.inputs.includes(candidate)) state.inputs.push(candidate);
|
||||
event.preventDefault();
|
||||
}
|
||||
};
|
||||
doc.addEventListener("click", state.listener, true);
|
||||
return true;
|
||||
}`,
|
||||
[stateKey],
|
||||
true,
|
||||
),
|
||||
deadline,
|
||||
options.signal,
|
||||
"arming frame-scoped file input probe timed out",
|
||||
);
|
||||
const armError = runtimeError(armed, "failed to arm file input probe");
|
||||
if (armError || armed.result?.value !== true) {
|
||||
throw armError ?? new Error("file input probe did not arm");
|
||||
}
|
||||
probeState = "active";
|
||||
} catch (err) {
|
||||
outcome = transferFailure(
|
||||
waitErrorCode(err),
|
||||
"file_input_probe_failed",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
"none",
|
||||
"arm_input_probe",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
|
||||
let click: ClickResult | RpcError;
|
||||
try {
|
||||
click = await waitBounded(
|
||||
options.trigger,
|
||||
deadline,
|
||||
options.signal,
|
||||
"upload trigger timed out",
|
||||
);
|
||||
} catch (err) {
|
||||
outcome = transferFailure(
|
||||
waitErrorCode(err),
|
||||
"file_input_probe_failed",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
"none",
|
||||
"trigger",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
if (isRpcError(click)) {
|
||||
outcome = enrichFailure(click, "none", "trigger");
|
||||
return outcome;
|
||||
}
|
||||
|
||||
try {
|
||||
const activationDeadline = Math.min(deadline, Date.now() + ACTIVATION_GRACE_MS);
|
||||
let summary: { count: number; multiple: boolean } = { count: 0, multiple: false };
|
||||
while (Date.now() < activationDeadline) {
|
||||
if (chooserEvents.length > 0) break;
|
||||
const reply = await waitBounded(
|
||||
() =>
|
||||
callOnTrigger<RuntimeReply>(
|
||||
options,
|
||||
triggerObjectId as string,
|
||||
`function(key) {
|
||||
const state = this.ownerDocument.defaultView?.[key];
|
||||
return state
|
||||
? { count: state.inputs.length, multiple: state.inputs[0]?.multiple === true }
|
||||
: { count: 0, multiple: false };
|
||||
}`,
|
||||
[stateKey],
|
||||
true,
|
||||
),
|
||||
activationDeadline,
|
||||
options.signal,
|
||||
"inspecting activated file input timed out",
|
||||
);
|
||||
const summaryError = runtimeError(reply, "failed to inspect activated file input");
|
||||
if (summaryError) throw summaryError;
|
||||
const value = reply.result?.value as { count?: unknown; multiple?: unknown } | undefined;
|
||||
summary = {
|
||||
count: typeof value?.count === "number" ? value.count : 0,
|
||||
multiple: value?.multiple === true,
|
||||
};
|
||||
if (summary.count > 0) break;
|
||||
await delay(Math.min(PROBE_INTERVAL_MS, remainingMs(activationDeadline)), options.signal);
|
||||
}
|
||||
|
||||
if (chooserEvents.length > 1) {
|
||||
outcome = transferFailure(
|
||||
"unsupported",
|
||||
"file_input_not_activated",
|
||||
"upload trigger activated more than one file chooser",
|
||||
"none",
|
||||
"resolve_input",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
|
||||
const chooser = chooserEvents[0];
|
||||
let backendNodeId: number | undefined;
|
||||
let multiple = summary.multiple;
|
||||
if (chooser) {
|
||||
if (options.actionTarget.frameId && chooser.frameId !== options.actionTarget.frameId) {
|
||||
outcome = transferFailure(
|
||||
"unsupported",
|
||||
"file_input_not_activated",
|
||||
"upload trigger activated a file chooser in a different frame",
|
||||
"none",
|
||||
"resolve_input",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
if (typeof chooser.backendNodeId !== "number") {
|
||||
outcome = transferFailure(
|
||||
"unsupported",
|
||||
"file_input_not_activated",
|
||||
"upload trigger invoked a non-input file picker",
|
||||
"none",
|
||||
"resolve_input",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
backendNodeId = chooser.backendNodeId;
|
||||
multiple = chooser.mode === "selectMultiple";
|
||||
} else {
|
||||
if (summary.count !== 1) {
|
||||
outcome = transferFailure(
|
||||
"unsupported",
|
||||
"file_input_not_activated",
|
||||
summary.count === 0
|
||||
? "upload trigger did not activate an input[type=file]"
|
||||
: "upload trigger activated more than one input[type=file]",
|
||||
"none",
|
||||
"resolve_input",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
const input = await waitBounded(
|
||||
() =>
|
||||
callOnTrigger<RuntimeReply>(
|
||||
options,
|
||||
triggerObjectId as string,
|
||||
`function(key) { return this.ownerDocument.defaultView?.[key]?.inputs[0]; }`,
|
||||
[stateKey],
|
||||
false,
|
||||
),
|
||||
deadline,
|
||||
options.signal,
|
||||
"resolving activated file input timed out",
|
||||
);
|
||||
const inputError = runtimeError(input, "failed to resolve activated file input object");
|
||||
if (inputError) throw inputError;
|
||||
if (!input.result?.objectId) throw new Error("activated file input returned no objectId");
|
||||
const described = await waitBounded(
|
||||
() =>
|
||||
sendToCdpTarget<{ node?: { backendNodeId?: number } }>(
|
||||
options.cdp,
|
||||
target,
|
||||
"DOM.describeNode",
|
||||
{ objectId: input.result?.objectId },
|
||||
),
|
||||
deadline,
|
||||
options.signal,
|
||||
"describing activated file input timed out",
|
||||
);
|
||||
backendNodeId = described.node?.backendNodeId;
|
||||
if (typeof backendNodeId !== "number") {
|
||||
throw new Error("DOM.describeNode returned no file input backendNodeId");
|
||||
}
|
||||
}
|
||||
|
||||
if (!multiple && options.files.length !== 1) {
|
||||
outcome = enrichFailure(
|
||||
{ code: "invalid_params", message: "file input accepts exactly one file" },
|
||||
"none",
|
||||
"resolve_input",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
|
||||
try {
|
||||
await waitBounded(
|
||||
() =>
|
||||
sendToCdpTarget(options.cdp, target, "DOM.setFileInputFiles", {
|
||||
files: options.files,
|
||||
backendNodeId,
|
||||
}),
|
||||
deadline,
|
||||
options.signal,
|
||||
"setting file input files timed out",
|
||||
);
|
||||
} catch (err) {
|
||||
outcome = transferFailure(
|
||||
waitErrorCode(err),
|
||||
"set_file_input_failed",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
"unknown",
|
||||
"set_files",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
outcome = { click, multiple };
|
||||
return outcome;
|
||||
} catch (err) {
|
||||
outcome = transferFailure(
|
||||
waitErrorCode(err),
|
||||
"file_input_probe_failed",
|
||||
err instanceof Error ? err.message : String(err),
|
||||
"none",
|
||||
"resolve_input",
|
||||
);
|
||||
return outcome;
|
||||
}
|
||||
} finally {
|
||||
chooserSubscription?.dispose();
|
||||
const cleanupDeadline = Date.now() + CLEANUP_TIMEOUT_MS;
|
||||
const cleanupTasks: Promise<boolean>[] = [];
|
||||
const cleanup = async (operation: () => Promise<unknown>, message: string) => {
|
||||
try {
|
||||
await waitBounded(operation, cleanupDeadline, undefined, message);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
if (probeState !== "not_started" && triggerObjectId) {
|
||||
cleanupTasks.push(
|
||||
cleanup(
|
||||
() =>
|
||||
callOnTrigger(
|
||||
options,
|
||||
triggerObjectId as string,
|
||||
`function(key) {
|
||||
const owner = this.ownerDocument.defaultView;
|
||||
const state = owner?.[key];
|
||||
if (state?.listener) this.ownerDocument.removeEventListener("click", state.listener, true);
|
||||
if (owner) delete owner[key];
|
||||
}`,
|
||||
[stateKey],
|
||||
true,
|
||||
),
|
||||
"cleaning file input probe timed out",
|
||||
),
|
||||
);
|
||||
}
|
||||
if (interceptionState !== "not_started") {
|
||||
cleanupTasks.push(
|
||||
cleanup(
|
||||
() =>
|
||||
sendToCdpTarget(options.cdp, target, "Page.setInterceptFileChooserDialog", {
|
||||
enabled: false,
|
||||
cancel: true,
|
||||
}),
|
||||
"disabling file chooser interception timed out",
|
||||
),
|
||||
);
|
||||
}
|
||||
let cleanupFailed = (await Promise.all(cleanupTasks)).some((completed) => !completed);
|
||||
if (!cleanupFailed) {
|
||||
cleanupFailed = !(await cleanup(
|
||||
() => sendToCdpTarget(options.cdp, target, "Runtime.releaseObjectGroup", { objectGroup }),
|
||||
"releasing upload object group timed out",
|
||||
));
|
||||
}
|
||||
|
||||
const effect = isRpcError(outcome)
|
||||
? (outcome.data?.effect_state as TransferEffectState | undefined)
|
||||
: "committed";
|
||||
if (effect === "unknown" || cleanupFailed) {
|
||||
try {
|
||||
await waitBounded(
|
||||
() => options.cdp.detach?.(target.tabId) ?? Promise.resolve(),
|
||||
Date.now() + DETACH_TIMEOUT_MS,
|
||||
undefined,
|
||||
"detaching upload target timed out",
|
||||
);
|
||||
} catch {
|
||||
cleanupFailed = true;
|
||||
}
|
||||
}
|
||||
if (cleanupFailed && isRpcError(outcome)) {
|
||||
outcome.data = { ...outcome.data, cleanup_state: "failed" };
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -40,6 +40,7 @@ import {
|
||||
enforceAgentWindow,
|
||||
isRpcError,
|
||||
lookupSession,
|
||||
type ResolvedTargetTab,
|
||||
resolveTargetTab,
|
||||
} from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
@@ -56,6 +57,15 @@ export interface InteractionDeps {
|
||||
keepOverlayBypassAfterHover?: boolean;
|
||||
}
|
||||
|
||||
export interface ResolvedActionTarget {
|
||||
tab: ResolvedTargetTab;
|
||||
backendNodeId: number;
|
||||
cdpTarget: CdpTarget;
|
||||
frameId?: string;
|
||||
usedRef?: string;
|
||||
usedSelector?: string;
|
||||
}
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 30_000;
|
||||
const DEFAULT_HOVER_SETTLE_MS = 200;
|
||||
|
||||
@@ -128,7 +138,7 @@ async function wait(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
* `RpcError` if the caller supplied neither (or both), or if neither
|
||||
* lookup matched.
|
||||
*/
|
||||
async function resolveBackendNode(
|
||||
export async function resolveBackendNode(
|
||||
cdp: CdpRunner,
|
||||
ctx: SessionContext,
|
||||
target: { tabId: number },
|
||||
@@ -215,6 +225,17 @@ async function resolveBackendNode(
|
||||
}
|
||||
}
|
||||
|
||||
export async function resolveActionTarget(
|
||||
cdp: CdpRunner,
|
||||
ctx: SessionContext,
|
||||
target: ResolvedTargetTab,
|
||||
params: { ref?: string; selector?: string },
|
||||
toolName: string,
|
||||
): Promise<ResolvedActionTarget | RpcError> {
|
||||
const node = await resolveBackendNode(cdp, ctx, target, params, toolName);
|
||||
return isRpcError(node) ? node : { tab: target, ...node };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// tool.click
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -233,10 +254,19 @@ export async function handleClick(
|
||||
if (isRpcError(target)) return target;
|
||||
const denied = enforceAgentWindow(ctx, target, "click");
|
||||
if (denied) return denied;
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
const resolved = await resolveActionTarget(deps.cdp, ctx, target, params, "click");
|
||||
if (isRpcError(resolved)) return resolved;
|
||||
return clickResolvedTarget(ctx, resolved, params, deps);
|
||||
}
|
||||
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "click");
|
||||
if (isRpcError(node)) return node;
|
||||
export async function clickResolvedTarget(
|
||||
ctx: SessionContext,
|
||||
resolved: ResolvedActionTarget,
|
||||
params: Pick<ClickParams, "button" | "click_count" | "modifiers">,
|
||||
deps: InteractionDeps,
|
||||
): Promise<ClickResult | RpcError> {
|
||||
const { tab: target } = resolved;
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "click aborted" };
|
||||
@@ -247,9 +277,9 @@ export async function handleClick(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
{
|
||||
target: node.cdpTarget,
|
||||
backendNodeId: node.backendNodeId,
|
||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||
target: resolved.cdpTarget,
|
||||
backendNodeId: resolved.backendNodeId,
|
||||
...(resolved.frameId ? { frameId: resolved.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
@@ -337,8 +367,8 @@ export async function handleClick(
|
||||
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
tab_id: target.tabId,
|
||||
used_ref: node.usedRef,
|
||||
used_selector: node.usedSelector,
|
||||
used_ref: resolved.usedRef,
|
||||
used_selector: resolved.usedSelector,
|
||||
x: centre.x,
|
||||
y: centre.y,
|
||||
});
|
||||
|
||||
@@ -49,11 +49,14 @@ import {
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import {
|
||||
type CapturedNode,
|
||||
type CapturedSurfaceProbe,
|
||||
type CapturedViewModel,
|
||||
captureViewModel,
|
||||
collectOverlayExcludedBackendIds,
|
||||
probeHoverSurfaces,
|
||||
} from "./vom/capture";
|
||||
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
|
||||
import { withOverlayBypass } from "./vom/hover-perception";
|
||||
import { probeTooltipNames } from "./vom/name-enrichment";
|
||||
import {
|
||||
type CaptureVomObservationResult,
|
||||
@@ -525,8 +528,11 @@ function buildActiveScopeBlocks(nodes: VomNode[], signals: VomNodeDomSignals): A
|
||||
return blocks;
|
||||
}
|
||||
|
||||
function buildConditionalSurfaces(nodes: VomNode[], captured: CapturedViewModel): CondSurface[] {
|
||||
const probes = captured.surfaceProbes ?? [];
|
||||
function buildConditionalSurfaces(
|
||||
nodes: VomNode[],
|
||||
captured: CapturedViewModel,
|
||||
probes: CapturedSurfaceProbe[],
|
||||
): CondSurface[] {
|
||||
if (probes.length === 0) return [];
|
||||
|
||||
const surfaces: CondSurface[] = [];
|
||||
@@ -649,6 +655,7 @@ function findSurfaceNodeByPoint(
|
||||
export interface BuildVomSceneOptions {
|
||||
pageUrl?: string;
|
||||
supplementalNames?: ReadonlyMap<string, string>;
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
}
|
||||
|
||||
export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>;
|
||||
@@ -817,18 +824,19 @@ export function buildFrameVomScene(
|
||||
excludedBackendNodeIds: captured.excludedBackendNodeIds,
|
||||
supplementalNames: options.supplementalNames,
|
||||
});
|
||||
return attachCapturedSceneAnnotations(scene, documents, captured);
|
||||
return attachCapturedSceneAnnotations(scene, documents, captured, options.surfaceProbes ?? []);
|
||||
}
|
||||
|
||||
function attachCapturedSceneAnnotations(
|
||||
scene: VomScene,
|
||||
documents: VomFrameDocument[],
|
||||
captured: CapturedViewModel,
|
||||
surfaceProbes: CapturedSurfaceProbe[],
|
||||
): VomScene {
|
||||
const rootDocument = documents.find((document) => document.frameId === scene.rootFrameId);
|
||||
const signals = capturedOnlySignals(rootDocument?.domNodes ?? captured.nodes);
|
||||
const activeScopeBlocks = buildActiveScopeBlocks(scene.nodes, signals);
|
||||
const surfaces = buildConditionalSurfaces(scene.nodes, captured);
|
||||
const surfaces = buildConditionalSurfaces(scene.nodes, captured, surfaceProbes);
|
||||
return {
|
||||
...scene,
|
||||
...(surfaces.length > 0 ? { surfaces } : {}),
|
||||
@@ -1010,17 +1018,69 @@ async function captureForVom(
|
||||
options: CaptureVomObservationOptions,
|
||||
): Promise<CapturedViewModel> {
|
||||
try {
|
||||
return await captureViewModel(cdp, tabId, {
|
||||
conditionalSurfaceProbe: options.conditionalSurfaceProbe ?? false,
|
||||
hoverProbeBypassOverlay: options.hoverProbeBypassOverlay,
|
||||
signal: options.signal,
|
||||
});
|
||||
return await captureViewModel(cdp, tabId, { signal: options.signal });
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
return fallbackCapturedViewModel(cdp, tabId, options.signal);
|
||||
}
|
||||
}
|
||||
|
||||
export interface HoverProbeOutcome {
|
||||
/** Whether any active hover was dispatched during this observation. */
|
||||
performed: boolean;
|
||||
/**
|
||||
* Whether hovering surfaced content absent from the static snapshot. Cached
|
||||
* tooltip names count, so this may over-report on repeat observations — it
|
||||
* errs towards telling the caller the snapshot is less fresh than it looks.
|
||||
*/
|
||||
revealedContent: boolean;
|
||||
surfaceProbes: CapturedSurfaceProbe[];
|
||||
tooltipNames: Map<string, string>;
|
||||
}
|
||||
|
||||
const NO_HOVER_PROBES: HoverProbeOutcome = {
|
||||
performed: false,
|
||||
revealedContent: false,
|
||||
surfaceProbes: [],
|
||||
tooltipNames: new Map(),
|
||||
};
|
||||
|
||||
/**
|
||||
* Runs both active-hover chains back to back.
|
||||
*
|
||||
* Ordering matters: this happens after DOM *and* accessibility capture so a
|
||||
* hover that opens a menu cannot leave one half of the observation describing
|
||||
* the page before the change and the other half after it. Sharing one overlay
|
||||
* bypass span also means the agent overlay toggles once per observation
|
||||
* instead of once per chain.
|
||||
*/
|
||||
async function runHoverProbes(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
captured: CapturedViewModel,
|
||||
documents: VomFrameDocument[],
|
||||
staticSemantics: ReturnType<typeof resolveSemanticGraph>,
|
||||
options: CaptureVomObservationOptions,
|
||||
): Promise<HoverProbeOutcome> {
|
||||
if (!options.conditionalSurfaceProbe) return NO_HOVER_PROBES;
|
||||
|
||||
return withOverlayBypass(options.hoverProbeBypassOverlay, tabId, async () => {
|
||||
const surfaceProbes = await probeHoverSurfaces(cdp, tabId, captured.nodes, {
|
||||
signal: options.signal,
|
||||
});
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const tooltipNames = await probeTooltipNames(cdp, tabId, documents, staticSemantics, {
|
||||
signal: options.signal,
|
||||
});
|
||||
return {
|
||||
performed: true,
|
||||
revealedContent: surfaceProbes.length > 0 || tooltipNames.size > 0,
|
||||
surfaceProbes,
|
||||
tooltipNames,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export interface CaptureVomObservationOptions extends VomOptions {
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
@@ -1051,19 +1111,26 @@ export async function captureVomObservation(
|
||||
excludedBackendNodeIds: captured.excludedBackendNodeIds,
|
||||
});
|
||||
const staticSemantics = resolveSemanticGraph(semanticGraph, { identifierFallback: false });
|
||||
const tooltipNames = options.conditionalSurfaceProbe
|
||||
? await probeTooltipNames(cdp, tabId, normalizedDocuments, staticSemantics, {
|
||||
signal: options.signal,
|
||||
hoverProbeBypassOverlay: options.hoverProbeBypassOverlay,
|
||||
})
|
||||
: new Map<string, string>();
|
||||
const hoverProbes = await runHoverProbes(
|
||||
cdp,
|
||||
tabId,
|
||||
captured,
|
||||
normalizedDocuments,
|
||||
staticSemantics,
|
||||
options,
|
||||
);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const scene = projectSemanticGraph(
|
||||
normalizeSemanticStructure(
|
||||
resolveSemanticGraph(semanticGraph, { supplementalNames: tooltipNames }),
|
||||
resolveSemanticGraph(semanticGraph, { supplementalNames: hoverProbes.tooltipNames }),
|
||||
),
|
||||
);
|
||||
const decoratedScene = attachCapturedSceneAnnotations(scene, normalizedDocuments, captured);
|
||||
const decoratedScene = attachCapturedSceneAnnotations(
|
||||
scene,
|
||||
normalizedDocuments,
|
||||
captured,
|
||||
hoverProbes.surfaceProbes,
|
||||
);
|
||||
const rendered = renderVom(decoratedScene, {
|
||||
maxDepth: options.maxDepth,
|
||||
maxTokens: options.maxTokens,
|
||||
@@ -1075,7 +1142,11 @@ export async function captureVomObservation(
|
||||
rootFrameId: captured.rootFrameId ?? normalizedDocuments[0]?.frameId ?? "root",
|
||||
frameDocuments: normalizedDocuments,
|
||||
rendered,
|
||||
surfaceProbes: captured.surfaceProbes,
|
||||
surfaceProbes: hoverProbes.surfaceProbes,
|
||||
hoverProbe: {
|
||||
performed: hoverProbes.performed,
|
||||
revealedContent: hoverProbes.revealedContent,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1141,6 +1212,14 @@ async function handleVomObservation(
|
||||
ref_count: observation.refs.length,
|
||||
tab_id: target.tabId,
|
||||
truncated: observation.truncated,
|
||||
...(toolName === "observe" && observation.hoverProbe?.performed
|
||||
? {
|
||||
hover_probe: {
|
||||
performed: true,
|
||||
revealed_content: observation.hoverProbe.revealedContent,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
|
||||
? {
|
||||
debug: {
|
||||
@@ -1179,5 +1258,13 @@ export async function handleObserve(
|
||||
deps: SnapshotDeps = getDefaultDeps(),
|
||||
signal?: AbortSignal,
|
||||
): Promise<ObserveResult | RpcError> {
|
||||
return handleVomObservation(manager, params, "observe", "transient_input", true, deps, signal);
|
||||
return handleVomObservation(
|
||||
manager,
|
||||
params,
|
||||
"observe",
|
||||
"transient_input",
|
||||
params.probe_hover === true,
|
||||
deps,
|
||||
signal,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -53,6 +53,7 @@ export type { DialogCursor };
|
||||
export interface CdpRunner {
|
||||
send<T = unknown>(tabId: number, method: string, params?: object): Promise<T>;
|
||||
sendToTarget?<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T>;
|
||||
detach?(tabId: number): Promise<void>;
|
||||
getFrameGraph?(tabId: number): Promise<CdpFrameGraph>;
|
||||
ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>;
|
||||
trackSessionTab?(sessionId: string, tabId: number): void;
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
// Upload orchestration: validate the session-scoped request, resolve its click
|
||||
// target, then delegate the browser protocol transaction to the file-input
|
||||
// transaction module.
|
||||
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import type { RpcError, UploadParams, UploadResult } from "@/transport/types";
|
||||
import { uploadThroughActivatedFileInput } from "./file-input-transaction";
|
||||
import { clickResolvedTarget, type InteractionDeps, resolveActionTarget } from "./interaction";
|
||||
import {
|
||||
type CdpRunner,
|
||||
enforceAgentWindow,
|
||||
isRpcError,
|
||||
lookupSession,
|
||||
resolveTargetTab,
|
||||
} from "./shared";
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 120_000;
|
||||
|
||||
export interface UploadDeps extends InteractionDeps {
|
||||
cdp: CdpRunner;
|
||||
}
|
||||
|
||||
export async function handleUpload(
|
||||
manager: SessionManager,
|
||||
params: UploadParams,
|
||||
deps: UploadDeps,
|
||||
): Promise<UploadResult | RpcError> {
|
||||
const ctx = lookupSession(manager, params, "upload");
|
||||
if (isRpcError(ctx)) return ctx;
|
||||
const target = await resolveTargetTab(manager, ctx, params.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
const denied = enforceAgentWindow(ctx, target, "upload");
|
||||
if (denied) return denied;
|
||||
if (
|
||||
params.files.length === 0 ||
|
||||
params.files.length > 20 ||
|
||||
params.files.some((file) => !file.staged_path)
|
||||
) {
|
||||
return { code: "invalid_params", message: "upload requires daemon-staged files" };
|
||||
}
|
||||
const address = await resolveActionTarget(deps.cdp, ctx, target, params, "upload");
|
||||
if (isRpcError(address)) return address;
|
||||
const timeoutMs = params.timeout_ms ?? DEFAULT_TIMEOUT_MS;
|
||||
const transaction = await uploadThroughActivatedFileInput({
|
||||
cdp: deps.cdp,
|
||||
actionTarget: address,
|
||||
files: params.files.map((file) => file.staged_path as string),
|
||||
timeoutMs,
|
||||
signal: deps.signal,
|
||||
trigger: () => clickResolvedTarget(ctx, address, {}, deps),
|
||||
});
|
||||
if (isRpcError(transaction)) return transaction;
|
||||
return {
|
||||
tab_id: target.tabId,
|
||||
used_ref: transaction.click.used_ref,
|
||||
used_selector: transaction.click.used_selector,
|
||||
file_names: params.files.map((file) => file.name),
|
||||
};
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "../../../lib/overlay-bridge";
|
||||
import { captureViewModel, collectOverlayExcludedBackendIds } from "../capture";
|
||||
import { captureViewModel, collectOverlayExcludedBackendIds, probeHoverSurfaces } from "../capture";
|
||||
|
||||
// Minimal but format-accurate captureSnapshot reply: a body with one
|
||||
// fixed full-screen overlay div carrying a password input.
|
||||
@@ -251,11 +251,10 @@ describe("captureViewModel", () => {
|
||||
expect(input?.attrs).toEqual({ type: "password" });
|
||||
});
|
||||
|
||||
it("does not run conditional surface probes by default", async () => {
|
||||
it("never hovers the page while capturing", async () => {
|
||||
const cdp = makeCdp(fakeSnapshotReply());
|
||||
const result = await captureViewModel(cdp, 4);
|
||||
await captureViewModel(cdp, 4);
|
||||
|
||||
expect(result.surfaceProbes).toEqual([]);
|
||||
expect(cdp.send).not.toHaveBeenCalledWith(4, "Input.dispatchMouseEvent", expect.anything());
|
||||
});
|
||||
|
||||
@@ -319,9 +318,10 @@ describe("captureViewModel", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||
const captured = await captureViewModel(cdp, 4);
|
||||
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
|
||||
|
||||
expect(result.surfaceProbes).toEqual([
|
||||
expect(surfaceProbes).toEqual([
|
||||
{
|
||||
triggerBackendNodeId: 12,
|
||||
triggerPoint: { x: 956, y: 32 },
|
||||
@@ -378,9 +378,10 @@ describe("captureViewModel", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
const result = await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||
const captured = await captureViewModel(cdp, 4);
|
||||
const surfaceProbes = await probeHoverSurfaces(cdp, 4, captured.nodes);
|
||||
|
||||
expect(result.surfaceProbes).toEqual([
|
||||
expect(surfaceProbes).toEqual([
|
||||
expect.objectContaining({
|
||||
triggerBackendNodeId: 12,
|
||||
subItems: ["My profile", "Sign out"],
|
||||
@@ -422,7 +423,8 @@ describe("captureViewModel", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
|
||||
await captureViewModel(cdp, 4, { conditionalSurfaceProbe: true });
|
||||
const captured = await captureViewModel(cdp, 4);
|
||||
await probeHoverSurfaces(cdp, 4, captured.nodes);
|
||||
|
||||
expect(hoverMoves).toBe(1);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { ProbeBudget, resetOverlayBypassDepth, withOverlayBypass } from "../hover-perception";
|
||||
|
||||
afterEach(() => {
|
||||
resetOverlayBypassDepth();
|
||||
});
|
||||
|
||||
describe("ProbeBudget", () => {
|
||||
it("refuses a candidate that cannot finish inside the budget", () => {
|
||||
const budget = new ProbeBudget(500);
|
||||
|
||||
expect(budget.canAfford(400)).toBe(true);
|
||||
expect(budget.canAfford(600)).toBe(false);
|
||||
});
|
||||
|
||||
it("stops admitting candidates once the remaining budget is smaller than one", async () => {
|
||||
const budget = new ProbeBudget(60);
|
||||
await new Promise((resolve) => setTimeout(resolve, 40));
|
||||
|
||||
// A top-of-loop `elapsed > total` check would still admit this candidate
|
||||
// and then overshoot by its full cost.
|
||||
expect(budget.canAfford(50)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("withOverlayBypass", () => {
|
||||
it("toggles the overlay once around nested probe phases", async () => {
|
||||
const bypass = vi.fn(async () => {});
|
||||
|
||||
await withOverlayBypass(bypass, 4, async () => {
|
||||
await withOverlayBypass(bypass, 4, async () => {
|
||||
expect(bypass).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
expect(bypass).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
expect(bypass.mock.calls).toEqual([
|
||||
[4, true],
|
||||
[4, false],
|
||||
]);
|
||||
});
|
||||
|
||||
it("restores the overlay when the probe throws", async () => {
|
||||
const bypass = vi.fn(async () => {});
|
||||
|
||||
await expect(
|
||||
withOverlayBypass(bypass, 4, async () => {
|
||||
throw new Error("probe exploded");
|
||||
}),
|
||||
).rejects.toThrow("probe exploded");
|
||||
|
||||
expect(bypass).toHaveBeenLastCalledWith(4, false);
|
||||
});
|
||||
|
||||
it("reports a failed restore instead of swallowing it", async () => {
|
||||
const onRestoreFailure = vi.fn();
|
||||
const bypass = vi.fn(async (_tabId: number, enabled: boolean) => {
|
||||
if (!enabled) throw new Error("tab is gone");
|
||||
});
|
||||
|
||||
await withOverlayBypass(bypass, 4, async () => undefined, { onRestoreFailure });
|
||||
|
||||
expect(onRestoreFailure).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("does not pin the overlay off for the tab when a restore fails", async () => {
|
||||
const failing = vi.fn(async (_tabId: number, enabled: boolean) => {
|
||||
if (!enabled) throw new Error("tab is gone");
|
||||
});
|
||||
await withOverlayBypass(failing, 4, async () => undefined, { onRestoreFailure: () => {} });
|
||||
|
||||
const bypass = vi.fn(async () => {});
|
||||
await withOverlayBypass(bypass, 4, async () => undefined);
|
||||
|
||||
expect(bypass.mock.calls).toEqual([
|
||||
[4, true],
|
||||
[4, false],
|
||||
]);
|
||||
});
|
||||
|
||||
it("runs the probe untouched when no bypass is wired up", async () => {
|
||||
await expect(withOverlayBypass(undefined, 4, async () => "done")).resolves.toBe("done");
|
||||
});
|
||||
});
|
||||
@@ -132,6 +132,37 @@ describe("tooltip name enrichment", () => {
|
||||
expect(cdp.sendToTarget).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("remembers controls that revealed no tooltip so the miss is paid once", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async () => ({})) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "Page.createIsolatedWorld") return { executionContextId: 31 };
|
||||
if (method === "DOM.resolveNode") return { object: { objectId: "button-801" } };
|
||||
if (method === "Runtime.callFunctionOn") return { result: { value: [] } };
|
||||
return {};
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
};
|
||||
|
||||
const documents = [
|
||||
document({ tabId: 11, sessionId: "miss-session" }, 801, undefined, "miss-frame"),
|
||||
];
|
||||
const graph = semantics(documents);
|
||||
|
||||
const first = await probeTooltipNames(cdp, 11, documents, graph);
|
||||
expect(first.size).toBe(0);
|
||||
const hoversAfterFirst = vi
|
||||
.mocked(cdp.send)
|
||||
.mock.calls.filter(([, method]) => method === "Input.dispatchMouseEvent").length;
|
||||
expect(hoversAfterFirst).toBeGreaterThan(0);
|
||||
|
||||
const second = await probeTooltipNames(cdp, 11, documents, graph);
|
||||
expect(second.size).toBe(0);
|
||||
expect(
|
||||
vi.mocked(cdp.send).mock.calls.filter(([, method]) => method === "Input.dispatchMouseEvent")
|
||||
.length,
|
||||
).toBe(hoversAfterFirst);
|
||||
});
|
||||
|
||||
it("keeps tooltip evidence isolated across sibling frame sessions", async () => {
|
||||
let hoveredX = -10;
|
||||
const cdp: CdpRunner = {
|
||||
|
||||
@@ -9,7 +9,7 @@ import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
||||
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import type { CdpRunner } from "../shared";
|
||||
import { clearHover, waitForHover } from "./hover-perception";
|
||||
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
|
||||
|
||||
const REQUESTED_STYLES = ["position", "pointer-events", "cursor", "visibility", "opacity"] as const;
|
||||
const STYLE_COL = Object.fromEntries(
|
||||
@@ -71,14 +71,11 @@ export interface CapturedViewModel {
|
||||
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
|
||||
frameParentIds?: Map<string, string>;
|
||||
rootFrameId?: string;
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
export interface CaptureViewModelOptions {
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
@@ -368,7 +365,16 @@ function collectBackendIdsFromDomNode(node: CdpDomNode | undefined, out: Set<num
|
||||
}
|
||||
}
|
||||
|
||||
const MAX_HOVER_PROBE_MS = 2_000;
|
||||
/**
|
||||
* Ceiling for the whole hover-surface phase.
|
||||
*
|
||||
* The previous 2000 was only a floor: the loop checked elapsed time at the top,
|
||||
* so a candidate could start with 1ms left and still run its two settle
|
||||
* windows, pushing real cost to ~2.6s. This is that true ceiling, now actually
|
||||
* enforced by an up-front affordability check, so the same number of candidates
|
||||
* get probed under a bound that no longer lies.
|
||||
*/
|
||||
const MAX_HOVER_PROBE_MS = 2_600;
|
||||
const MAX_HOVER_TRIGGERS = 6;
|
||||
const MAX_HOVER_SURFACES = 3;
|
||||
const HOVER_SETTLE_MS = 300;
|
||||
@@ -614,13 +620,35 @@ function confidenceForHover(
|
||||
return "low";
|
||||
}
|
||||
|
||||
async function probeHoverSurfaces(
|
||||
/**
|
||||
* One candidate costs two settle windows (baseline + post-hover) plus a few
|
||||
* CDP round trips. Used to decide whether the next candidate still fits the
|
||||
* budget before paying for it.
|
||||
*/
|
||||
const HOVER_CANDIDATE_COST_MS = HOVER_SETTLE_MS * 2;
|
||||
|
||||
export interface HoverSurfaceProbeOptions {
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hovers a bounded set of likely menu triggers and reports the sub-items each
|
||||
* one reveals.
|
||||
*
|
||||
* Runs after DOM *and* accessibility capture. Hovering can open menus and
|
||||
* change layout, so probing between the two captures would leave the DOM half
|
||||
* of an observation describing the page before the change and the AX half
|
||||
* describing it after.
|
||||
*
|
||||
* The caller owns the overlay bypass span (see `withOverlayBypass`).
|
||||
*/
|
||||
export async function probeHoverSurfaces(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
nodes: CapturedNode[],
|
||||
options: CaptureViewModelOptions,
|
||||
options: HoverSurfaceProbeOptions = {},
|
||||
): Promise<CapturedSurfaceProbe[]> {
|
||||
const started = Date.now();
|
||||
const budget = new ProbeBudget(MAX_HOVER_PROBE_MS);
|
||||
try {
|
||||
throwIfAborted(options.signal);
|
||||
const cssScan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
@@ -635,60 +663,54 @@ async function probeHoverSurfaces(
|
||||
const results: CapturedSurfaceProbe[] = [];
|
||||
const seen = new Set<number>();
|
||||
throwIfAborted(options.signal);
|
||||
await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
|
||||
try {
|
||||
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
|
||||
throwIfAborted(options.signal);
|
||||
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
|
||||
if (!budget.canAfford(HOVER_CANDIDATE_COST_MS)) break;
|
||||
if (results.length >= MAX_HOVER_SURFACES) break;
|
||||
if (seen.has(candidate.backendNodeId)) continue;
|
||||
try {
|
||||
await clearHover(cdp, tabId);
|
||||
throwIfAborted(options.signal);
|
||||
if (Date.now() - started > MAX_HOVER_PROBE_MS) break;
|
||||
if (results.length >= MAX_HOVER_SURFACES) break;
|
||||
if (seen.has(candidate.backendNodeId)) continue;
|
||||
try {
|
||||
await clearHover(cdp, tabId);
|
||||
throwIfAborted(options.signal);
|
||||
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
||||
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
const baselineItems = runtimeValue<HoverRuntimeItem[]>(baselineReply) ?? [];
|
||||
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
||||
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
const baselineItems = runtimeValue<HoverRuntimeItem[]>(baselineReply) ?? [];
|
||||
|
||||
throwIfAborted(options.signal);
|
||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: candidate.x,
|
||||
y: candidate.y,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
||||
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
const subItems = diffHoverItems(
|
||||
baselineItems,
|
||||
runtimeValue<HoverRuntimeItem[]>(collected) ?? [],
|
||||
);
|
||||
if (subItems.length === 0) continue;
|
||||
seen.add(candidate.backendNodeId);
|
||||
results.push({
|
||||
triggerBackendNodeId: candidate.backendNodeId,
|
||||
triggerPoint: { x: candidate.x, y: candidate.y },
|
||||
triggerAction: "hover",
|
||||
subItems,
|
||||
confidence: confidenceForHover(candidate, subItems),
|
||||
});
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
continue;
|
||||
} finally {
|
||||
await clearHover(cdp, tabId);
|
||||
}
|
||||
throwIfAborted(options.signal);
|
||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: candidate.x,
|
||||
y: candidate.y,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
await waitForHover(HOVER_SETTLE_MS, options.signal);
|
||||
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
const subItems = diffHoverItems(
|
||||
baselineItems,
|
||||
runtimeValue<HoverRuntimeItem[]>(collected) ?? [],
|
||||
);
|
||||
if (subItems.length === 0) continue;
|
||||
seen.add(candidate.backendNodeId);
|
||||
results.push({
|
||||
triggerBackendNodeId: candidate.backendNodeId,
|
||||
triggerPoint: { x: candidate.x, y: candidate.y },
|
||||
triggerAction: "hover",
|
||||
subItems,
|
||||
confidence: confidenceForHover(candidate, subItems),
|
||||
});
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
continue;
|
||||
} finally {
|
||||
await clearHover(cdp, tabId);
|
||||
}
|
||||
} finally {
|
||||
await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
|
||||
}
|
||||
return results;
|
||||
} catch (err) {
|
||||
@@ -1042,7 +1064,6 @@ export async function captureViewModel(
|
||||
nodes: [],
|
||||
viewport,
|
||||
iframeNodes: new Map(),
|
||||
surfaceProbes: [],
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
}
|
||||
@@ -1071,11 +1092,6 @@ export async function captureViewModel(
|
||||
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()], options.signal);
|
||||
throwIfAborted(options.signal);
|
||||
|
||||
const surfaceProbes = options.conditionalSurfaceProbe
|
||||
? await probeHoverSurfaces(cdp, tabId, nodes, options)
|
||||
: [];
|
||||
throwIfAborted(options.signal);
|
||||
|
||||
const frameNodes = new Map<string, CapturedNode[]>();
|
||||
if (topContext.frameId) frameNodes.set(topContext.frameId, nodes);
|
||||
for (const iframeFrameNodes of iframeNodes.values()) {
|
||||
@@ -1096,7 +1112,6 @@ export async function captureViewModel(
|
||||
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
|
||||
frameParentIds: frameParsed.frameParentIds,
|
||||
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
|
||||
surfaceProbes,
|
||||
excludedBackendNodeIds,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -35,3 +35,91 @@ export async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wall-clock budget for a probe phase.
|
||||
*
|
||||
* Callers ask whether the *next* candidate still fits before starting it,
|
||||
* rather than checking elapsed time at the top of the loop. A top-of-loop
|
||||
* check lets a candidate start with 1ms of budget left and then run to
|
||||
* completion, overshooting by a full settle window.
|
||||
*/
|
||||
export class ProbeBudget {
|
||||
private readonly startedAt = Date.now();
|
||||
|
||||
constructor(private readonly totalMs: number) {}
|
||||
|
||||
get elapsedMs(): number {
|
||||
return Date.now() - this.startedAt;
|
||||
}
|
||||
|
||||
/** True when a candidate costing `estimatedMs` still fits in the budget. */
|
||||
canAfford(estimatedMs: number): boolean {
|
||||
return this.elapsedMs + estimatedMs <= this.totalMs;
|
||||
}
|
||||
}
|
||||
|
||||
export type OverlayBypass = (tabId: number, enabled: boolean) => Promise<void>;
|
||||
|
||||
export interface OverlayBypassScopeOptions {
|
||||
/**
|
||||
* Called when the overlay could not be restored. Defaults to `console.error`
|
||||
* so a stuck-bypassed overlay is never silent; tests inject a spy.
|
||||
*/
|
||||
onRestoreFailure?: (error: unknown) => void;
|
||||
}
|
||||
|
||||
const overlayBypassDepth = new Map<number, number>();
|
||||
|
||||
/**
|
||||
* Runs `body` with the agent overlay bypassed, restoring it afterwards.
|
||||
*
|
||||
* Reference counted per tab so nested probe phases share a single bypass span
|
||||
* instead of toggling the overlay once per phase. Restore failures are
|
||||
* reported rather than swallowed, and the depth is released even when restore
|
||||
* throws so a failure cannot pin the overlay off for the tab's lifetime.
|
||||
*/
|
||||
export async function withOverlayBypass<T>(
|
||||
bypass: OverlayBypass | undefined,
|
||||
tabId: number,
|
||||
body: () => Promise<T>,
|
||||
options: OverlayBypassScopeOptions = {},
|
||||
): Promise<T> {
|
||||
if (!bypass) return body();
|
||||
|
||||
const depth = overlayBypassDepth.get(tabId) ?? 0;
|
||||
overlayBypassDepth.set(tabId, depth + 1);
|
||||
if (depth === 0) {
|
||||
try {
|
||||
await bypass(tabId, true);
|
||||
} catch (error) {
|
||||
// Failing to hide the overlay only risks probing our own UI, so continue.
|
||||
console.debug("[bsk hover] overlay bypass could not be enabled", error);
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
return await body();
|
||||
} finally {
|
||||
const current = overlayBypassDepth.get(tabId) ?? 1;
|
||||
if (current <= 1) {
|
||||
overlayBypassDepth.delete(tabId);
|
||||
try {
|
||||
await bypass(tabId, false);
|
||||
} catch (error) {
|
||||
(options.onRestoreFailure ?? defaultRestoreFailure)(error);
|
||||
}
|
||||
} else {
|
||||
overlayBypassDepth.set(tabId, current - 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function defaultRestoreFailure(error: unknown): void {
|
||||
console.error("[bsk hover] agent overlay could not be restored after probing", error);
|
||||
}
|
||||
|
||||
/** Test seam: drops any bypass depth left behind by a crashed probe. */
|
||||
export function resetOverlayBypassDepth(): void {
|
||||
overlayBypassDepth.clear();
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { isVomReferenceNode } from "@browser-skill/vom";
|
||||
import { type CdpRunner, sendToCdpTarget } from "../shared";
|
||||
import type { FrameDocument, FrameOwnedAxNode } from "./frame-document";
|
||||
import { clearHover, waitForHover } from "./hover-perception";
|
||||
import { clearHover, ProbeBudget, waitForHover } from "./hover-perception";
|
||||
import { stableIdentifierName } from "./semantic-graph/name-evidence";
|
||||
import { frameBackendKey, type ResolvedSemanticGraph } from "./semantic-graph/types";
|
||||
|
||||
@@ -9,11 +9,18 @@ const MAX_TOOLTIP_CANDIDATES = 8;
|
||||
const MAX_TOOLTIP_PROBE_MS = 2_400;
|
||||
const TOOLTIP_SETTLE_MS = 250;
|
||||
const CACHE_TTL_MS = 5 * 60_000;
|
||||
/**
|
||||
* Misses expire sooner than hits: a control that gains a tooltip once the app
|
||||
* finishes booting should still be discoverable, but not at the cost of
|
||||
* re-probing every unlabelled control on every observation.
|
||||
*/
|
||||
const MISS_CACHE_TTL_MS = 60_000;
|
||||
const MAX_CACHE_ENTRIES = 500;
|
||||
const FORM_CONTROL_TAGS = new Set(["input", "select", "textarea"]);
|
||||
|
||||
interface TooltipCacheEntry {
|
||||
name: string;
|
||||
/** `null` records "probed, revealed no tooltip" so the miss is not re-paid. */
|
||||
name: string | null;
|
||||
expiresAt: number;
|
||||
}
|
||||
|
||||
@@ -27,7 +34,6 @@ interface TooltipCandidate<T extends FrameOwnedAxNode> {
|
||||
|
||||
export interface TooltipNameProbeOptions {
|
||||
signal?: AbortSignal;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
}
|
||||
|
||||
const tooltipCache = new Map<string, TooltipCacheEntry>();
|
||||
@@ -39,10 +45,11 @@ function cacheKey<T extends FrameOwnedAxNode>(
|
||||
return `${document.target.tabId}\u0000${document.frameId}\u0000${document.url ?? ""}\u0000${backendNodeId}`;
|
||||
}
|
||||
|
||||
function cachedName<T extends FrameOwnedAxNode>(
|
||||
/** Returns the cache entry, or `undefined` when this control was never probed. */
|
||||
function cachedEntry<T extends FrameOwnedAxNode>(
|
||||
document: FrameDocument<T>,
|
||||
backendNodeId: number,
|
||||
): string | undefined {
|
||||
): TooltipCacheEntry | undefined {
|
||||
const key = cacheKey(document, backendNodeId);
|
||||
const entry = tooltipCache.get(key);
|
||||
if (!entry) return undefined;
|
||||
@@ -50,13 +57,13 @@ function cachedName<T extends FrameOwnedAxNode>(
|
||||
tooltipCache.delete(key);
|
||||
return undefined;
|
||||
}
|
||||
return entry.name;
|
||||
return entry;
|
||||
}
|
||||
|
||||
function storeName<T extends FrameOwnedAxNode>(
|
||||
document: FrameDocument<T>,
|
||||
backendNodeId: number,
|
||||
name: string,
|
||||
name: string | null,
|
||||
): void {
|
||||
if (tooltipCache.size >= MAX_CACHE_ENTRIES) {
|
||||
const oldest = tooltipCache.keys().next().value;
|
||||
@@ -64,7 +71,7 @@ function storeName<T extends FrameOwnedAxNode>(
|
||||
}
|
||||
tooltipCache.set(cacheKey(document, backendNodeId), {
|
||||
name,
|
||||
expiresAt: Date.now() + CACHE_TTL_MS,
|
||||
expiresAt: Date.now() + (name === null ? MISS_CACHE_TTL_MS : CACHE_TTL_MS),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -237,22 +244,27 @@ export async function probeTooltipNames<T extends FrameOwnedAxNode>(
|
||||
const names = new Map<string, string>();
|
||||
const pending: TooltipCandidate<T>[] = [];
|
||||
for (const candidate of candidates(documents, graph)) {
|
||||
const cached = cachedName(candidate.document, candidate.backendNodeId);
|
||||
const cached = cachedEntry(candidate.document, candidate.backendNodeId);
|
||||
if (cached) {
|
||||
names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), cached);
|
||||
// A cached miss is still a decision: skip the control instead of re-probing.
|
||||
if (cached.name !== null) {
|
||||
names.set(
|
||||
frameBackendKey(candidate.document.frameId, candidate.backendNodeId),
|
||||
cached.name,
|
||||
);
|
||||
}
|
||||
} else if (pending.length < MAX_TOOLTIP_CANDIDATES) {
|
||||
pending.push(candidate);
|
||||
}
|
||||
}
|
||||
if (pending.length === 0) return names;
|
||||
|
||||
const started = Date.now();
|
||||
const budget = new ProbeBudget(MAX_TOOLTIP_PROBE_MS);
|
||||
const contexts = new Map<string, number>();
|
||||
await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
|
||||
try {
|
||||
for (const candidate of pending) {
|
||||
if (options.signal?.aborted) throw new DOMException("observation aborted", "AbortError");
|
||||
if (Date.now() - started >= MAX_TOOLTIP_PROBE_MS) break;
|
||||
if (!budget.canAfford(TOOLTIP_SETTLE_MS)) break;
|
||||
let objectId: string | undefined;
|
||||
try {
|
||||
let contextId = contexts.get(candidate.document.frameId);
|
||||
@@ -278,9 +290,9 @@ export async function probeTooltipNames<T extends FrameOwnedAxNode>(
|
||||
await waitForHover(TOOLTIP_SETTLE_MS, options.signal);
|
||||
const after = await tooltipTexts(cdp, candidate.document, objectId);
|
||||
const name = cleanTooltipDiff(before, after);
|
||||
storeName(candidate.document, candidate.backendNodeId, name ?? null);
|
||||
if (!name) continue;
|
||||
names.set(frameBackendKey(candidate.document.frameId, candidate.backendNodeId), name);
|
||||
storeName(candidate.document, candidate.backendNodeId, name);
|
||||
} catch (error) {
|
||||
if (error instanceof Error && error.name === "AbortError") throw error;
|
||||
} finally {
|
||||
@@ -290,7 +302,6 @@ export async function probeTooltipNames<T extends FrameOwnedAxNode>(
|
||||
}
|
||||
} finally {
|
||||
await clearHover(cdp, tabId);
|
||||
await options.hoverProbeBypassOverlay?.(tabId, false).catch(() => undefined);
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
@@ -37,6 +37,17 @@ export interface CaptureVomObservationResult {
|
||||
frames: CaptureVomFrame[];
|
||||
matchNodes: CaptureVomMatchNode[];
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
/**
|
||||
* Whether this observation actively hovered the page, and whether that
|
||||
* revealed content the static tree does not contain. Lets callers judge how
|
||||
* far the page may have drifted from the returned snapshot.
|
||||
*/
|
||||
hoverProbe?: HoverProbeReport;
|
||||
}
|
||||
|
||||
export interface HoverProbeReport {
|
||||
performed: boolean;
|
||||
revealedContent: boolean;
|
||||
}
|
||||
|
||||
function projectFrames(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomFrame[] {
|
||||
@@ -75,6 +86,7 @@ export function projectRecordSafeObservation(input: {
|
||||
frameDocuments: CapturedFrameDocument<FrameAxNode>[];
|
||||
rendered: VomResult;
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
hoverProbe?: HoverProbeReport;
|
||||
}): CaptureVomObservationResult {
|
||||
return {
|
||||
text: input.rendered.text,
|
||||
@@ -84,5 +96,6 @@ export function projectRecordSafeObservation(input: {
|
||||
frames: projectFrames(input.frameDocuments),
|
||||
matchNodes: projectMatchNodes(input.frameDocuments),
|
||||
surfaceProbes: input.surfaceProbes,
|
||||
...(input.hoverProbe ? { hoverProbe: input.hoverProbe } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
DEFAULT_DAEMON_PORT,
|
||||
normalizeDaemonPort,
|
||||
parseDaemonPortInput,
|
||||
resolveDaemonWsUrl,
|
||||
} from "../daemon-endpoint";
|
||||
|
||||
describe("daemon-endpoint", () => {
|
||||
it("resolveDaemonWsUrl(DEFAULT) matches the build-time constant", () => {
|
||||
expect(resolveDaemonWsUrl(DEFAULT_DAEMON_PORT)).toBe(__BSK_DAEMON_WS_URL__);
|
||||
});
|
||||
|
||||
it("resolveDaemonWsUrl replaces only the port", () => {
|
||||
expect(resolveDaemonWsUrl(53200)).toBe("ws://127.0.0.1:53200");
|
||||
});
|
||||
|
||||
it("normalizeDaemonPort falls back to default for invalid values", () => {
|
||||
expect(normalizeDaemonPort(undefined)).toBe(DEFAULT_DAEMON_PORT);
|
||||
expect(normalizeDaemonPort(0)).toBe(DEFAULT_DAEMON_PORT);
|
||||
expect(normalizeDaemonPort(65536)).toBe(DEFAULT_DAEMON_PORT);
|
||||
expect(normalizeDaemonPort("abc")).toBe(DEFAULT_DAEMON_PORT);
|
||||
expect(normalizeDaemonPort(1.5)).toBe(DEFAULT_DAEMON_PORT);
|
||||
});
|
||||
|
||||
it("normalizeDaemonPort accepts valid numbers and numeric strings", () => {
|
||||
expect(normalizeDaemonPort(53200)).toBe(53200);
|
||||
expect(normalizeDaemonPort("53200")).toBe(53200);
|
||||
});
|
||||
|
||||
it("parseDaemonPortInput treats empty as default and rejects invalid", () => {
|
||||
expect(parseDaemonPortInput("")).toBe(DEFAULT_DAEMON_PORT);
|
||||
expect(parseDaemonPortInput(" ")).toBe(DEFAULT_DAEMON_PORT);
|
||||
expect(parseDaemonPortInput("abc")).toBeNull();
|
||||
expect(parseDaemonPortInput("0")).toBeNull();
|
||||
expect(parseDaemonPortInput("65536")).toBeNull();
|
||||
});
|
||||
|
||||
it("parseDaemonPortInput accepts valid ports", () => {
|
||||
expect(parseDaemonPortInput("53200")).toBe(53200);
|
||||
expect(parseDaemonPortInput(" 52800 ")).toBe(52800);
|
||||
});
|
||||
});
|
||||
@@ -270,4 +270,19 @@ describe("WSTransport", () => {
|
||||
lastSocket().receive({ id: "2", result: 2 });
|
||||
expect(handler).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("setUrl returns false for the same URL and uses the new URL on the next connect", async () => {
|
||||
const t = new WSTransport({
|
||||
url: "ws://127.0.0.1:52800",
|
||||
webSocketFactory: (url) => new FakeSocket(url) as unknown as WebSocket,
|
||||
});
|
||||
expect(t.setUrl("ws://127.0.0.1:52800")).toBe(false);
|
||||
expect(t.setUrl("ws://127.0.0.1:53200")).toBe(true);
|
||||
|
||||
const p = t.connect();
|
||||
expect(lastSocket().url).toBe("ws://127.0.0.1:53200");
|
||||
lastSocket().open();
|
||||
await p;
|
||||
expect(t.state).toBe("connected");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
/** Build-time default WebSocket URL (see wxt.config.ts). */
|
||||
const base = new URL(__BSK_DAEMON_WS_URL__);
|
||||
|
||||
export const DEFAULT_DAEMON_PORT = Number(base.port) || 52800;
|
||||
|
||||
const MIN_PORT = 1;
|
||||
const MAX_PORT = 65535;
|
||||
|
||||
function isValidPort(n: number): boolean {
|
||||
return Number.isInteger(n) && n >= MIN_PORT && n <= MAX_PORT;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compose the daemon WebSocket URL for a loopback port, preserving the
|
||||
* build-time host and path from {@link __BSK_DAEMON_WS_URL__}.
|
||||
*/
|
||||
export function resolveDaemonWsUrl(port: number): string {
|
||||
const path = base.pathname === "/" ? "" : base.pathname;
|
||||
return `${base.protocol}//${base.hostname}:${port}${path}`;
|
||||
}
|
||||
|
||||
/** Non-integer / out-of-range / unset → {@link DEFAULT_DAEMON_PORT}. */
|
||||
export function normalizeDaemonPort(raw: unknown): number {
|
||||
if (typeof raw === "number" && isValidPort(raw)) return raw;
|
||||
if (typeof raw === "string" && raw.trim() !== "") {
|
||||
const parsed = Number.parseInt(raw.trim(), 10);
|
||||
if (isValidPort(parsed)) return parsed;
|
||||
}
|
||||
return DEFAULT_DAEMON_PORT;
|
||||
}
|
||||
|
||||
/** Input semantics: empty string → default port; invalid → null. */
|
||||
export function parseDaemonPortInput(text: string): number | null {
|
||||
const trimmed = text.trim();
|
||||
if (trimmed === "") return DEFAULT_DAEMON_PORT;
|
||||
if (!/^\d+$/.test(trimmed)) return null;
|
||||
const parsed = Number.parseInt(trimmed, 10);
|
||||
if (!isValidPort(parsed)) return null;
|
||||
return parsed;
|
||||
}
|
||||
@@ -33,10 +33,22 @@ export type RpcErrorReason =
|
||||
| "restricted_tab_url"
|
||||
| "borrow_conflict"
|
||||
| "screenshot_capture_failed"
|
||||
| "file_input_probe_failed"
|
||||
| "file_input_not_activated"
|
||||
| "set_file_input_failed"
|
||||
| "download_capture_failed"
|
||||
| "transfer_outcome_unknown"
|
||||
| "transfer_timeout"
|
||||
| "cleanup_failed";
|
||||
|
||||
export type TransferEffectState = "none" | "committed" | "unknown";
|
||||
export type TransferCleanupState = "complete" | "failed";
|
||||
|
||||
export interface RpcErrorData {
|
||||
reason?: RpcErrorReason;
|
||||
effect_state?: TransferEffectState;
|
||||
phase?: string;
|
||||
cleanup_state?: TransferCleanupState;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
@@ -318,9 +330,14 @@ export interface SnapshotResult {
|
||||
|
||||
export interface ObserveParams extends SnapshotParams {
|
||||
debug_surfaces?: boolean;
|
||||
probe_hover?: boolean;
|
||||
}
|
||||
|
||||
export interface ObserveResult extends SnapshotResult {
|
||||
hover_probe?: {
|
||||
performed: boolean;
|
||||
revealed_content: boolean;
|
||||
};
|
||||
debug?: {
|
||||
surface_probes?: Array<{
|
||||
trigger_backend_node_id: number;
|
||||
@@ -500,6 +517,50 @@ export interface SelectResult {
|
||||
dialogs?: JavaScriptDialogInfo[];
|
||||
}
|
||||
|
||||
export interface UploadFile {
|
||||
transfer_id: string;
|
||||
name: string;
|
||||
staged_path?: string;
|
||||
}
|
||||
|
||||
export interface UploadParams {
|
||||
session_id: string;
|
||||
ref?: string;
|
||||
selector?: string;
|
||||
tab_id?: number;
|
||||
files: UploadFile[];
|
||||
timeout_ms?: number;
|
||||
}
|
||||
|
||||
export interface UploadResult {
|
||||
tab_id: number;
|
||||
used_ref?: string;
|
||||
used_selector?: string;
|
||||
file_names: string[];
|
||||
}
|
||||
|
||||
export interface DownloadParams {
|
||||
session_id: string;
|
||||
ref?: string;
|
||||
selector?: string;
|
||||
tab_id?: number;
|
||||
timeout_ms?: number;
|
||||
browser_relative_dir?: string;
|
||||
max_byte_size?: number;
|
||||
}
|
||||
|
||||
export interface DownloadResult {
|
||||
tab_id: number;
|
||||
used_ref?: string;
|
||||
used_selector?: string;
|
||||
suggested_filename: string;
|
||||
byte_size: number;
|
||||
mime?: string;
|
||||
danger?: string;
|
||||
browser_path?: string;
|
||||
transfer_id?: string;
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// M9 tool payloads — evaluate / wait_for_navigation / wait_ms
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
@@ -40,7 +40,7 @@ interface MessageLikeEvent {
|
||||
* (1s, 2s, 4s, …, capped at 5s) until `disconnect()` is called.
|
||||
*/
|
||||
export class WSTransport implements Transport {
|
||||
private readonly url: string;
|
||||
private url: string;
|
||||
private readonly factory: WebSocketFactory;
|
||||
private readonly initialDelayMs: number;
|
||||
private readonly maxDelayMs: number;
|
||||
@@ -69,6 +69,13 @@ export class WSTransport implements Transport {
|
||||
return this.currentState;
|
||||
}
|
||||
|
||||
/** Returns whether the URL changed. Does not reconnect. */
|
||||
setUrl(url: string): boolean {
|
||||
if (url === this.url) return false;
|
||||
this.url = url;
|
||||
return true;
|
||||
}
|
||||
|
||||
connect(): Promise<void> {
|
||||
if (this.currentState === "connected") return Promise.resolve();
|
||||
|
||||
|
||||
@@ -30,6 +30,7 @@ export default defineConfig({
|
||||
permissions: [
|
||||
"alarms",
|
||||
"debugger",
|
||||
"downloads",
|
||||
"idle",
|
||||
"notifications",
|
||||
"tabs",
|
||||
|
||||
@@ -44,6 +44,7 @@
|
||||
"!packages/dsh-plugin-browserskill/src/client/bsk-ui.nomodule.css",
|
||||
"!packages/dsh-plugin-browserskill/.bsk-ui.raw.css",
|
||||
"!packages/dsh-plugin-browserskill/.bsk-ui.input.css",
|
||||
"!evals/browser/results/**",
|
||||
"!**/target/**",
|
||||
"!**/.wxt/**",
|
||||
"!**/.output/**",
|
||||
|
||||
@@ -53,6 +53,7 @@ flate2 = { workspace = true }
|
||||
tar = { workspace = true }
|
||||
zip = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
base64 = { workspace = true }
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
nix = { workspace = true }
|
||||
@@ -63,6 +64,7 @@ windows-sys = { version = "0.59", features = [
|
||||
"Win32_Foundation",
|
||||
"Win32_System_Threading",
|
||||
"Win32_Security",
|
||||
"Win32_Storage_FileSystem",
|
||||
] }
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
+127
-161
@@ -1,160 +1,125 @@
|
||||
---
|
||||
name: browser-skill
|
||||
description: |
|
||||
Use when the user asks to perform browser automation tasks against their
|
||||
logged-in browser: visit and read pages, fill forms, scrape data, click
|
||||
through a flow, regression-test a PR's UI, validate a deployed page.
|
||||
Requires the bsk CLI installed and the browser-skill extension loaded.
|
||||
Use when the user asks to automate their logged-in Chromium browser: visit
|
||||
and read pages, fill forms, scrape data, click through flows, regression-test
|
||||
a PR's UI, validate a deployed page, or operate a tab they identify. Requires
|
||||
the bsk CLI and browser extension.
|
||||
---
|
||||
|
||||
# browser-skill
|
||||
|
||||
Drive the user's **real Chromium browser** (with their logins and cookies) through the `bsk` CLI. The extension opens an isolated **Agent Window** for automation; the user's normal windows stay protected unless you explicitly borrow a tab.
|
||||
Drive the user's real Chromium browser through `bsk`. Automation runs in an isolated **Agent
|
||||
Window** with the user's existing logins and cookies. User-window tabs remain protected unless they
|
||||
are explicitly borrowed.
|
||||
|
||||
## When to use
|
||||
Do not use this skill for tasks with no browser, for extension installation, or when the user only
|
||||
wants instructions. Never extract credentials, cookies, tokens, or other secrets from pages.
|
||||
|
||||
- Open pages, read titles/text, scrape structured data from sites the user can already access
|
||||
- Fill forms, click through multi-step flows, smoke-test a UI change
|
||||
- Understand pages with `bsk snapshot` first; use `bsk get-html` or `bsk screenshot` only when the snapshot is insufficient
|
||||
- Operate on a specific user tab they point you at (after `bsk tab borrow`)
|
||||
## Required lifecycle
|
||||
|
||||
## When NOT to use
|
||||
Every browser task owns a bounded session:
|
||||
|
||||
- Tasks with **no browser** involved (files, APIs, databases only)
|
||||
- Installing or configuring the extension (point the user to setup docs instead)
|
||||
- **Credential harvesting** — never run `bsk evaluate` on banking, SSO, or password-manager pages to extract tokens, cookies, or secrets
|
||||
- Long-lived control of a user's personal login window — borrow only for the immediate step, then `bsk tab return` or end the session
|
||||
- Replacing the user's manual browsing when they only wanted an explanation
|
||||
|
||||
## Prerequisites
|
||||
|
||||
1. `bsk` on `PATH` (Rust CLI from browser-skill)
|
||||
2. browser-skill **extension** loaded in Chromium and connected (popup shows green)
|
||||
3. Any `bsk` command auto-starts background services as needed; use `bsk doctor` if anything fails
|
||||
|
||||
## Mandatory workflow
|
||||
|
||||
Every automation task **must** follow this lifecycle. Do **not** rely on idle timeouts (default session idle is 5 minutes).
|
||||
|
||||
```
|
||||
1. bsk session start → capture the 4-letter session id printed on stdout
|
||||
2. … every tool command … → always pass --session <id>
|
||||
3. bsk session stop <id> → REQUIRED when done (even on error paths)
|
||||
```text
|
||||
1. bsk session start # retain the printed 4-letter session id
|
||||
2. bsk ... --session <id> # pass it to every session-scoped command
|
||||
3. bsk session stop <id> # always run on success and error paths
|
||||
```
|
||||
|
||||
Optional: `bsk session start --browser <instance-id-or-label>` when multiple browsers are connected (`bsk browsers` / error output lists them). Add `--no-focus` to open the Agent Window in the background without stealing focus from the user's current window.
|
||||
Do not rely on the idle timeout for cleanup. Stop the session as soon as the goal is met unless the
|
||||
user explicitly asks to keep it open. Stopping also returns borrowed tabs.
|
||||
|
||||
Emergency cleanup: `bsk session stop --all` or the Agent Window overlay **Stop all**.
|
||||
Any `bsk` command auto-starts the background services it needs; never manage the daemon by hand.
|
||||
When multiple browsers are connected, use `bsk browsers` and start with
|
||||
`bsk session start --browser <id-or-label>`. Add `--no-focus` to that same start command when the
|
||||
Agent Window does not need to interrupt the user's current work; it is not a flag on other commands.
|
||||
Run `bsk doctor` when startup or transport problems persist after one retry.
|
||||
|
||||
## Stop when the goal is met
|
||||
## Work toward one observable goal
|
||||
|
||||
Every task is a **bounded goal**, not open-ended browsing. The goal may come from the user's request, a recorded `trace.json`, or both.
|
||||
- Derive a concrete success condition from the user's request or a supplied trace.
|
||||
- Take the shortest purposeful path: observe, act, then make at most one observation to confirm an
|
||||
ambiguous result.
|
||||
- Once success is visible, do not click, refresh, navigate, switch tabs, or perform extra checks.
|
||||
- If a human-only step appears or two attempts make no progress, request help instead of
|
||||
brute-forcing.
|
||||
|
||||
1. **Define success first** — one concrete, observable condition derived from the user's words, `purpose`, or the last meaningful step in a trace (e.g. "form submitted", "item added to cart", "playback started").
|
||||
2. **Take the shortest path** — snapshot → act → at most one check. Do not wander, re-try unrelated actions, or stack exploratory steps.
|
||||
3. **Stop as soon as success is reached** — run `bsk session stop <id>` immediately unless the user explicitly asked to keep the session open (e.g. "don't close yet", "keep browsing").
|
||||
4. **No post-success work** — once the goal is met, do not click, refresh, navigate, re-search, switch tabs, or "double-check" that it worked. Further verification is a new task.
|
||||
5. **When blocked, pause — do not brute-force** — if the page requires human input (login, captcha, OTP, payment confirmation) or an action fails twice with no progress, call `bsk request-help` instead of retrying blindly. See **Ask the human for help** below.
|
||||
6. **When unsure** — at most one extra `bsk snapshot`. If success looks met, stop. If not, ask the user; do not keep clicking.
|
||||
With a trace, follow its semantic target information and values in order, but treat its refs as
|
||||
record-local hints. Stop when its purpose or last meaningful effect is satisfied. A trace guides the
|
||||
task; it does not expand the user's goal or authorize additional actions.
|
||||
|
||||
**With a trace:** replay steps in order using `target` role/name/tag and raw `value`/`selection` fields. After the last step (or when its `effect.navigated_to` / success hint is satisfied), apply rules 3–4 immediately. The trace guides execution; it does not extend control beyond the goal.
|
||||
## Observe, act, observe
|
||||
|
||||
**Without a trace:** the user's request *is* the success condition. Satisfying it ends the task — same stop rules apply.
|
||||
Use this default loop:
|
||||
|
||||
## Core interaction loop
|
||||
|
||||
Write operations only affect tabs in the **Agent Window** (or tabs you **borrowed** into it).
|
||||
|
||||
```
|
||||
```text
|
||||
bsk navigate <url> --session <id>
|
||||
bsk observe --session <id> → primary semantic VOM view; reveals hover/focus surfaces
|
||||
bsk snapshot --session <id> → static aria tree fallback when VOM is insufficient
|
||||
bsk hover @e3 --session <id> → reveal hover-triggered menus before re-observing/clicking
|
||||
bsk click @e4 --session <id> → or bsk fill, bsk select, bsk press
|
||||
bsk observe --session <id> → again after navigation / DOM change
|
||||
bsk observe --session <id>
|
||||
bsk click|hover|fill|select|press ... --session <id>
|
||||
bsk observe --session <id> # after navigation or a meaningful DOM change
|
||||
```
|
||||
|
||||
**Refs invalidate after navigation** — always re-snapshot before clicking, filling, or selecting on a new page.
|
||||
Prefer fresh `@eN` refs over CSS selectors. Navigation invalidates refs; large DOM changes may also
|
||||
make them stale. Observe again before the next interaction.
|
||||
|
||||
Prefer `@eN` refs from the latest snapshot over raw CSS selectors. Use `--ref` / `--selector` when ambiguous (`bsk click --help`).
|
||||
An observation marks a hover-only surface as `@e1 button "Products" [hover first: Shoes | Bags]`.
|
||||
The listed items are labels, not usable refs: hover the trigger, observe again, then act on the
|
||||
revealed item's own ref. Do not click the trigger itself unless the user wants the trigger's action.
|
||||
`[has-submenu]` and `[expanded]` mark the same kind of trigger without listing what it hides.
|
||||
|
||||
When VOM renders `[hover first: …]` on an element, the listed items are not currently clickable refs. Run `bsk hover <that-ref> --session <id>`, then immediately run `bsk snapshot` or `bsk observe` again and click the newly visible menu item ref. Do not click the trigger itself unless the user explicitly wants the trigger action.
|
||||
`bsk observe` does not hover the page on its own. Reach for `--probe-hover` when a control you have
|
||||
good reason to expect is absent **and** no marker points at a trigger — that combination is what a
|
||||
CSS-only hover menu looks like from here. It hovers a bounded set of likely triggers, so it costs a
|
||||
few seconds and touches the live page; once you know which element hides the menu, `bsk hover <ref>`
|
||||
is cheaper and more precise.
|
||||
|
||||
## Observation priority
|
||||
Escalate page reading only as needed:
|
||||
|
||||
Start with `bsk observe` to understand page structure, text, controls, element refs, and conditional hover/focus surfaces. Use `bsk snapshot` only when you need the stricter static accessibility tree or VOM is insufficient. Only escalate to raw HTML or screenshots when the latest observation cannot answer the question:
|
||||
1. `bsk observe` for normal semantic understanding, text, controls, and refs.
|
||||
2. `bsk observe --probe-hover` once when an expected control is missing and no marker points at a
|
||||
trigger.
|
||||
3. `bsk snapshot` when a stricter static accessibility tree is more useful.
|
||||
4. `bsk get-html` for exact markup or hidden metadata that semantic views cannot provide.
|
||||
5. `bsk screenshot` for layout, styling, canvas, images, or requested visual evidence.
|
||||
|
||||
1. `bsk observe` — primary semantic VOM observation; may run bounded perception probes such as hover-surface discovery
|
||||
2. `bsk snapshot` — strict static accessibility tree fallback
|
||||
3. `bsk get-html` — when hidden DOM, metadata, or markup details are required
|
||||
4. `bsk screenshot` — when visual layout, canvas/image content, or styling cannot be inferred from the observation. Use `--ref @eN` (from the latest snapshot/observe) to crop to one element; omit `--ref` for the full visible tab.
|
||||
Do not start with raw HTML or screenshots merely to discover ordinary controls. When interaction is
|
||||
needed, obtain a fresh observation before acting on screenshot or HTML findings.
|
||||
|
||||
Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page.
|
||||
## Respect the Agent Window boundary
|
||||
|
||||
## Sandbox rules
|
||||
Normal page writes affect only Agent Window tabs. To operate a user tab, first list it with
|
||||
`bsk tab list --scope user --session <id>`, then `bsk tab borrow <tab-id>`. Return it immediately
|
||||
after the relevant step with `bsk tab return <tab-id>`; never invent a tab id or keep a personal tab
|
||||
borrowed across unrelated work.
|
||||
|
||||
| Rule | Detail |
|
||||
|------|--------|
|
||||
| Agent Window | `bsk tab create`, `bsk navigate`, `bsk click`, etc. work on agent tabs by default |
|
||||
| User tabs | Read-only until borrowed: `bsk tab list --session <id> --scope user` then `bsk tab borrow <tab-id> --session <id>` |
|
||||
| Return borrowed tabs | Call `bsk tab return <tab-id> --session <id>` when finished; unreturned tabs are **auto-returned** on `bsk session stop` |
|
||||
| Writes off-agent | Commands that mutate the page fail if the tab is not in the Agent Window — borrow or create a tab first |
|
||||
## Ask the human when needed
|
||||
|
||||
## Global flags
|
||||
Use `bsk request-help` for login, captcha, OTP, payment confirmation, consent, or another step the
|
||||
user must complete. Give a precise prompt and pass fresh `--target` refs/selectors when concrete
|
||||
controls can be highlighted. Use completion criteria only when the page has a clear stable success
|
||||
signal.
|
||||
|
||||
| Flag | Purpose |
|
||||
|------|---------|
|
||||
| `--json` | Machine-readable JSON on stdout (errors too) |
|
||||
| `--quiet` | Suppress informational stderr |
|
||||
| `-v` / `-vv` | More verbose logging |
|
||||
The result `outcome` is one of `continued`, `completed`, `cancelled`, `timed_out`, or `disabled`
|
||||
(`navigated` is deprecated — never treat navigation as a completion signal). Resume only after
|
||||
`continued` or `completed`. Treat `cancelled` as rejection, and `timed_out` or `disabled` as a
|
||||
blocker rather than a reason to retry. After control returns, run a fresh `bsk observe` before
|
||||
reasoning about the page or using refs.
|
||||
|
||||
Auto-update is **on by default** — the background daemon upgrades `bsk` itself when a new release is available (postponed while a session is active). Set `BSK_AUTO_UPDATE=off` to disable it and upgrade manually with `bsk update`.
|
||||
## Command inventory
|
||||
|
||||
Command-specific flags (timeouts, `--tab-id`, `--wait-until`, …): **`bsk <cmd> --help`**
|
||||
This list of names is complete. Never invent a command outside it; read
|
||||
`bsk <command...> --help` for flags instead of guessing them.
|
||||
|
||||
## CLI command reference (one line each)
|
||||
|
||||
Details and flags: **`bsk <cmd> --help`**
|
||||
|
||||
### Diagnostics
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk status` | Connection health, connected browsers, active sessions |
|
||||
| `bsk doctor` | Deep diagnostics and repair hints |
|
||||
| `bsk browsers` | List connected browser instances (ids, labels, versions) |
|
||||
|
||||
### Session
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk session start` | Open Agent Window (`--width`/`--height` for initial size); prints **4-letter session id** |
|
||||
| `bsk session start --no-focus` | Open Agent Window in the background without stealing focus |
|
||||
| `bsk session stop <id>` | End session, close Agent Window, auto-return borrowed tabs |
|
||||
| `bsk session stop --all` | Stop every active session |
|
||||
| `bsk session list` | List active sessions |
|
||||
|
||||
### Window (require `--session <id>`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk window resize` | Resize the Agent Window (`--width`, `--height`; 100..=7680 CSS px) |
|
||||
|
||||
### Device emulation — `bsk emulate` (requires `--session <id>`)
|
||||
|
||||
Emulate a mobile device environment on the agent tab via CDP — viewport, User-Agent, and touch — to debug a page's mobile layout and behaviour:
|
||||
|
||||
```bash
|
||||
bsk emulate --session <id> --device iphone-14
|
||||
bsk emulate --session <id> --width 390 --height 844 --dpr 3 --mobile --ua "Mozilla/5.0 (iPhone…" --touch
|
||||
bsk emulate --session <id> --off
|
||||
```text
|
||||
session start|stop|list browsers status doctor update logs
|
||||
navigate navigate-back navigate-forward reload wait-for-navigation wait-ms
|
||||
observe snapshot get-html screenshot console network
|
||||
click hover fill select press evaluate
|
||||
tab list|create|close|select|borrow|return window resize emulate
|
||||
request-help record start|stop
|
||||
```
|
||||
|
||||
- Presets (`--device`): `iphone-14`, `iphone-14-pro-max`, `iphone-se`, `pixel-7`, `galaxy-s23`, `ipad-mini`, `galaxy-tab-s8`. Manual flags (`--width`/`--height`/`--dpr`/`--mobile`/`--ua`/`--accept-language`/`--touch`/`--max-touch-points`) also work without a preset, or override individual preset fields; `--no-mobile`/`--no-touch` turn a preset's mobile viewport / touch emulation back off.
|
||||
- Repeated runs merge field by field onto the tab's current emulation state — only the flags you pass change. E.g. after `--device iphone-14`, `bsk emulate --session <id> --width 390 --height 844` keeps the preset's dpr (`3`) and mobile viewport. (The extension remembers the state per tab; after an extension reload the next run applies only the fields it carries.)
|
||||
- `--off` clears every override (viewport, UA, touch) and the remembered state, restoring the tab's real environment.
|
||||
- Scope: overrides apply to **one tab only** (CDP per-target) and are **not inherited by new tabs** — re-run `bsk emulate` after opening or switching to another tab (default target is the session's active tab; `--tab-id` overrides).
|
||||
- Emulation covers viewport/UA/touch only; it does not throttle the network, fake geolocation, or synthesise real touch-event streams.
|
||||
Required flags that are easy to get wrong:
|
||||
|
||||
### Tabs (require `--session <id>`)
|
||||
|
||||
@@ -206,11 +171,34 @@ Both capture from the moment the tab is attached and read a bounded per-tab buff
|
||||
| `bsk select <ref-or-selector> --value <v>` | Set `<select>` option(s) by `value` (repeat `--value` for multi-select) |
|
||||
| `bsk press <key>` | Key/combo (`Enter`, `Ctrl+A`, …; optional `--ref` to focus first) |
|
||||
|
||||
### File transfer (require `--session`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk upload <ref-or-selector> --file <path>` | Click one upload trigger and attach an agent-readable local file (`--file` is repeatable) |
|
||||
| `bsk download <ref-or-selector> --out <path>` | Click one download trigger and copy the single completed file to an exact local path (`--overwrite` is opt-in) |
|
||||
|
||||
The agent/harness decides whether a file transfer is appropriate and which local path belongs to the task. Treat upload as disclosure of that file to the current website, and download as accepting website-controlled bytes onto the local filesystem. Use only paths that are necessary for the user's bounded goal.
|
||||
|
||||
BrowserSkill enforces the mechanical boundary: files are staged under a session-scoped opaque transfer, only daemon-minted capabilities reach the extension, upload/download still obey Agent Window tab checks, and transfers are chunk/size bounded. Upload intercepts the native chooser for one transaction, locates the input activated in the resolved target's document, and assigns only the staged file paths. Download uniquely correlates one exact-target browser intent with one Chrome download in either event order, routes it through a daemon-minted relative directory, and lets the daemon validate and import it. Upload staging remains available for a later form submission and is removed when the session ends. Downloads cannot overwrite an existing destination unless `--overwrite` is explicit. BrowserSkill does not inspect file content or decide whether its meaning is sensitive.
|
||||
|
||||
Do not use `request-help` merely because a native file chooser or browser download is involved; try these commands first. For transfer failures, use the structured error instead of retrying blindly:
|
||||
|
||||
- `reason=file_input_not_activated` means the requested click did not activate exactly one `<input type="file">`; the page may use a non-input picker such as `window.showOpenFilePicker()`. Do not retry blindly. If human help is available, call `request-help` and tell the user the exact original local path to choose. The staged daemon path is internal and must not be shown to the user.
|
||||
- `reason=file_input_probe_failed` means BrowserSkill could not safely establish the browser-side upload transaction. Do not repeat the same action; use `request-help` when available.
|
||||
- `reason=set_file_input_failed` means BrowserSkill found the activated file input but Chrome rejected the staged path or assignment. Check the extension's file-URL access permission; otherwise use `request-help`.
|
||||
- `reason=download_capture_failed` means BrowserSkill could not attribute exactly one completed download to the requested target. Do not retry blindly or accept an unrelated browser download; use `request-help` when available.
|
||||
- `effect_state=none` means BrowserSkill confirmed that no file-transfer effect was committed. Follow the accompanying reason; a corrected target or explicit human fallback may be attempted.
|
||||
- `effect_state=unknown` means the browser may already have attached or created the file. Do not repeat the transfer. Observe the page if that can establish the result; otherwise stop and report the uncertainty.
|
||||
- `effect_state=committed` means the browser-side effect occurred even if later completion or cleanup failed. Do not repeat it; continue only after verifying the resulting page/download state.
|
||||
|
||||
If `request-help` returns `outcome="disabled"`, do not retry it. Stop gracefully and report the transfer mechanism that requires human intervention.
|
||||
|
||||
### Scripting & timing
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk evaluate <expression>` | Run JS in agent tab (see red lines); JS throw → stderr, **exit 0** |
|
||||
| `bsk evaluate <expression>` | Run JS in agent tab (see red lines). JS throw → stderr, **exit 0** (RPC success); use `--json` and check `.ok` to detect JS errors |
|
||||
| `bsk wait-for-navigation` | Block until load/DOM idle/etc. (`--wait-until`, `--timeout`) |
|
||||
| `bsk wait-ms <duration>` | Sleep (`500ms`, `2s`, `1m`; **no** `--session`) |
|
||||
|
||||
@@ -280,48 +268,26 @@ bsk record start --browser <instance-id-or-label> \
|
||||
bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable
|
||||
```
|
||||
|
||||
- **v3 bundle (preferred):** `--output` is a directory (default `./trace`) containing `trace.json` and `states/`. `trace.json` lists `states[]` (page observation ids) and `steps[]` (each step binds `state` = observe snapshot *before* the action and `result.state` = snapshot *after* settle). Page bodies live in `states/sN.txt` using the same VOM format as `bsk observe`.
|
||||
- **v2 fallback:** when the connected extension is older, export may contain only legacy `trace.json` with `pages[]` action context and no `states/` observation files. Update the extension for full v3 bundles.
|
||||
- Each `states[]` entry is one **settled page observation** (captured after recording start, navigation landing, or action settle — not on a timer).
|
||||
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
|
||||
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
||||
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
|
||||
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
||||
`select` matches an option's `value` attribute, not its visible label. Device preset ids are
|
||||
lowercase and hyphenated, such as `iphone-14`.
|
||||
|
||||
## Error handling
|
||||
- `console` and `network` provide bounded, read-only debugging evidence.
|
||||
- `emulate` applies viewport, user-agent, and touch overrides to one tab; new tabs do not inherit
|
||||
them. Use `--off` to restore the real environment.
|
||||
- `evaluate` is a last resort when observe plus normal interactions cannot complete the task. With
|
||||
`--json`, inspect `.ok`: a JavaScript exception may still have CLI exit code 0 because the RPC
|
||||
succeeded. Never evaluate credential surfaces to read storage, cookies, or auth data.
|
||||
- `record` captures a user's actions for later replay. Read `bsk record start --help` before use,
|
||||
and never record banking, SSO, password-manager, or other sensitive pages.
|
||||
|
||||
### Exit codes (`echo $?` after `bsk …`)
|
||||
## Recover without wandering
|
||||
|
||||
| Code | Meaning | What to do |
|
||||
|------|---------|------------|
|
||||
| `0` | Success (including `evaluate` where JS threw but RPC succeeded) | Continue |
|
||||
| `1` | User error — bad args, unknown session, tab not in Agent Window, stale ref | Fix args; `bsk session list`; re-snapshot |
|
||||
| `2` | Protocol / transport — service unreachable, IPC failure | `bsk doctor`; check extension connected; retry the command |
|
||||
| `3` | Browser / CDP execution failed | Retry; simplify selector; check tab still open |
|
||||
| `4` | Timeout | Increase `--timeout`; try `--wait-until domcontentloaded` |
|
||||
| `5` | Version skew (CLI vs extension) | Upgrade/reinstall matching versions |
|
||||
- Stale ref: observe again and retry the intended action once.
|
||||
- Unknown tab or session: list current tabs/sessions; never guess identifiers.
|
||||
- Timeout: inspect current page state before deciding whether one longer purposeful wait is useful.
|
||||
- Unsupported command: continue with available capabilities; suggest updating only when the missing
|
||||
command is necessary.
|
||||
- Unrecoverable failure: report the blocker and stop the session in a finally-style path.
|
||||
|
||||
Human errors print `error:` + `hint:` on stderr; `--json` includes `code`, `message`, `hint`, `exit_code`.
|
||||
|
||||
### When to run diagnostics
|
||||
|
||||
| Situation | Command |
|
||||
|-----------|---------|
|
||||
| Before first task in a session | `bsk status` — extension connected? |
|
||||
| Any failure you cannot fix in one retry | `bsk doctor` |
|
||||
| Multiple browsers / wrong target | `bsk browsers` then `bsk session start --browser <id>` |
|
||||
|
||||
Always **`bsk session stop <id>`** in a `finally`-style path so the Agent Window closes and borrowed tabs return.
|
||||
|
||||
## Red lines
|
||||
|
||||
1. **No token theft** — do not `bsk evaluate` on sensitive sites to read `localStorage`, cookies, or auth headers for exfiltration.
|
||||
2. **No long borrow** — do not leave a user's personal tab in the Agent Window across unrelated tasks.
|
||||
3. **No skip stop** — always `bsk session stop <id>`; never assume idle timeout will clean up.
|
||||
4. **No post-success control** — once the user’s goal (or last trace step) is met, do not keep operating the page; stop the session unless they asked to keep it open.
|
||||
5. **No raw observe escalation before snapshot/observe** — use `bsk snapshot` first; use `bsk observe` when VOM semantics or conditional surfaces help. Only use `bsk get-html` or `bsk screenshot` when snapshot/observe is insufficient. Element screenshots (`--ref @eN`) still require a fresh snapshot/observe ref — never skip observation just to grab a visual.
|
||||
6. **`evaluate` is powerful and risky** — use only when snapshot + click/fill/select cannot suffice; never on credential surfaces.
|
||||
|
||||
---
|
||||
|
||||
**More detail for any command:** `bsk <cmd> --help`
|
||||
The CLI's current help and error hints are authoritative for flags, parameters, and recovery
|
||||
details.
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
//! Atomic visibility boundary for files received into a same-directory temp.
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
pub fn commit(temp: &Path, out: &Path, overwrite: bool) -> std::io::Result<()> {
|
||||
if !overwrite {
|
||||
std::fs::hard_link(temp, out)?;
|
||||
std::fs::remove_file(temp)?;
|
||||
return Ok(());
|
||||
}
|
||||
replace(temp, out)
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn replace(temp: &Path, out: &Path) -> std::io::Result<()> {
|
||||
// POSIX rename replaces an existing non-directory destination atomically.
|
||||
std::fs::rename(temp, out)
|
||||
}
|
||||
|
||||
#[cfg(windows)]
|
||||
fn replace(temp: &Path, out: &Path) -> std::io::Result<()> {
|
||||
use std::os::windows::ffi::OsStrExt;
|
||||
use windows_sys::Win32::Storage::FileSystem::{
|
||||
MOVEFILE_REPLACE_EXISTING, MOVEFILE_WRITE_THROUGH, MoveFileExW,
|
||||
};
|
||||
|
||||
let from: Vec<u16> = temp.as_os_str().encode_wide().chain(Some(0)).collect();
|
||||
let to: Vec<u16> = out.as_os_str().encode_wide().chain(Some(0)).collect();
|
||||
// SAFETY: both buffers are NUL-terminated and remain alive for the call.
|
||||
let ok = unsafe {
|
||||
MoveFileExW(
|
||||
from.as_ptr(),
|
||||
to.as_ptr(),
|
||||
MOVEFILE_REPLACE_EXISTING | MOVEFILE_WRITE_THROUGH,
|
||||
)
|
||||
};
|
||||
if ok == 0 {
|
||||
Err(std::io::Error::last_os_error())
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
fn replace(temp: &Path, out: &Path) -> std::io::Result<()> {
|
||||
std::fs::rename(temp, out)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn no_replace_never_overwrites_existing_output() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let temp = dir.path().join("new.part");
|
||||
let out = dir.path().join("out.bin");
|
||||
std::fs::write(&temp, b"new").unwrap();
|
||||
std::fs::write(&out, b"old").unwrap();
|
||||
assert!(commit(&temp, &out, false).is_err());
|
||||
assert_eq!(std::fs::read(&out).unwrap(), b"old");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn overwrite_replaces_existing_output_without_predelete() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let temp = dir.path().join("new.part");
|
||||
let out = dir.path().join("out.bin");
|
||||
std::fs::write(&temp, b"new").unwrap();
|
||||
std::fs::write(&out, b"old").unwrap();
|
||||
commit(&temp, &out, true).unwrap();
|
||||
assert_eq!(std::fs::read(&out).unwrap(), b"new");
|
||||
assert!(!temp.exists());
|
||||
}
|
||||
}
|
||||
@@ -351,10 +351,10 @@ fn check_version_compatible(status: Option<&StatusResult>) -> CheckResult {
|
||||
}
|
||||
}
|
||||
|
||||
/// `bsk doctor` check: every connected browser should use the same
|
||||
/// protocol version as the daemon. Minor protocol drift is accepted by
|
||||
/// the daemon but flagged here so the user can see who needs updating
|
||||
/// (M10.4).
|
||||
/// `bsk doctor` check: connected browsers should speak a protocol the
|
||||
/// daemon accepts. A different protocol string is still a live
|
||||
/// connection — report Ok so agents keep working. The detail is an
|
||||
/// upgrade reminder, not a blocker.
|
||||
///
|
||||
/// Review M2 (round-1 minor): when no browsers are connected, the
|
||||
/// check has nothing to compare against, so it now reports
|
||||
@@ -396,13 +396,12 @@ fn check_browsers_protocol_compatible(status: Option<&StatusResult>) -> CheckRes
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(", ");
|
||||
CheckResult::fail(
|
||||
CheckResult::ok(
|
||||
name,
|
||||
format!(
|
||||
"{} browser(s) have protocol minor drift from the daemon: {stale}",
|
||||
"{} browser(s) report a different protocol version (still usable — continue, and upgrade soon): {stale}",
|
||||
status.version_skew_browsers.len()
|
||||
),
|
||||
"upgrade the browser-skill extension or bsk CLI so both sides use the same protocol version",
|
||||
)
|
||||
}
|
||||
|
||||
@@ -581,7 +580,7 @@ mod m2_tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn browsers_check_reports_fail_when_skew_present() {
|
||||
fn browsers_check_reports_ok_when_skew_present() {
|
||||
let status = fake_status(
|
||||
vec![BrowserStatusEntry {
|
||||
instance_id: "alpha".into(),
|
||||
@@ -605,9 +604,17 @@ mod m2_tests {
|
||||
}],
|
||||
);
|
||||
let check = check_browsers_protocol_compatible(Some(&status));
|
||||
assert_eq!(check.status, CheckStatus::Fail);
|
||||
assert_eq!(check.status, CheckStatus::Ok);
|
||||
assert!(check.detail.contains("alpha"));
|
||||
assert!(check.hint.is_some());
|
||||
assert!(
|
||||
check
|
||||
.detail
|
||||
.contains("still usable — continue, and upgrade soon")
|
||||
);
|
||||
assert!(
|
||||
check.hint.is_none(),
|
||||
"a protocol-version note must not hint to stop"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -635,7 +642,7 @@ mod m2_tests {
|
||||
}],
|
||||
);
|
||||
let check = check_browsers_protocol_compatible(Some(&status));
|
||||
assert_eq!(check.status, CheckStatus::Fail);
|
||||
assert_eq!(check.status, CheckStatus::Ok);
|
||||
assert!(
|
||||
check
|
||||
.detail
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
//! `bsk download` — capture one browser download and atomically commit it.
|
||||
|
||||
use std::fs::OpenOptions;
|
||||
use std::io::Write;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::Context;
|
||||
use base64::Engine;
|
||||
use bsk_protocol::Method;
|
||||
use bsk_protocol::tools::{
|
||||
DownloadParams, DownloadResult, TransferChunkParams, TransferChunkResult, TransferIdParams,
|
||||
TransferReleaseResult,
|
||||
};
|
||||
use clap::Args;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::cli::atomic_output;
|
||||
use crate::cli::ensure_daemon::ensure_daemon;
|
||||
use crate::cli::error::{CliError, Format};
|
||||
use crate::cli::interaction::split_target;
|
||||
use crate::cli::navigate::parse_timeout_ms;
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct DownloadArgs {
|
||||
/// Snapshot ref (`@e3`) or CSS selector for the download trigger.
|
||||
pub target: Option<String>,
|
||||
#[arg(long = "ref")]
|
||||
pub ref_: Option<String>,
|
||||
#[arg(long = "selector")]
|
||||
pub selector: Option<String>,
|
||||
#[arg(long)]
|
||||
pub out: PathBuf,
|
||||
#[arg(long)]
|
||||
pub session: String,
|
||||
#[arg(long = "tab-id")]
|
||||
pub tab_id: Option<i64>,
|
||||
#[arg(long, default_value = "2m", value_parser = parse_timeout_ms)]
|
||||
pub timeout: u32,
|
||||
#[arg(long)]
|
||||
pub overwrite: bool,
|
||||
}
|
||||
|
||||
pub fn dispatch(args: DownloadArgs, format: Format) -> Result<(), CliError> {
|
||||
if args.out.exists() && !args.overwrite {
|
||||
return Err(CliError::Local(anyhow::anyhow!(
|
||||
"output already exists (pass --overwrite to replace it): {}",
|
||||
args.out.display()
|
||||
)));
|
||||
}
|
||||
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||
let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?;
|
||||
let params = DownloadParams {
|
||||
session_id: args.session,
|
||||
ref_,
|
||||
selector,
|
||||
tab_id: args.tab_id,
|
||||
timeout_ms: Some(args.timeout),
|
||||
browser_relative_dir: None,
|
||||
max_byte_size: None,
|
||||
};
|
||||
let reply: DownloadResult = crate::cli::business_rpc::call(
|
||||
info.sock_path.clone(),
|
||||
"download",
|
||||
Method::ToolDownload,
|
||||
Some(params),
|
||||
ipc_timeout(args.timeout),
|
||||
)?;
|
||||
let transfer_id = reply.transfer_id.clone().ok_or_else(|| {
|
||||
CliError::Local(anyhow::anyhow!("daemon returned no download transfer id"))
|
||||
})?;
|
||||
let write_result = write_transfer(&info.sock_path, &transfer_id, &args.out, args.overwrite);
|
||||
let _: Result<TransferReleaseResult, CliError> = crate::cli::business_rpc::call(
|
||||
info.sock_path,
|
||||
"transfer-release",
|
||||
Method::TransferRelease,
|
||||
Some(TransferIdParams { transfer_id }),
|
||||
Duration::from_secs(5),
|
||||
);
|
||||
write_result?;
|
||||
match format {
|
||||
Format::Json => {
|
||||
let mut value = serde_json::to_value(&reply).unwrap();
|
||||
value["path"] = serde_json::json!(args.out.to_string_lossy());
|
||||
if let Some(obj) = value.as_object_mut() {
|
||||
obj.remove("transfer_id");
|
||||
obj.remove("browser_path");
|
||||
}
|
||||
println!("{}", serde_json::to_string_pretty(&value).unwrap());
|
||||
}
|
||||
Format::Human => println!("{}", args.out.display()),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_transfer(sock: &Path, id: &str, out: &Path, overwrite: bool) -> Result<(), CliError> {
|
||||
let parent = out
|
||||
.parent()
|
||||
.filter(|p| !p.as_os_str().is_empty())
|
||||
.unwrap_or(Path::new("."));
|
||||
let temp = parent.join(format!(".bsk-download-{}.part", Uuid::new_v4().simple()));
|
||||
let result = (|| {
|
||||
let mut file = OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(&temp)
|
||||
.with_context(|| format!("create temporary download {}", temp.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
let mut offset = 0u64;
|
||||
loop {
|
||||
let chunk: TransferChunkResult = crate::cli::business_rpc::call(
|
||||
sock.to_path_buf(),
|
||||
"transfer-read",
|
||||
Method::TransferRead,
|
||||
Some(TransferChunkParams {
|
||||
transfer_id: id.to_string(),
|
||||
offset,
|
||||
data_base64: String::new(),
|
||||
}),
|
||||
Duration::from_secs(30),
|
||||
)?;
|
||||
let encoded = chunk.data_base64.as_deref().unwrap_or("");
|
||||
let bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(encoded)
|
||||
.map_err(|e| CliError::Local(anyhow::anyhow!("decode download chunk: {e}")))?;
|
||||
file.write_all(&bytes)
|
||||
.map_err(|e| CliError::Local(e.into()))?;
|
||||
offset = chunk.next_offset;
|
||||
if chunk.eof {
|
||||
break;
|
||||
}
|
||||
}
|
||||
file.sync_all().map_err(|e| CliError::Local(e.into()))?;
|
||||
drop(file);
|
||||
atomic_output::commit(&temp, out, overwrite)
|
||||
.with_context(|| format!("atomically commit download to {}", out.display()))
|
||||
.map_err(CliError::Local)
|
||||
})();
|
||||
if result.is_err() {
|
||||
let _ = std::fs::remove_file(&temp);
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn ipc_timeout(timeout_ms: u32) -> Duration {
|
||||
Duration::from_millis(u64::from(timeout_ms) + 5_000)
|
||||
}
|
||||
@@ -71,7 +71,7 @@ pub(crate) fn parse_modifiers(input: &str) -> Result<Vec<KeyModifier>, String> {
|
||||
Ok(out)
|
||||
}
|
||||
|
||||
fn split_target(
|
||||
pub(crate) fn split_target(
|
||||
positional: Option<String>,
|
||||
explicit_ref: Option<String>,
|
||||
explicit_selector: Option<String>,
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
mod atomic_output;
|
||||
pub mod browser_wait;
|
||||
pub mod browsers;
|
||||
pub mod business_rpc;
|
||||
@@ -9,6 +10,7 @@ pub mod console;
|
||||
pub mod daemon;
|
||||
pub mod dialogs;
|
||||
pub mod doctor;
|
||||
pub mod download;
|
||||
pub mod emulate;
|
||||
pub mod ensure_daemon;
|
||||
pub mod error;
|
||||
@@ -31,6 +33,7 @@ pub mod snapshot;
|
||||
pub mod status;
|
||||
pub mod tab;
|
||||
pub mod update;
|
||||
pub mod upload;
|
||||
pub mod waits;
|
||||
pub mod window;
|
||||
|
||||
@@ -38,6 +41,7 @@ use clap::{Args, Parser, Subcommand};
|
||||
|
||||
use crate::cli::console::ConsoleArgs;
|
||||
use crate::cli::daemon::DaemonCmd;
|
||||
use crate::cli::download::DownloadArgs;
|
||||
use crate::cli::emulate::EmulateArgs;
|
||||
use crate::cli::evaluate::EvaluateArgs;
|
||||
use crate::cli::get_html::GetHtmlArgs;
|
||||
@@ -53,6 +57,7 @@ use crate::cli::session::SessionCmd;
|
||||
use crate::cli::snapshot::SnapshotArgs;
|
||||
use crate::cli::tab::TabCmd;
|
||||
use crate::cli::update::UpdateArgs;
|
||||
use crate::cli::upload::UploadArgs;
|
||||
use crate::cli::waits::{WaitForNavigationArgs, WaitMsArgs};
|
||||
use crate::cli::window::WindowCmd;
|
||||
|
||||
@@ -177,6 +182,12 @@ pub enum Command {
|
||||
/// Set `<select>` option values by `value` attribute.
|
||||
Select(SelectArgs),
|
||||
|
||||
/// Upload one or more local files through a page file chooser.
|
||||
Upload(UploadArgs),
|
||||
|
||||
/// Capture one browser download and write it to a local path.
|
||||
Download(DownloadArgs),
|
||||
|
||||
/// Evaluate a JavaScript expression inside the Agent Window.
|
||||
Evaluate(EvaluateArgs),
|
||||
|
||||
|
||||
@@ -33,6 +33,12 @@ pub struct ObserveArgs {
|
||||
/// Include conditional surface probe diagnostics in JSON output.
|
||||
#[arg(long = "debug-surfaces")]
|
||||
pub debug_surfaces: bool,
|
||||
|
||||
/// Actively hover page controls to reveal hover-only menus and tooltips.
|
||||
/// Costs seconds of wall clock and touches the live page, so it is off
|
||||
/// unless a static observation looks like it is missing hover content.
|
||||
#[arg(long = "probe-hover")]
|
||||
pub probe_hover: bool,
|
||||
}
|
||||
|
||||
pub fn dispatch(args: ObserveArgs, format: Format) -> Result<(), CliError> {
|
||||
@@ -47,6 +53,7 @@ fn run(sock: PathBuf, args: ObserveArgs, format: Format) -> Result<(), CliError>
|
||||
max_depth: args.max_depth,
|
||||
max_tokens: args.max_tokens,
|
||||
debug_surfaces: args.debug_surfaces,
|
||||
probe_hover: args.probe_hover,
|
||||
};
|
||||
let reply: ObserveResult = call(sock, params)?;
|
||||
match format {
|
||||
|
||||
@@ -49,6 +49,12 @@ pub mod reason {
|
||||
pub const TAB_NOT_ACTIVE: &str = "tab_not_active";
|
||||
pub const BORROW_CONFLICT: &str = "borrow_conflict";
|
||||
pub const SCREENSHOT_CAPTURE_FAILED: &str = "screenshot_capture_failed";
|
||||
pub const FILE_INPUT_PROBE_FAILED: &str = "file_input_probe_failed";
|
||||
pub const FILE_INPUT_NOT_ACTIVATED: &str = "file_input_not_activated";
|
||||
pub const SET_FILE_INPUT_FAILED: &str = "set_file_input_failed";
|
||||
pub const DOWNLOAD_CAPTURE_FAILED: &str = "download_capture_failed";
|
||||
pub const TRANSFER_OUTCOME_UNKNOWN: &str = "transfer_outcome_unknown";
|
||||
pub const TRANSFER_TIMEOUT: &str = "transfer_timeout";
|
||||
pub const SESSION_BUSY: &str = crate::rpc_reason::SESSION_BUSY;
|
||||
pub const RECORD_START_PAGE_UNREACHABLE: &str = "record_start_page_unreachable";
|
||||
}
|
||||
@@ -197,6 +203,20 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
||||
return base;
|
||||
};
|
||||
match (code, reason) {
|
||||
(_, reason::TRANSFER_OUTCOME_UNKNOWN) => RenderInfo {
|
||||
summary: "the file transfer outcome could not be confirmed",
|
||||
hint: Some(
|
||||
"do not retry the transfer: the browser may already have applied it; inspect the page or stop safely",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::Timeout, reason::TRANSFER_TIMEOUT) => RenderInfo {
|
||||
summary: "the file transfer timed out after browser dispatch",
|
||||
hint: Some(
|
||||
"do not retry when effect_state is unknown or committed; inspect the page before taking another action",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::PermissionDenied, reason::ELEMENT_NOT_VISIBLE) => RenderInfo {
|
||||
summary: "target element has no visible geometry",
|
||||
hint: Some(
|
||||
@@ -280,6 +300,36 @@ pub fn info_for_error(code: ErrorCode, data: Option<&serde_json::Value>) -> Rend
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::Unsupported, reason::FILE_INPUT_NOT_ACTIVATED) => RenderInfo {
|
||||
summary: "the upload trigger did not activate a file input",
|
||||
hint: Some(
|
||||
"the page may use a non-input picker such as showOpenFilePicker(); do not retry blindly — use `bsk request-help` with the exact original local path, or stop if human help is disabled",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::Timeout | ErrorCode::CdpFailed, reason::FILE_INPUT_PROBE_FAILED) => {
|
||||
RenderInfo {
|
||||
summary: "the browser upload transaction could not be established",
|
||||
hint: Some(
|
||||
"do not retry the same upload blindly; use `bsk request-help` when human help is available",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
}
|
||||
}
|
||||
(ErrorCode::Timeout | ErrorCode::CdpFailed, reason::SET_FILE_INPUT_FAILED) => RenderInfo {
|
||||
summary: "the browser could not attach the staged file to the input",
|
||||
hint: Some(
|
||||
"check that BrowserSkill has Chrome's 'Allow access to file URLs' permission; otherwise use `bsk request-help`",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
(ErrorCode::CdpFailed, reason::DOWNLOAD_CAPTURE_FAILED) => RenderInfo {
|
||||
summary: "the browser download could not be attributed or completed",
|
||||
hint: Some(
|
||||
"do not retry blindly; the target browser may not expose a reliable download intent signal",
|
||||
),
|
||||
exit_code: base.exit_code,
|
||||
},
|
||||
_ => base,
|
||||
}
|
||||
}
|
||||
@@ -419,6 +469,38 @@ mod tests {
|
||||
assert_eq!(info.exit_code, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn file_transfer_reasons_render_actionable_fallbacks() {
|
||||
let unsupported = serde_json::json!({ "reason": reason::FILE_INPUT_NOT_ACTIVATED });
|
||||
let info = info_for_error(ErrorCode::Unsupported, Some(&unsupported));
|
||||
assert_eq!(
|
||||
info.summary,
|
||||
"the upload trigger did not activate a file input"
|
||||
);
|
||||
assert!(info.hint.unwrap().contains("exact original local path"));
|
||||
|
||||
let control = serde_json::json!({ "reason": reason::FILE_INPUT_PROBE_FAILED });
|
||||
let info = info_for_error(ErrorCode::Timeout, Some(&control));
|
||||
assert!(
|
||||
info.summary
|
||||
.contains("transaction could not be established")
|
||||
);
|
||||
assert!(info.hint.unwrap().contains("do not retry"));
|
||||
|
||||
let download = serde_json::json!({ "reason": reason::DOWNLOAD_CAPTURE_FAILED });
|
||||
let info = info_for_error(ErrorCode::CdpFailed, Some(&download));
|
||||
assert!(info.summary.contains("download could not be attributed"));
|
||||
|
||||
let unknown = serde_json::json!({ "reason": reason::TRANSFER_OUTCOME_UNKNOWN });
|
||||
let info = info_for_error(ErrorCode::ProtocolError, Some(&unknown));
|
||||
assert!(info.summary.contains("outcome could not be confirmed"));
|
||||
assert!(info.hint.unwrap().contains("do not retry"));
|
||||
|
||||
let timeout = serde_json::json!({ "reason": reason::TRANSFER_TIMEOUT });
|
||||
let info = info_for_error(ErrorCode::Timeout, Some(&timeout));
|
||||
assert!(info.summary.contains("timed out after browser dispatch"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ref_not_found_overrides_not_found_copy() {
|
||||
let data = serde_json::json!({ "reason": reason::REF_NOT_FOUND });
|
||||
|
||||
@@ -65,7 +65,7 @@ fn render_human(s: &StatusResult) {
|
||||
let bold = "\x1b[1m";
|
||||
let reset = "\x1b[0m";
|
||||
eprintln!(
|
||||
"{bold}{yellow}warning:{reset} {} browser(s) have protocol version drift from the daemon{reset}",
|
||||
"{bold}{yellow}note:{reset} {} browser(s) report a different protocol version; still usable — continue, and run `bsk update` soon{reset}",
|
||||
s.version_skew_browsers.len()
|
||||
);
|
||||
for skew in &s.version_skew_browsers {
|
||||
@@ -75,7 +75,7 @@ fn render_human(s: &StatusResult) {
|
||||
skew.label.clone()
|
||||
};
|
||||
eprintln!(
|
||||
" {} ({}) — protocol ext {} vs daemon {} (app ext v{}, daemon v{}) — please align protocol versions",
|
||||
" {} ({}) — protocol ext {} vs CLI {} (app ext v{}, CLI v{}) — still usable; please upgrade",
|
||||
skew.instance_id,
|
||||
label,
|
||||
display_protocol(&skew.client_protocol_version),
|
||||
|
||||
@@ -0,0 +1,185 @@
|
||||
//! `bsk upload` — stage caller-readable files and attach them to a page.
|
||||
|
||||
use std::fs::File;
|
||||
use std::io::Read;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::Context;
|
||||
use base64::Engine;
|
||||
use bsk_protocol::Method;
|
||||
use bsk_protocol::tools::{
|
||||
TransferBeginParams, TransferBeginResult, TransferChunkParams, TransferChunkResult,
|
||||
TransferIdParams, TransferReadyResult, TransferReleaseResult, UploadFile, UploadParams,
|
||||
UploadResult,
|
||||
};
|
||||
use clap::Args;
|
||||
|
||||
use crate::cli::ensure_daemon::ensure_daemon;
|
||||
use crate::cli::error::{CliError, Format};
|
||||
use crate::cli::interaction::split_target;
|
||||
use crate::cli::navigate::parse_timeout_ms;
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct UploadArgs {
|
||||
/// Snapshot ref (`@e3`) or CSS selector for the file input / chooser trigger.
|
||||
pub target: Option<String>,
|
||||
#[arg(long = "ref")]
|
||||
pub ref_: Option<String>,
|
||||
#[arg(long = "selector")]
|
||||
pub selector: Option<String>,
|
||||
/// Local file to upload. Repeat for a multiple-file input.
|
||||
#[arg(long = "file", required = true)]
|
||||
pub files: Vec<PathBuf>,
|
||||
#[arg(long)]
|
||||
pub session: String,
|
||||
#[arg(long = "tab-id")]
|
||||
pub tab_id: Option<i64>,
|
||||
#[arg(long, default_value = "2m", value_parser = parse_timeout_ms)]
|
||||
pub timeout: u32,
|
||||
}
|
||||
|
||||
pub fn dispatch(args: UploadArgs, format: Format) -> Result<(), CliError> {
|
||||
if args.files.len() > crate::daemon::file_transfer::MAX_UPLOAD_FILES {
|
||||
return Err(CliError::Local(anyhow::anyhow!(
|
||||
"upload accepts at most {} files",
|
||||
crate::daemon::file_transfer::MAX_UPLOAD_FILES
|
||||
)));
|
||||
}
|
||||
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||
let (ref_, selector) = split_target(args.target, args.ref_, args.selector)?;
|
||||
let mut staged = Vec::new();
|
||||
for path in &args.files {
|
||||
match stage_file(&info.sock_path, &args.session, path) {
|
||||
Ok(file) => staged.push(file),
|
||||
Err(err) => {
|
||||
for (id, _) in &staged {
|
||||
let _ = release(&info.sock_path, id);
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
let params = UploadParams {
|
||||
session_id: args.session,
|
||||
ref_,
|
||||
selector,
|
||||
tab_id: args.tab_id,
|
||||
files: staged
|
||||
.iter()
|
||||
.map(|(id, name)| UploadFile {
|
||||
transfer_id: id.clone(),
|
||||
name: name.clone(),
|
||||
staged_path: None,
|
||||
})
|
||||
.collect(),
|
||||
timeout_ms: Some(args.timeout),
|
||||
};
|
||||
let result = crate::cli::business_rpc::call::<_, UploadResult>(
|
||||
info.sock_path.clone(),
|
||||
"upload",
|
||||
Method::ToolUpload,
|
||||
Some(params),
|
||||
ipc_timeout(args.timeout),
|
||||
);
|
||||
// Once tool.upload is dispatched, staging ownership belongs to the
|
||||
// session. A transport timeout cannot prove that Chrome did not attach
|
||||
// the file, so releasing here could invalidate a late successful attach.
|
||||
// Session teardown remains the single cleanup boundary after dispatch.
|
||||
let reply = result?;
|
||||
match format {
|
||||
Format::Json => println!("{}", serde_json::to_string_pretty(&reply).unwrap()),
|
||||
Format::Human => println!(
|
||||
"upload ok tab={} files={}",
|
||||
reply.tab_id,
|
||||
reply.file_names.join(", ")
|
||||
),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn stage_file(sock: &Path, session: &str, path: &PathBuf) -> Result<(String, String), CliError> {
|
||||
let mut file = File::open(path)
|
||||
.with_context(|| format!("open upload file {}", path.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
let meta = file
|
||||
.metadata()
|
||||
.with_context(|| format!("read upload file metadata {}", path.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
if !meta.is_file() {
|
||||
return Err(CliError::Local(anyhow::anyhow!(
|
||||
"upload source is not a regular file: {}",
|
||||
path.display()
|
||||
)));
|
||||
}
|
||||
let name = path
|
||||
.file_name()
|
||||
.and_then(|v| v.to_str())
|
||||
.filter(|v| !v.is_empty())
|
||||
.ok_or_else(|| CliError::Local(anyhow::anyhow!("upload file has no valid basename")))?
|
||||
.to_string();
|
||||
let begin: TransferBeginResult = crate::cli::business_rpc::call(
|
||||
sock.to_path_buf(),
|
||||
"transfer-begin",
|
||||
Method::TransferBegin,
|
||||
Some(TransferBeginParams {
|
||||
session_id: session.to_string(),
|
||||
name: name.clone(),
|
||||
byte_size: meta.len(),
|
||||
}),
|
||||
Duration::from_secs(10),
|
||||
)?;
|
||||
let staged = (|| {
|
||||
let mut offset = 0u64;
|
||||
let mut buf = vec![0u8; begin.chunk_size as usize];
|
||||
loop {
|
||||
let n = file.read(&mut buf).map_err(|e| CliError::Local(e.into()))?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
let reply: TransferChunkResult = crate::cli::business_rpc::call(
|
||||
sock.to_path_buf(),
|
||||
"transfer-chunk",
|
||||
Method::TransferChunk,
|
||||
Some(TransferChunkParams {
|
||||
transfer_id: begin.transfer_id.clone(),
|
||||
offset,
|
||||
data_base64: base64::engine::general_purpose::STANDARD.encode(&buf[..n]),
|
||||
}),
|
||||
Duration::from_secs(30),
|
||||
)?;
|
||||
offset = reply.next_offset;
|
||||
}
|
||||
let _: TransferReadyResult = crate::cli::business_rpc::call(
|
||||
sock.to_path_buf(),
|
||||
"transfer-finish",
|
||||
Method::TransferFinish,
|
||||
Some(TransferIdParams {
|
||||
transfer_id: begin.transfer_id.clone(),
|
||||
}),
|
||||
Duration::from_secs(10),
|
||||
)?;
|
||||
Ok::<_, CliError>(())
|
||||
})();
|
||||
if let Err(err) = staged {
|
||||
let _ = release(sock, &begin.transfer_id);
|
||||
return Err(err);
|
||||
}
|
||||
Ok((begin.transfer_id, name))
|
||||
}
|
||||
|
||||
fn release(sock: &Path, id: &str) -> Result<TransferReleaseResult, CliError> {
|
||||
crate::cli::business_rpc::call(
|
||||
sock.to_path_buf(),
|
||||
"transfer-release",
|
||||
Method::TransferRelease,
|
||||
Some(TransferIdParams {
|
||||
transfer_id: id.to_string(),
|
||||
}),
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
}
|
||||
|
||||
fn ipc_timeout(timeout_ms: u32) -> Duration {
|
||||
Duration::from_millis(u64::from(timeout_ms) + 5_000)
|
||||
}
|
||||
@@ -0,0 +1,728 @@
|
||||
//! Session-scoped, content-transparent staging for upload/download tools.
|
||||
//!
|
||||
//! The registry deliberately exposes no arbitrary-path operations. Upload
|
||||
//! bytes arrive from the invoking CLI in bounded chunks; browser downloads
|
||||
//! land in a daemon-minted directory and are read back by the invoking CLI.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::fs::{self, File, OpenOptions};
|
||||
use std::io::{Read, Seek, SeekFrom, Write};
|
||||
use std::path::{Component, Path, PathBuf};
|
||||
use std::sync::Mutex;
|
||||
|
||||
use base64::Engine;
|
||||
use bsk_protocol::tools::{
|
||||
TransferBeginParams, TransferBeginResult, TransferChunkParams, TransferChunkResult,
|
||||
TransferIdParams, TransferReadyResult, TransferReleaseResult,
|
||||
};
|
||||
use bsk_protocol::{ErrorCode, RpcError};
|
||||
use uuid::Uuid;
|
||||
|
||||
use super::paths;
|
||||
|
||||
pub const TRANSFER_CHUNK_SIZE: u32 = 512 * 1024;
|
||||
pub const MAX_TRANSFER_BYTES: u64 = 512 * 1024 * 1024;
|
||||
pub const MAX_UPLOAD_FILES: usize = 20;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum Direction {
|
||||
Upload,
|
||||
Download,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Entry {
|
||||
session_id: String,
|
||||
direction: Direction,
|
||||
path: PathBuf,
|
||||
expected_size: Option<u64>,
|
||||
written: u64,
|
||||
ready: bool,
|
||||
}
|
||||
|
||||
/// Browser download path after the daemon has verified that it belongs to
|
||||
/// the capability minted for this transfer. From this point onward the daemon
|
||||
/// owns cleanup on every error path; unvalidated arbitrary paths are never
|
||||
/// removed.
|
||||
#[derive(Debug)]
|
||||
struct ValidatedBrowserFile {
|
||||
path: PathBuf,
|
||||
parent: PathBuf,
|
||||
cleanup_armed: bool,
|
||||
}
|
||||
|
||||
impl ValidatedBrowserFile {
|
||||
fn validate(reported_path: &Path, transfer_id: &str) -> Result<Self, RpcError> {
|
||||
if !reported_path.is_absolute() {
|
||||
return Err(permission(
|
||||
"download path is outside its browser capability",
|
||||
));
|
||||
}
|
||||
let expected_parent = Path::new("BrowserSkill").join(transfer_id);
|
||||
let parent = reported_path
|
||||
.parent()
|
||||
.ok_or_else(|| permission("download path has no parent directory"))?;
|
||||
if !parent.ends_with(&expected_parent) {
|
||||
return Err(permission(
|
||||
"download escaped its browser capability directory",
|
||||
));
|
||||
}
|
||||
for path in [reported_path, parent] {
|
||||
if fs::symlink_metadata(path)
|
||||
.map_err(io_error)?
|
||||
.file_type()
|
||||
.is_symlink()
|
||||
{
|
||||
return Err(permission("download capability path contains a symlink"));
|
||||
}
|
||||
}
|
||||
Ok(Self {
|
||||
path: reported_path.to_path_buf(),
|
||||
parent: parent.to_path_buf(),
|
||||
cleanup_armed: true,
|
||||
})
|
||||
}
|
||||
|
||||
fn remove_source(mut self) -> Result<(), RpcError> {
|
||||
remove_if_present(&self.path).map_err(io_error)?;
|
||||
remove_dir_if_present(&self.parent).map_err(io_error)?;
|
||||
self.cleanup_armed = false;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ValidatedBrowserFile {
|
||||
fn drop(&mut self) {
|
||||
if !self.cleanup_armed {
|
||||
return;
|
||||
}
|
||||
let _ = remove_if_present(&self.path);
|
||||
let _ = remove_dir_if_present(&self.parent);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct DownloadStaging {
|
||||
pub transfer_id: String,
|
||||
pub browser_relative_dir: String,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct TransferRegistry {
|
||||
root: PathBuf,
|
||||
initialized: Mutex<bool>,
|
||||
entries: Mutex<HashMap<String, Entry>>,
|
||||
}
|
||||
|
||||
impl TransferRegistry {
|
||||
pub fn new() -> anyhow::Result<Self> {
|
||||
Ok(Self {
|
||||
root: paths::bsk_home()?.join("run").join("transfers"),
|
||||
initialized: Mutex::new(false),
|
||||
entries: Mutex::new(HashMap::new()),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn at_root(root: PathBuf) -> anyhow::Result<Self> {
|
||||
let registry = Self {
|
||||
root,
|
||||
initialized: Mutex::new(false),
|
||||
entries: Mutex::new(HashMap::new()),
|
||||
};
|
||||
registry.ensure_root()?;
|
||||
Ok(registry)
|
||||
}
|
||||
|
||||
pub fn initialize(&self) -> anyhow::Result<()> {
|
||||
self.ensure_root()
|
||||
}
|
||||
|
||||
fn ensure_root(&self) -> anyhow::Result<()> {
|
||||
let mut initialized = self.initialized.lock().unwrap();
|
||||
if *initialized {
|
||||
return Ok(());
|
||||
}
|
||||
if self.root.exists() {
|
||||
// Transfers never survive a daemon; removing stale directories on
|
||||
// startup gives crash cleanup without a second persistence model.
|
||||
let _ = fs::remove_dir_all(&self.root);
|
||||
}
|
||||
fs::create_dir_all(&self.root)?;
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
fs::set_permissions(&self.root, fs::Permissions::from_mode(0o700))?;
|
||||
}
|
||||
*initialized = true;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn begin_upload(&self, p: TransferBeginParams) -> Result<TransferBeginResult, RpcError> {
|
||||
self.ensure_root()
|
||||
.map_err(|err| protocol_error(err.to_string()))?;
|
||||
let basename = safe_upload_basename(&p.name)?;
|
||||
if p.byte_size > MAX_TRANSFER_BYTES {
|
||||
return Err(invalid(format!(
|
||||
"file size {} exceeds transfer limit {}",
|
||||
p.byte_size, MAX_TRANSFER_BYTES
|
||||
)));
|
||||
}
|
||||
let mut entries = self.entries.lock().unwrap();
|
||||
let session_bytes: u64 = entries
|
||||
.values()
|
||||
.filter(|entry| entry.session_id == p.session_id)
|
||||
.map(|entry| entry.expected_size.unwrap_or(entry.written))
|
||||
.sum();
|
||||
if session_bytes.saturating_add(p.byte_size) > MAX_TRANSFER_BYTES {
|
||||
return Err(invalid(format!(
|
||||
"session transfer staging exceeds limit {}",
|
||||
MAX_TRANSFER_BYTES
|
||||
)));
|
||||
}
|
||||
let id = new_id();
|
||||
let dir = self.root.join(&id);
|
||||
fs::create_dir(&dir).map_err(io_error)?;
|
||||
set_private_dir(&dir).map_err(io_error)?;
|
||||
let path = dir.join(basename);
|
||||
let file = OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(&path)
|
||||
.map_err(io_error)?;
|
||||
set_private_file(&file).map_err(io_error)?;
|
||||
entries.insert(
|
||||
id.clone(),
|
||||
Entry {
|
||||
session_id: p.session_id,
|
||||
direction: Direction::Upload,
|
||||
path,
|
||||
expected_size: Some(p.byte_size),
|
||||
written: 0,
|
||||
ready: false,
|
||||
},
|
||||
);
|
||||
Ok(TransferBeginResult {
|
||||
transfer_id: id,
|
||||
chunk_size: TRANSFER_CHUNK_SIZE,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn write_chunk(&self, p: TransferChunkParams) -> Result<TransferChunkResult, RpcError> {
|
||||
let bytes = base64::engine::general_purpose::STANDARD
|
||||
.decode(p.data_base64)
|
||||
.map_err(|e| invalid(format!("invalid transfer chunk: {e}")))?;
|
||||
if bytes.len() > TRANSFER_CHUNK_SIZE as usize {
|
||||
return Err(invalid("transfer chunk exceeds negotiated size"));
|
||||
}
|
||||
let mut entries = self.entries.lock().unwrap();
|
||||
let entry = entries
|
||||
.get_mut(&p.transfer_id)
|
||||
.ok_or_else(|| not_found("transfer not found"))?;
|
||||
if entry.direction != Direction::Upload || entry.ready {
|
||||
return Err(invalid("transfer is not writable"));
|
||||
}
|
||||
if p.offset != entry.written {
|
||||
return Err(invalid(format!(
|
||||
"non-sequential transfer offset: expected {}, got {}",
|
||||
entry.written, p.offset
|
||||
)));
|
||||
}
|
||||
let next = entry.written.saturating_add(bytes.len() as u64);
|
||||
if next > entry.expected_size.unwrap_or(MAX_TRANSFER_BYTES) {
|
||||
return Err(invalid("transfer exceeds declared byte size"));
|
||||
}
|
||||
let mut file = OpenOptions::new()
|
||||
.append(true)
|
||||
.open(&entry.path)
|
||||
.map_err(io_error)?;
|
||||
file.write_all(&bytes).map_err(io_error)?;
|
||||
entry.written = next;
|
||||
Ok(TransferChunkResult {
|
||||
next_offset: next,
|
||||
eof: false,
|
||||
data_base64: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn finish_upload(&self, p: TransferIdParams) -> Result<TransferReadyResult, RpcError> {
|
||||
let mut entries = self.entries.lock().unwrap();
|
||||
let entry = entries
|
||||
.get_mut(&p.transfer_id)
|
||||
.ok_or_else(|| not_found("transfer not found"))?;
|
||||
if entry.direction != Direction::Upload {
|
||||
return Err(invalid("transfer is not an upload"));
|
||||
}
|
||||
if entry.written != entry.expected_size.unwrap_or(entry.written) {
|
||||
return Err(invalid(format!(
|
||||
"incomplete transfer: expected {} bytes, received {}",
|
||||
entry.expected_size.unwrap_or(0),
|
||||
entry.written
|
||||
)));
|
||||
}
|
||||
entry.ready = true;
|
||||
Ok(TransferReadyResult {
|
||||
transfer_id: p.transfer_id,
|
||||
byte_size: entry.written,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn resolve_uploads(
|
||||
&self,
|
||||
session_id: &str,
|
||||
ids: &[String],
|
||||
) -> Result<Vec<PathBuf>, RpcError> {
|
||||
let entries = self.entries.lock().unwrap();
|
||||
ids.iter()
|
||||
.map(|id| {
|
||||
let entry = entries
|
||||
.get(id)
|
||||
.ok_or_else(|| not_found("upload transfer not found"))?;
|
||||
if entry.session_id != session_id
|
||||
|| entry.direction != Direction::Upload
|
||||
|| !entry.ready
|
||||
{
|
||||
return Err(permission(
|
||||
"upload transfer is outside this session or not ready",
|
||||
));
|
||||
}
|
||||
Ok(entry.path.clone())
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn begin_download(&self, session_id: &str) -> Result<DownloadStaging, RpcError> {
|
||||
self.ensure_root()
|
||||
.map_err(|err| protocol_error(err.to_string()))?;
|
||||
let id = new_id();
|
||||
let dir = self.root.join(&id);
|
||||
fs::create_dir(&dir).map_err(io_error)?;
|
||||
set_private_dir(&dir).map_err(io_error)?;
|
||||
self.entries.lock().unwrap().insert(
|
||||
id.clone(),
|
||||
Entry {
|
||||
session_id: session_id.to_string(),
|
||||
direction: Direction::Download,
|
||||
path: dir.clone(),
|
||||
expected_size: None,
|
||||
written: 0,
|
||||
ready: false,
|
||||
},
|
||||
);
|
||||
Ok(DownloadStaging {
|
||||
browser_relative_dir: format!("BrowserSkill/{id}"),
|
||||
transfer_id: id,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn import_download(&self, id: &str, reported_path: &Path) -> Result<u64, RpcError> {
|
||||
let mut entries = self.entries.lock().unwrap();
|
||||
let entry = entries
|
||||
.get_mut(id)
|
||||
.ok_or_else(|| not_found("download transfer not found"))?;
|
||||
if entry.direction != Direction::Download {
|
||||
return Err(permission("transfer is not a download capability"));
|
||||
}
|
||||
let browser_file = ValidatedBrowserFile::validate(reported_path, id)?;
|
||||
let canonical = browser_file.path.canonicalize().map_err(io_error)?;
|
||||
let meta = fs::metadata(&canonical).map_err(io_error)?;
|
||||
if !meta.is_file() || meta.len() > MAX_TRANSFER_BYTES {
|
||||
return Err(invalid(
|
||||
"download is not a regular file or exceeds the transfer limit",
|
||||
));
|
||||
}
|
||||
|
||||
let destination = entry.path.join("payload");
|
||||
let source = File::open(&canonical).map_err(io_error)?;
|
||||
let mut destination_file = OpenOptions::new()
|
||||
.write(true)
|
||||
.create_new(true)
|
||||
.open(&destination)
|
||||
.map_err(io_error)?;
|
||||
let copied = std::io::copy(
|
||||
&mut source.take(MAX_TRANSFER_BYTES.saturating_add(1)),
|
||||
&mut destination_file,
|
||||
)
|
||||
.map_err(io_error)?;
|
||||
if copied > MAX_TRANSFER_BYTES {
|
||||
let _ = fs::remove_file(&destination);
|
||||
return Err(invalid("download exceeds the transfer limit"));
|
||||
}
|
||||
entry.path = destination;
|
||||
entry.written = copied;
|
||||
entry.expected_size = Some(copied);
|
||||
entry.ready = true;
|
||||
browser_file.remove_source()?;
|
||||
Ok(copied)
|
||||
}
|
||||
|
||||
pub fn read_chunk(&self, p: TransferChunkParams) -> Result<TransferChunkResult, RpcError> {
|
||||
let entries = self.entries.lock().unwrap();
|
||||
let entry = entries
|
||||
.get(&p.transfer_id)
|
||||
.ok_or_else(|| not_found("transfer not found"))?;
|
||||
if entry.direction != Direction::Download || !entry.ready {
|
||||
return Err(invalid("transfer is not readable"));
|
||||
}
|
||||
if p.offset > entry.written {
|
||||
return Err(invalid("read offset is beyond transfer length"));
|
||||
}
|
||||
let mut file = File::open(&entry.path).map_err(io_error)?;
|
||||
file.seek(SeekFrom::Start(p.offset)).map_err(io_error)?;
|
||||
let mut buf = vec![0u8; TRANSFER_CHUNK_SIZE as usize];
|
||||
let count = file.read(&mut buf).map_err(io_error)?;
|
||||
buf.truncate(count);
|
||||
let next = p.offset + count as u64;
|
||||
Ok(TransferChunkResult {
|
||||
next_offset: next,
|
||||
eof: next >= entry.written,
|
||||
data_base64: Some(base64::engine::general_purpose::STANDARD.encode(buf)),
|
||||
})
|
||||
}
|
||||
|
||||
pub fn release(&self, p: TransferIdParams) -> TransferReleaseResult {
|
||||
let entry = self.entries.lock().unwrap().remove(&p.transfer_id);
|
||||
if let Some(entry) = entry {
|
||||
let dir = if entry.path.is_dir() {
|
||||
entry.path
|
||||
} else {
|
||||
entry.path.parent().unwrap_or(&self.root).to_path_buf()
|
||||
};
|
||||
let _ = fs::remove_dir_all(dir);
|
||||
TransferReleaseResult { released: true }
|
||||
} else {
|
||||
TransferReleaseResult { released: false }
|
||||
}
|
||||
}
|
||||
|
||||
pub fn release_session(&self, session_id: &str) {
|
||||
let ids: Vec<String> = self
|
||||
.entries
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.filter(|(_, entry)| entry.session_id == session_id)
|
||||
.map(|(id, _)| id.clone())
|
||||
.collect();
|
||||
for id in ids {
|
||||
self.release(TransferIdParams { transfer_id: id });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn new_id() -> String {
|
||||
format!("tr_{}", Uuid::new_v4().simple())
|
||||
}
|
||||
|
||||
fn set_private_dir(path: &Path) -> std::io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
fs::set_permissions(path, fs::Permissions::from_mode(0o700))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn set_private_file(file: &File) -> std::io::Result<()> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
file.set_permissions(fs::Permissions::from_mode(0o600))?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn remove_if_present(path: &Path) -> std::io::Result<()> {
|
||||
match fs::remove_file(path) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_dir_if_present(path: &Path) -> std::io::Result<()> {
|
||||
match fs::remove_dir(path) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(()),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
fn safe_upload_basename(name: &str) -> Result<&str, RpcError> {
|
||||
if name.is_empty() || name.contains(['/', '\\', '\0']) {
|
||||
return Err(invalid("upload name must be a safe basename"));
|
||||
}
|
||||
let mut components = Path::new(name).components();
|
||||
match (components.next(), components.next()) {
|
||||
(Some(Component::Normal(_)), None) => Ok(name),
|
||||
_ => Err(invalid("upload name must be a safe basename")),
|
||||
}
|
||||
}
|
||||
|
||||
fn invalid(message: impl Into<String>) -> RpcError {
|
||||
RpcError {
|
||||
code: ErrorCode::InvalidParams,
|
||||
message: message.into(),
|
||||
data: None,
|
||||
}
|
||||
}
|
||||
fn not_found(message: impl Into<String>) -> RpcError {
|
||||
RpcError {
|
||||
code: ErrorCode::NotFound,
|
||||
message: message.into(),
|
||||
data: None,
|
||||
}
|
||||
}
|
||||
fn permission(message: impl Into<String>) -> RpcError {
|
||||
RpcError {
|
||||
code: ErrorCode::PermissionDenied,
|
||||
message: message.into(),
|
||||
data: None,
|
||||
}
|
||||
}
|
||||
fn io_error(err: std::io::Error) -> RpcError {
|
||||
RpcError {
|
||||
code: ErrorCode::ProtocolError,
|
||||
message: err.to_string(),
|
||||
data: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn protocol_error(message: impl Into<String>) -> RpcError {
|
||||
RpcError {
|
||||
code: ErrorCode::ProtocolError,
|
||||
message: message.into(),
|
||||
data: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn registry() -> (tempfile::TempDir, TransferRegistry) {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let registry = TransferRegistry::at_root(temp.path().join("transfers")).unwrap();
|
||||
(temp, registry)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upload_is_not_resolvable_until_complete_or_from_another_session() {
|
||||
let (_temp, registry) = registry();
|
||||
let begin = registry
|
||||
.begin_upload(TransferBeginParams {
|
||||
session_id: "s1".into(),
|
||||
name: "image.png".into(),
|
||||
byte_size: 3,
|
||||
})
|
||||
.unwrap();
|
||||
assert!(
|
||||
registry
|
||||
.resolve_uploads("s1", std::slice::from_ref(&begin.transfer_id))
|
||||
.is_err()
|
||||
);
|
||||
registry
|
||||
.write_chunk(TransferChunkParams {
|
||||
transfer_id: begin.transfer_id.clone(),
|
||||
offset: 0,
|
||||
data_base64: base64::engine::general_purpose::STANDARD.encode(b"abc"),
|
||||
})
|
||||
.unwrap();
|
||||
registry
|
||||
.finish_upload(TransferIdParams {
|
||||
transfer_id: begin.transfer_id.clone(),
|
||||
})
|
||||
.unwrap();
|
||||
assert!(
|
||||
registry
|
||||
.resolve_uploads("s2", std::slice::from_ref(&begin.transfer_id))
|
||||
.is_err()
|
||||
);
|
||||
let [path] = registry
|
||||
.resolve_uploads("s1", &[begin.transfer_id])
|
||||
.unwrap()
|
||||
.try_into()
|
||||
.unwrap();
|
||||
assert_eq!(path.file_name().unwrap(), "image.png");
|
||||
assert_eq!(fs::read(path).unwrap(), b"abc");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upload_name_must_be_one_safe_path_component() {
|
||||
let (_temp, registry) = registry();
|
||||
for name in ["", ".", "..", "../secret", "folder/file", "folder\\file"] {
|
||||
let result = registry.begin_upload(TransferBeginParams {
|
||||
session_id: "s1".into(),
|
||||
name: name.into(),
|
||||
byte_size: 0,
|
||||
});
|
||||
assert!(result.is_err(), "accepted unsafe upload name");
|
||||
}
|
||||
assert!(registry.entries.lock().unwrap().is_empty());
|
||||
assert!(fs::read_dir(®istry.root).unwrap().next().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn same_upload_names_are_isolated_by_transfer_directory() {
|
||||
let (_temp, registry) = registry();
|
||||
let first = registry
|
||||
.begin_upload(TransferBeginParams {
|
||||
session_id: "s1".into(),
|
||||
name: "image.png".into(),
|
||||
byte_size: 0,
|
||||
})
|
||||
.unwrap();
|
||||
let second = registry
|
||||
.begin_upload(TransferBeginParams {
|
||||
session_id: "s1".into(),
|
||||
name: "image.png".into(),
|
||||
byte_size: 0,
|
||||
})
|
||||
.unwrap();
|
||||
let entries = registry.entries.lock().unwrap();
|
||||
let first_path = &entries[&first.transfer_id].path;
|
||||
let second_path = &entries[&second.transfer_id].path;
|
||||
assert_ne!(first_path.parent(), second_path.parent());
|
||||
assert_eq!(first_path.file_name().unwrap(), "image.png");
|
||||
assert_eq!(second_path.file_name().unwrap(), "image.png");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn upload_staging_is_private() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
|
||||
let (_temp, registry) = registry();
|
||||
let begin = registry
|
||||
.begin_upload(TransferBeginParams {
|
||||
session_id: "s1".into(),
|
||||
name: "private.png".into(),
|
||||
byte_size: 0,
|
||||
})
|
||||
.unwrap();
|
||||
let entries = registry.entries.lock().unwrap();
|
||||
let path = &entries[&begin.transfer_id].path;
|
||||
assert_eq!(
|
||||
fs::metadata(path.parent().unwrap())
|
||||
.unwrap()
|
||||
.permissions()
|
||||
.mode()
|
||||
& 0o777,
|
||||
0o700
|
||||
);
|
||||
assert_eq!(
|
||||
fs::metadata(path).unwrap().permissions().mode() & 0o777,
|
||||
0o600
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn download_must_import_from_its_browser_capability_directory() {
|
||||
let (temp, registry) = registry();
|
||||
let staging = registry.begin_download("s1").unwrap();
|
||||
let outside = temp.path().join("outside.bin");
|
||||
fs::write(&outside, b"secret").unwrap();
|
||||
assert!(
|
||||
registry
|
||||
.import_download(&staging.transfer_id, &outside)
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(fs::read(&outside).unwrap(), b"secret");
|
||||
|
||||
let browser_dir = temp
|
||||
.path()
|
||||
.join("Downloads")
|
||||
.join("BrowserSkill")
|
||||
.join(&staging.transfer_id);
|
||||
fs::create_dir_all(&browser_dir).unwrap();
|
||||
let inside = browser_dir.join("result.bin");
|
||||
fs::write(&inside, b"result").unwrap();
|
||||
assert_eq!(
|
||||
registry
|
||||
.import_download(&staging.transfer_id, &inside)
|
||||
.unwrap(),
|
||||
6
|
||||
);
|
||||
assert!(!inside.exists());
|
||||
let chunk = registry
|
||||
.read_chunk(TransferChunkParams {
|
||||
transfer_id: staging.transfer_id,
|
||||
offset: 0,
|
||||
data_base64: String::new(),
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
base64::engine::general_purpose::STANDARD
|
||||
.decode(chunk.data_base64.unwrap())
|
||||
.unwrap(),
|
||||
b"result"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validated_oversized_browser_download_is_removed_on_rejection() {
|
||||
let (temp, registry) = registry();
|
||||
let staging = registry.begin_download("s1").unwrap();
|
||||
let browser_dir = temp
|
||||
.path()
|
||||
.join("Downloads")
|
||||
.join("BrowserSkill")
|
||||
.join(&staging.transfer_id);
|
||||
fs::create_dir_all(&browser_dir).unwrap();
|
||||
let inside = browser_dir.join("oversized.bin");
|
||||
File::create(&inside)
|
||||
.unwrap()
|
||||
.set_len(MAX_TRANSFER_BYTES + 1)
|
||||
.unwrap();
|
||||
|
||||
assert!(
|
||||
registry
|
||||
.import_download(&staging.transfer_id, &inside)
|
||||
.is_err()
|
||||
);
|
||||
assert!(!inside.exists());
|
||||
assert!(!browser_dir.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn releasing_a_session_removes_all_staging() {
|
||||
let (_temp, registry) = registry();
|
||||
let first = registry.begin_download("s1").unwrap();
|
||||
let second = registry.begin_download("s2").unwrap();
|
||||
let first_directory = registry.root.join(&first.transfer_id);
|
||||
let second_directory = registry.root.join(&second.transfer_id);
|
||||
registry.release_session("s1");
|
||||
assert!(!first_directory.exists());
|
||||
assert!(second_directory.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn upload_staging_is_bounded_across_a_session() {
|
||||
let (_temp, registry) = registry();
|
||||
registry
|
||||
.begin_upload(TransferBeginParams {
|
||||
session_id: "s1".into(),
|
||||
name: "large.bin".into(),
|
||||
byte_size: MAX_TRANSFER_BYTES,
|
||||
})
|
||||
.unwrap();
|
||||
assert!(
|
||||
registry
|
||||
.begin_upload(TransferBeginParams {
|
||||
session_id: "s1".into(),
|
||||
name: "one-more-byte.bin".into(),
|
||||
byte_size: 1,
|
||||
})
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
registry
|
||||
.begin_upload(TransferBeginParams {
|
||||
session_id: "s2".into(),
|
||||
name: "other-session.bin".into(),
|
||||
byte_size: 1,
|
||||
})
|
||||
.is_ok()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -27,7 +27,8 @@ use bsk_protocol::system::{
|
||||
VersionSkewEntry,
|
||||
};
|
||||
use bsk_protocol::tools::{
|
||||
HelpOutcome, RequestHelpResult, ReturnFailure, WaitMsParams, WaitMsResult,
|
||||
DownloadParams, DownloadResult, HelpOutcome, RequestHelpResult, ReturnFailure,
|
||||
TransferBeginParams, TransferIdParams, UploadParams, WaitMsParams, WaitMsResult,
|
||||
};
|
||||
use bsk_protocol::{
|
||||
CancelParams, CancelResult, ErrorCode, Method, PingResult, ResponseBody, RpcError, RpcId,
|
||||
@@ -61,6 +62,10 @@ pub type RpcHandler = Arc<
|
||||
>;
|
||||
|
||||
const DEFAULT_RPC_TIMEOUT: Duration = Duration::from_secs(15);
|
||||
// The upload transaction owns its operation deadline and may need a bounded
|
||||
// cleanup before it can return a useful structured error. Keep only that
|
||||
// transport alive slightly longer so it does not replace the result.
|
||||
const EXTENSION_RESPONSE_GRACE: Duration = Duration::from_secs(2);
|
||||
/// Upper bound on `wait_for_browser_ms` accepted over IPC.
|
||||
const MAX_BROWSER_WAIT: Duration = Duration::from_secs(60);
|
||||
// `session.stop` fast-fails while another tool is active; this budget
|
||||
@@ -223,6 +228,11 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
|
||||
Ok(v) => ResponseBody::Ok(v),
|
||||
Err(e) => ResponseBody::Err(e),
|
||||
},
|
||||
Method::TransferBegin => handle_transfer_begin(&state, params),
|
||||
Method::TransferChunk => handle_transfer_chunk(&state, params),
|
||||
Method::TransferFinish => handle_transfer_finish(&state, params),
|
||||
Method::TransferRead => handle_transfer_read(&state, params),
|
||||
Method::TransferRelease => handle_transfer_release(&state, params),
|
||||
Method::ToolTabList
|
||||
| Method::ToolTabCreate
|
||||
| Method::ToolTabClose
|
||||
@@ -246,6 +256,8 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
|
||||
| Method::ToolFill
|
||||
| Method::ToolPress
|
||||
| Method::ToolSelect
|
||||
| Method::ToolUpload
|
||||
| Method::ToolDownload
|
||||
| Method::ToolEvaluate
|
||||
| Method::ToolWaitForNavigation
|
||||
| Method::ToolRequestHelp
|
||||
@@ -324,7 +336,7 @@ async fn handle_tool_dispatch(
|
||||
data: None,
|
||||
});
|
||||
}
|
||||
let timeout = match tool_dispatch_timeout(¶ms) {
|
||||
let timeout = match tool_dispatch_transport_timeout(&method, ¶ms) {
|
||||
Ok(timeout) => timeout,
|
||||
Err(err) => return ResponseBody::Err(err),
|
||||
};
|
||||
@@ -338,24 +350,180 @@ async fn handle_tool_dispatch(
|
||||
});
|
||||
}
|
||||
};
|
||||
let mut params = params;
|
||||
let mut download_transfer_id: Option<String> = None;
|
||||
if method == Method::ToolUpload {
|
||||
let mut upload: UploadParams = match serde_json::from_value(params) {
|
||||
Ok(v) => v,
|
||||
Err(err) => return ResponseBody::Err(invalid_params(err.to_string())),
|
||||
};
|
||||
if upload.files.is_empty() || upload.files.len() > super::file_transfer::MAX_UPLOAD_FILES {
|
||||
return ResponseBody::Err(invalid_params(format!(
|
||||
"upload requires 1..={} files",
|
||||
super::file_transfer::MAX_UPLOAD_FILES
|
||||
)));
|
||||
}
|
||||
let ids: Vec<String> = upload.files.iter().map(|f| f.transfer_id.clone()).collect();
|
||||
let paths = match state.transfers.resolve_uploads(&session_id.0, &ids) {
|
||||
Ok(v) => v,
|
||||
Err(err) => return ResponseBody::Err(err),
|
||||
};
|
||||
for (file, path) in upload.files.iter_mut().zip(paths) {
|
||||
file.staged_path = Some(path.to_string_lossy().into_owned());
|
||||
}
|
||||
params = serde_json::to_value(upload).unwrap_or(Value::Null);
|
||||
} else if method == Method::ToolDownload {
|
||||
let mut download: DownloadParams = match serde_json::from_value(params) {
|
||||
Ok(v) => v,
|
||||
Err(err) => return ResponseBody::Err(invalid_params(err.to_string())),
|
||||
};
|
||||
let staging = match state.transfers.begin_download(&session_id.0) {
|
||||
Ok(v) => v,
|
||||
Err(err) => return ResponseBody::Err(err),
|
||||
};
|
||||
download.browser_relative_dir = Some(staging.browser_relative_dir);
|
||||
download.max_byte_size = Some(super::file_transfer::MAX_TRANSFER_BYTES);
|
||||
download_transfer_id = Some(staging.transfer_id);
|
||||
params = serde_json::to_value(download).unwrap_or(Value::Null);
|
||||
}
|
||||
let entry = inflight_guard.entry();
|
||||
// `record_stop` must reach the extension while `record_await` holds the
|
||||
// serial busy lock — finishing the recording unblocks await.
|
||||
let outcome = if method == Method::ToolRecordStop {
|
||||
state
|
||||
.tool_queues
|
||||
.dispatch_unlocked(&session_id, method, params, timeout, Some(entry))
|
||||
.dispatch_unlocked(&session_id, method.clone(), params, timeout, Some(entry))
|
||||
.await
|
||||
} else {
|
||||
state
|
||||
.tool_queues
|
||||
.dispatch(&session_id, method, params, timeout, Some(entry))
|
||||
.dispatch(&session_id, method.clone(), params, timeout, Some(entry))
|
||||
.await
|
||||
};
|
||||
drop(inflight_guard);
|
||||
match outcome {
|
||||
Ok(v) if method == Method::ToolDownload => {
|
||||
let id = download_transfer_id.expect("download transfer allocated");
|
||||
let mut result: DownloadResult = match serde_json::from_value(v) {
|
||||
Ok(v) => v,
|
||||
Err(err) => {
|
||||
state
|
||||
.transfers
|
||||
.release(TransferIdParams { transfer_id: id });
|
||||
return ResponseBody::Err(RpcError {
|
||||
code: ErrorCode::ProtocolError,
|
||||
message: format!("invalid tool.download result: {err}"),
|
||||
data: None,
|
||||
});
|
||||
}
|
||||
};
|
||||
let Some(path) = result.browser_path.take() else {
|
||||
state
|
||||
.transfers
|
||||
.release(TransferIdParams { transfer_id: id });
|
||||
return ResponseBody::Err(RpcError {
|
||||
code: ErrorCode::ProtocolError,
|
||||
message: "tool.download returned no browser_path".into(),
|
||||
data: None,
|
||||
});
|
||||
};
|
||||
match state
|
||||
.transfers
|
||||
.import_download(&id, std::path::Path::new(&path))
|
||||
{
|
||||
Ok(size) => {
|
||||
result.byte_size = size;
|
||||
result.transfer_id = Some(id);
|
||||
ResponseBody::Ok(serde_json::to_value(result).unwrap_or(Value::Null))
|
||||
}
|
||||
Err(err) => {
|
||||
state
|
||||
.transfers
|
||||
.release(TransferIdParams { transfer_id: id });
|
||||
ResponseBody::Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(v) => ResponseBody::Ok(v),
|
||||
Err(err) => ResponseBody::Err(err.into_rpc()),
|
||||
Err(err) => {
|
||||
if let Some(id) = download_transfer_id {
|
||||
state
|
||||
.transfers
|
||||
.release(TransferIdParams { transfer_id: id });
|
||||
}
|
||||
ResponseBody::Err(err.into_rpc())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_transfer_begin(state: &Arc<DaemonState>, params: Value) -> ResponseBody {
|
||||
let p: TransferBeginParams = match serde_json::from_value(params) {
|
||||
Ok(v) => v,
|
||||
Err(err) => return ResponseBody::Err(invalid_params(err.to_string())),
|
||||
};
|
||||
if state
|
||||
.sessions
|
||||
.get(&SessionId(p.session_id.clone()))
|
||||
.is_none()
|
||||
{
|
||||
return ResponseBody::Err(RpcError {
|
||||
code: ErrorCode::NotFound,
|
||||
message: format!("session {} unknown", p.session_id),
|
||||
data: None,
|
||||
});
|
||||
}
|
||||
match state.transfers.begin_upload(p) {
|
||||
Ok(v) => ResponseBody::Ok(serde_json::to_value(v).unwrap_or(Value::Null)),
|
||||
Err(err) => ResponseBody::Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_transfer_chunk(state: &Arc<DaemonState>, params: Value) -> ResponseBody {
|
||||
let p = match serde_json::from_value(params) {
|
||||
Ok(v) => v,
|
||||
Err(err) => return ResponseBody::Err(invalid_params(err.to_string())),
|
||||
};
|
||||
match state.transfers.write_chunk(p) {
|
||||
Ok(v) => ResponseBody::Ok(serde_json::to_value(v).unwrap_or(Value::Null)),
|
||||
Err(err) => ResponseBody::Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_transfer_finish(state: &Arc<DaemonState>, params: Value) -> ResponseBody {
|
||||
let p = match serde_json::from_value(params) {
|
||||
Ok(v) => v,
|
||||
Err(err) => return ResponseBody::Err(invalid_params(err.to_string())),
|
||||
};
|
||||
match state.transfers.finish_upload(p) {
|
||||
Ok(v) => ResponseBody::Ok(serde_json::to_value(v).unwrap_or(Value::Null)),
|
||||
Err(err) => ResponseBody::Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_transfer_read(state: &Arc<DaemonState>, params: Value) -> ResponseBody {
|
||||
let p = match serde_json::from_value(params) {
|
||||
Ok(v) => v,
|
||||
Err(err) => return ResponseBody::Err(invalid_params(err.to_string())),
|
||||
};
|
||||
match state.transfers.read_chunk(p) {
|
||||
Ok(v) => ResponseBody::Ok(serde_json::to_value(v).unwrap_or(Value::Null)),
|
||||
Err(err) => ResponseBody::Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
fn handle_transfer_release(state: &Arc<DaemonState>, params: Value) -> ResponseBody {
|
||||
let p: TransferIdParams = match serde_json::from_value(params) {
|
||||
Ok(v) => v,
|
||||
Err(err) => return ResponseBody::Err(invalid_params(err.to_string())),
|
||||
};
|
||||
ResponseBody::Ok(serde_json::to_value(state.transfers.release(p)).unwrap_or(Value::Null))
|
||||
}
|
||||
|
||||
fn invalid_params(message: impl Into<String>) -> RpcError {
|
||||
RpcError {
|
||||
code: ErrorCode::InvalidParams,
|
||||
message: message.into(),
|
||||
data: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -533,6 +701,16 @@ fn tool_dispatch_timeout(params: &Value) -> Result<Duration, RpcError> {
|
||||
Ok(Duration::from_millis(u64::from(ms)))
|
||||
}
|
||||
|
||||
fn tool_dispatch_transport_timeout(method: &Method, params: &Value) -> Result<Duration, RpcError> {
|
||||
tool_dispatch_timeout(params).map(|timeout| {
|
||||
if matches!(method, Method::ToolUpload | Method::ToolDownload) {
|
||||
timeout.saturating_add(EXTENSION_RESPONSE_GRACE)
|
||||
} else {
|
||||
timeout
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Local CLI-facing shapes. Intentionally distinct from
|
||||
// `bsk_protocol::tools::SessionStart*` (which describes the WS-facing
|
||||
// `tool.session_*` round-trip with the extension) so the two sides can
|
||||
@@ -798,6 +976,7 @@ async fn handle_session_stop(
|
||||
.await
|
||||
{
|
||||
Ok(stop) => {
|
||||
state.transfers.release_session(&session_id.0);
|
||||
let result = CliSessionStopResult {
|
||||
stopped: vec![session_id.0],
|
||||
failed: Vec::new(),
|
||||
@@ -897,6 +1076,7 @@ async fn handle_session_stop_all(
|
||||
.await
|
||||
{
|
||||
Ok(stop) => {
|
||||
state.transfers.release_session(&id.0);
|
||||
stopped.push(id.0);
|
||||
returned_tab_ids.extend(stop.returned_tab_ids);
|
||||
return_failures.extend(stop.return_failures);
|
||||
@@ -1329,6 +1509,30 @@ mod tests {
|
||||
assert_eq!(got, std::time::Duration::from_millis(300_000));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extension_transport_outlives_the_operation_deadline() {
|
||||
let params = serde_json::json!({
|
||||
"session_id": "abcd",
|
||||
"timeout_ms": 60_000,
|
||||
});
|
||||
let dispatch_timeout =
|
||||
tool_dispatch_transport_timeout(&Method::ToolUpload, ¶ms).unwrap();
|
||||
|
||||
assert_eq!(dispatch_timeout, Duration::from_secs(62));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extension_response_grace_does_not_change_other_tools() {
|
||||
let params = serde_json::json!({
|
||||
"session_id": "abcd",
|
||||
"timeout_ms": 60_000,
|
||||
});
|
||||
let dispatch_timeout =
|
||||
tool_dispatch_transport_timeout(&Method::ToolClick, ¶ms).unwrap();
|
||||
|
||||
assert_eq!(dispatch_timeout, Duration::from_secs(60));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tool_dispatch_timeout_rejects_zero_timeout_ms() {
|
||||
let params = serde_json::json!({
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
pub mod abort;
|
||||
pub mod browsers;
|
||||
mod cancel_forward;
|
||||
pub mod file_transfer;
|
||||
pub mod inflight;
|
||||
pub mod info;
|
||||
pub mod ipc;
|
||||
|
||||
@@ -438,6 +438,7 @@ async fn dispatch_with_sender(
|
||||
lifecycle_cancel: Option<AbortToken>,
|
||||
cancel_cleanup_timeout: Duration,
|
||||
) -> Result<Value, DispatchError> {
|
||||
let effect_aware_transfer = is_effect_aware_transfer(&method);
|
||||
let (respond_tx, respond_rx) = oneshot::channel();
|
||||
let job = ToolJob {
|
||||
method,
|
||||
@@ -474,15 +475,21 @@ async fn dispatch_with_sender(
|
||||
mpsc::error::TrySendError::Closed(_) => DispatchError::QueueClosed,
|
||||
});
|
||||
}
|
||||
let mut response_timeout = timeout.saturating_add(Duration::from_secs(1));
|
||||
if lifecycle_cancellable {
|
||||
// A lifecycle cancel can arrive at the original RPC deadline and
|
||||
// intentionally keeps the worker alive while extension cleanup
|
||||
// settles. Do not let this outer waiter abandon that reconciliation
|
||||
// early, which could reopen a daemon session whose window is gone.
|
||||
response_timeout = response_timeout.saturating_add(cancel_cleanup_timeout);
|
||||
}
|
||||
let waited = tokio::time::timeout(response_timeout, respond_rx).await;
|
||||
// Lifecycle teardown and effect-aware transfers both keep the worker busy
|
||||
// for bounded compensation after their original deadline. Keep the outer
|
||||
// waiter alive for the same grace period so it cannot abandon reconciliation.
|
||||
let response_grace = if lifecycle_cancellable || effect_aware_transfer {
|
||||
cancel_cleanup_timeout
|
||||
} else {
|
||||
Duration::ZERO
|
||||
};
|
||||
let waited = tokio::time::timeout(
|
||||
timeout
|
||||
.saturating_add(response_grace)
|
||||
.saturating_add(Duration::from_secs(1)),
|
||||
respond_rx,
|
||||
)
|
||||
.await;
|
||||
match waited {
|
||||
Ok(Ok(Ok(v))) => Ok(v),
|
||||
Ok(Ok(Err(rpc))) => Err(DispatchError::Rpc(rpc)),
|
||||
@@ -636,44 +643,85 @@ async fn forward_one(
|
||||
.lifecycle_cancel
|
||||
.as_ref()
|
||||
.map(|_| &forward_lifecycle_cancel as &(dyn Fn() + Sync));
|
||||
let send_deadline_cancel = || {
|
||||
client
|
||||
.sink
|
||||
.send(Frame::Request(RequestFrame {
|
||||
id: format!("deadline-cancel-{rpc_id}"),
|
||||
method: Method::Cancel,
|
||||
params: Some(serde_json::json!({ "rpc_id": rpc_id })),
|
||||
}))
|
||||
.is_ok()
|
||||
};
|
||||
let deadline_cancel: Option<&(dyn Fn() -> bool + Sync)> =
|
||||
is_effect_aware_transfer(&job.method).then_some(&send_deadline_cancel);
|
||||
let waited = await_with_optional_cancel(
|
||||
job.timeout,
|
||||
job.cancel_cleanup_timeout,
|
||||
waiter,
|
||||
cancel_token.as_ref(),
|
||||
on_abort,
|
||||
deadline_cancel,
|
||||
)
|
||||
.await;
|
||||
let response = match waited {
|
||||
WaitOutcome::Response(resp) => resp,
|
||||
WaitOutcome::CancelledAfterResponse(resp) => {
|
||||
if job.method == Method::ToolSessionStop
|
||||
&& let ResponseBody::Ok(value) = resp.body
|
||||
&& let ResponseBody::Ok(value) = &resp.body
|
||||
{
|
||||
// Teardown crossed its final irreversible boundary before
|
||||
// the cancel landed. Commit the real extension result so the
|
||||
// daemon does not retain a session whose window is gone.
|
||||
return Ok(value);
|
||||
return Ok(value.clone());
|
||||
}
|
||||
// Cancellation wins the external verdict, but only after the
|
||||
// extension's original RPC has settled. Preserve any non-cancel
|
||||
// error so compensation failures remain explicit instead of
|
||||
// being hidden behind a generic cancelled result.
|
||||
if let ResponseBody::Err(err) = resp.body
|
||||
&& !matches!(err.code, ErrorCode::Cancelled | ErrorCode::UserAborted)
|
||||
{
|
||||
return Err(err);
|
||||
// File transfer commits are irreversible. A late cancel cannot
|
||||
// overwrite a confirmed success, and an unknown transfer effect
|
||||
// must remain explicit so callers do not retry or release upload
|
||||
// staging as though nothing happened.
|
||||
if is_effect_aware_transfer(&job.method) {
|
||||
match &resp.body {
|
||||
ResponseBody::Ok(_) => resp,
|
||||
ResponseBody::Err(err)
|
||||
if transfer_effect(err) == Some("unknown")
|
||||
|| transfer_effect(err) == Some("committed") =>
|
||||
{
|
||||
return Err(err.clone());
|
||||
}
|
||||
ResponseBody::Err(err)
|
||||
if !matches!(err.code, ErrorCode::Cancelled | ErrorCode::UserAborted) =>
|
||||
{
|
||||
return Err(err.clone());
|
||||
}
|
||||
ResponseBody::Err(_) => {
|
||||
return Err(cancelled_error(
|
||||
job.inflight.as_deref(),
|
||||
"tool dispatch cancelled after extension cleanup",
|
||||
));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// For ordinary tools cancellation keeps the existing verdict,
|
||||
// while non-cancel errors still expose compensation failures.
|
||||
if let ResponseBody::Err(err) = resp.body
|
||||
&& !matches!(err.code, ErrorCode::Cancelled | ErrorCode::UserAborted)
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
return Err(cancelled_error(
|
||||
job.inflight.as_deref(),
|
||||
"tool dispatch cancelled after extension cleanup",
|
||||
));
|
||||
}
|
||||
return Err(cancelled_error(
|
||||
job.inflight.as_deref(),
|
||||
"tool dispatch cancelled after extension cleanup",
|
||||
));
|
||||
}
|
||||
WaitOutcome::TimedOutAfterResponse(resp) => match resp.body {
|
||||
ResponseBody::Ok(value) if job.method == Method::ToolSessionStop => {
|
||||
ResponseBody::Ok(value)
|
||||
if job.method == Method::ToolSessionStop
|
||||
|| is_effect_aware_transfer(&job.method) =>
|
||||
{
|
||||
// The close crossed its irreversible boundary during the
|
||||
// timeout cleanup grace. Reconcile daemon state instead of
|
||||
// reopening a session whose window is already gone.
|
||||
// timeout cleanup grace, or the transfer committed before its
|
||||
// deadline cancel settled. Preserve the irreversible result.
|
||||
return Ok(value);
|
||||
}
|
||||
ResponseBody::Err(err)
|
||||
@@ -681,6 +729,9 @@ async fn forward_one(
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
ResponseBody::Err(err) if is_effect_aware_transfer(&job.method) => {
|
||||
return Err(timed_out_transfer_error(&err));
|
||||
}
|
||||
_ => {
|
||||
return Err(RpcError {
|
||||
code: ErrorCode::Timeout,
|
||||
@@ -691,6 +742,14 @@ async fn forward_one(
|
||||
},
|
||||
WaitOutcome::CleanupTimeout => {
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
if is_effect_aware_transfer(&job.method) {
|
||||
return Err(unknown_transfer_error(
|
||||
ErrorCode::Timeout,
|
||||
"cancelled file transfer did not confirm its outcome before cleanup timed out",
|
||||
"cleanup",
|
||||
true,
|
||||
));
|
||||
}
|
||||
return Err(RpcError {
|
||||
code: ErrorCode::Timeout,
|
||||
message: format!(
|
||||
@@ -700,8 +759,25 @@ async fn forward_one(
|
||||
data: Some(serde_json::json!({ "reason": "cancel_cleanup_timeout" })),
|
||||
});
|
||||
}
|
||||
WaitOutcome::TimeoutCleanupFailed => {
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
return Err(unknown_transfer_error(
|
||||
ErrorCode::Timeout,
|
||||
"file transfer timed out and cleanup could not be confirmed",
|
||||
"cleanup",
|
||||
true,
|
||||
));
|
||||
}
|
||||
WaitOutcome::WaiterClosed => {
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
if is_effect_aware_transfer(&job.method) {
|
||||
return Err(unknown_transfer_error(
|
||||
ErrorCode::ProtocolError,
|
||||
"file transfer transport closed after dispatch; outcome is unknown",
|
||||
"transport",
|
||||
false,
|
||||
));
|
||||
}
|
||||
return Err(RpcError {
|
||||
code: ErrorCode::ProtocolError,
|
||||
message: "transport closed mid-call".into(),
|
||||
@@ -710,6 +786,14 @@ async fn forward_one(
|
||||
}
|
||||
WaitOutcome::Timeout => {
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
if is_effect_aware_transfer(&job.method) {
|
||||
return Err(unknown_transfer_error(
|
||||
ErrorCode::Timeout,
|
||||
"file transfer timed out after dispatch; outcome is unknown",
|
||||
"transport",
|
||||
false,
|
||||
));
|
||||
}
|
||||
return Err(RpcError {
|
||||
code: ErrorCode::Timeout,
|
||||
message: format!("tool RPC timed out after {:?}", job.timeout),
|
||||
@@ -723,12 +807,65 @@ async fn forward_one(
|
||||
}
|
||||
}
|
||||
|
||||
fn is_effect_aware_transfer(method: &Method) -> bool {
|
||||
matches!(method, Method::ToolUpload | Method::ToolDownload)
|
||||
}
|
||||
|
||||
fn transfer_effect(err: &RpcError) -> Option<&str> {
|
||||
err.data.as_ref()?.get("effect_state")?.as_str()
|
||||
}
|
||||
|
||||
fn unknown_transfer_error(
|
||||
code: ErrorCode,
|
||||
message: impl Into<String>,
|
||||
phase: &str,
|
||||
cleanup_failed: bool,
|
||||
) -> RpcError {
|
||||
let mut data = serde_json::json!({
|
||||
"reason": "transfer_outcome_unknown",
|
||||
"effect_state": "unknown",
|
||||
"phase": phase,
|
||||
});
|
||||
if cleanup_failed {
|
||||
data["cleanup_state"] = serde_json::json!("failed");
|
||||
}
|
||||
RpcError {
|
||||
code,
|
||||
message: message.into(),
|
||||
data: Some(data),
|
||||
}
|
||||
}
|
||||
|
||||
fn timed_out_transfer_error(err: &RpcError) -> RpcError {
|
||||
let data = err.data.as_ref();
|
||||
RpcError {
|
||||
code: ErrorCode::Timeout,
|
||||
message: "file transfer timed out after dispatch".into(),
|
||||
data: Some(serde_json::json!({
|
||||
"reason": "transfer_timeout",
|
||||
"effect_state": data
|
||||
.and_then(|value| value.get("effect_state"))
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("unknown"),
|
||||
"phase": data
|
||||
.and_then(|value| value.get("phase"))
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("cleanup"),
|
||||
"cleanup_state": data
|
||||
.and_then(|value| value.get("cleanup_state"))
|
||||
.and_then(Value::as_str)
|
||||
.unwrap_or("complete"),
|
||||
})),
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
enum WaitOutcome {
|
||||
Response(bsk_protocol::ResponseFrame),
|
||||
CancelledAfterResponse(bsk_protocol::ResponseFrame),
|
||||
TimedOutAfterResponse(bsk_protocol::ResponseFrame),
|
||||
CleanupTimeout,
|
||||
TimeoutCleanupFailed,
|
||||
WaiterClosed,
|
||||
Timeout,
|
||||
}
|
||||
@@ -739,6 +876,7 @@ async fn await_with_optional_cancel(
|
||||
mut waiter: oneshot::Receiver<bsk_protocol::ResponseFrame>,
|
||||
cancel: Option<&super::abort::AbortToken>,
|
||||
on_abort: Option<&(dyn Fn() + Sync)>,
|
||||
on_deadline: Option<&(dyn Fn() -> bool + Sync)>,
|
||||
) -> WaitOutcome {
|
||||
match cancel {
|
||||
Some(token) => {
|
||||
@@ -764,6 +902,16 @@ async fn await_with_optional_cancel(
|
||||
Err(_) => WaitOutcome::WaiterClosed,
|
||||
},
|
||||
_ = &mut deadline => {
|
||||
if let Some(send_cancel) = on_deadline {
|
||||
return if send_cancel() {
|
||||
match tokio::time::timeout(cleanup_timeout, &mut waiter).await {
|
||||
Ok(Ok(resp)) => WaitOutcome::TimedOutAfterResponse(resp),
|
||||
Ok(Err(_)) | Err(_) => WaitOutcome::TimeoutCleanupFailed,
|
||||
}
|
||||
} else {
|
||||
WaitOutcome::TimeoutCleanupFailed
|
||||
};
|
||||
}
|
||||
let Some(on_abort) = on_abort else {
|
||||
return WaitOutcome::Timeout;
|
||||
};
|
||||
@@ -779,7 +927,16 @@ async fn await_with_optional_cancel(
|
||||
None => match tokio::time::timeout(timeout, &mut waiter).await {
|
||||
Ok(Ok(resp)) => WaitOutcome::Response(resp),
|
||||
Ok(Err(_)) => WaitOutcome::WaiterClosed,
|
||||
Err(_) => WaitOutcome::Timeout,
|
||||
Err(_) => match on_deadline {
|
||||
Some(send_cancel) if send_cancel() => {
|
||||
match tokio::time::timeout(cleanup_timeout, &mut waiter).await {
|
||||
Ok(Ok(resp)) => WaitOutcome::TimedOutAfterResponse(resp),
|
||||
Ok(Err(_)) | Err(_) => WaitOutcome::TimeoutCleanupFailed,
|
||||
}
|
||||
}
|
||||
Some(_) => WaitOutcome::TimeoutCleanupFailed,
|
||||
None => WaitOutcome::Timeout,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -864,6 +1021,7 @@ mod await_with_optional_cancel_tests {
|
||||
rx,
|
||||
Some(&token),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
@@ -887,6 +1045,7 @@ mod await_with_optional_cancel_tests {
|
||||
rx,
|
||||
Some(&token),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
match outcome {
|
||||
@@ -907,6 +1066,7 @@ mod await_with_optional_cancel_tests {
|
||||
rx,
|
||||
None,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(outcome, WaitOutcome::Response(_)));
|
||||
@@ -924,6 +1084,7 @@ mod await_with_optional_cancel_tests {
|
||||
rx,
|
||||
Some(&token),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
});
|
||||
@@ -948,6 +1109,7 @@ mod await_with_optional_cancel_tests {
|
||||
rx,
|
||||
Some(&token),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(outcome, WaitOutcome::CleanupTimeout));
|
||||
@@ -972,12 +1134,49 @@ mod await_with_optional_cancel_tests {
|
||||
rx,
|
||||
Some(&token),
|
||||
Some(&forward_abort),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(abort_forwarded.load(Ordering::SeqCst));
|
||||
assert!(matches!(outcome, WaitOutcome::TimedOutAfterResponse(_)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn transfer_deadline_sends_cancel_and_waits_for_cleanup_response() {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(Duration::from_millis(5)).await;
|
||||
tx.send(dummy_response()).unwrap();
|
||||
});
|
||||
let send_cancel = || true;
|
||||
let outcome = await_with_optional_cancel(
|
||||
Duration::from_millis(1),
|
||||
Duration::from_secs(1),
|
||||
rx,
|
||||
None,
|
||||
None,
|
||||
Some(&send_cancel),
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(outcome, WaitOutcome::TimedOutAfterResponse(_)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn transfer_deadline_is_unknown_when_cancel_cannot_be_sent() {
|
||||
let (_tx, rx) = oneshot::channel();
|
||||
let send_cancel = || false;
|
||||
let outcome = await_with_optional_cancel(
|
||||
Duration::from_millis(1),
|
||||
Duration::from_secs(1),
|
||||
rx,
|
||||
None,
|
||||
None,
|
||||
Some(&send_cancel),
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(outcome, WaitOutcome::TimeoutCleanupFailed));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -225,6 +225,10 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
|
||||
.with_context(|| format!("bind IPC endpoint {}", sock_path.display()))?;
|
||||
|
||||
let state = Arc::new(DaemonState::new(cfg.clone()));
|
||||
state
|
||||
.transfers
|
||||
.initialize()
|
||||
.context("initialize transfer staging")?;
|
||||
let session_idle_task = spawn_session_idle_reaper(Arc::clone(&state));
|
||||
let browser_liveness_task = spawn_browser_liveness_reaper(Arc::clone(&state));
|
||||
// Fired by the update check task after a successful auto-update:
|
||||
@@ -448,6 +452,7 @@ pub(crate) fn spawn_browser_liveness_reaper(
|
||||
for s in state.sessions.purge_browser(&client.id) {
|
||||
state.tool_queues.remove(&s.id);
|
||||
state.session_interrupts.drop_session(&s.id);
|
||||
state.transfers.release_session(&s.id.0);
|
||||
debug!(session = %s.id, "purged session on browser liveness timeout");
|
||||
}
|
||||
}
|
||||
@@ -485,7 +490,10 @@ pub(crate) fn spawn_session_idle_reaper(state: Arc<DaemonState>) -> tokio::task:
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => info!(session = %session_id, "idle session stopped"),
|
||||
Ok(_) => {
|
||||
state.transfers.release_session(&session_id.0);
|
||||
info!(session = %session_id, "idle session stopped");
|
||||
}
|
||||
Err(StopSessionError::SessionBusy | StopSessionError::Stopping) => {
|
||||
debug!(session = %session_id, "idle session still active; retrying later");
|
||||
}
|
||||
@@ -496,6 +504,7 @@ pub(crate) fn spawn_session_idle_reaper(state: Arc<DaemonState>) -> tokio::task:
|
||||
&state.session_interrupts,
|
||||
&session_id,
|
||||
);
|
||||
state.transfers.release_session(&session_id.0);
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(session = %session_id, error = %err, "failed to stop idle session");
|
||||
|
||||
@@ -7,6 +7,7 @@ use tokio::task::JoinHandle;
|
||||
|
||||
use super::abort::AbortRegistry;
|
||||
use super::browsers::BrowserRegistry;
|
||||
use super::file_transfer::TransferRegistry;
|
||||
use super::inflight::ToolInflightRegistry;
|
||||
use super::ipc::IpcHandle;
|
||||
use super::queue::ToolQueueRegistry;
|
||||
@@ -51,6 +52,9 @@ pub struct DaemonState {
|
||||
/// `SessionRegistry` because the signal is a transient runtime
|
||||
/// control state.
|
||||
pub session_interrupts: Arc<SessionInterruptRegistry>,
|
||||
/// Operation-scoped local file staging. The extension only sees paths
|
||||
/// minted here; agent-facing RPCs use opaque transfer ids.
|
||||
pub transfers: Arc<TransferRegistry>,
|
||||
}
|
||||
|
||||
impl DaemonState {
|
||||
@@ -64,6 +68,7 @@ impl DaemonState {
|
||||
));
|
||||
let abort_registry = Arc::new(AbortRegistry::new());
|
||||
let session_interrupts = Arc::new(SessionInterruptRegistry::new());
|
||||
let transfers = Arc::new(TransferRegistry::new().expect("initialise transfer staging"));
|
||||
Self {
|
||||
config,
|
||||
browsers,
|
||||
@@ -72,6 +77,7 @@ impl DaemonState {
|
||||
abort_registry,
|
||||
tool_inflight,
|
||||
session_interrupts,
|
||||
transfers,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -295,6 +295,7 @@ async fn drive_connection(
|
||||
for session in state.sessions.purge_browser(&browser_id) {
|
||||
state.tool_queues.remove(&session.id);
|
||||
state.session_interrupts.drop_session(&session.id);
|
||||
state.transfers.release_session(&session.id.0);
|
||||
debug!(session = %session.id, "purged stale session before browser reconnect");
|
||||
}
|
||||
let (tx, mut rx) = mpsc::unbounded_channel::<Frame>();
|
||||
@@ -407,6 +408,7 @@ async fn drive_connection(
|
||||
for s in state.sessions.purge_browser(&browser_id) {
|
||||
state.tool_queues.remove(&s.id);
|
||||
state.session_interrupts.drop_session(&s.id);
|
||||
state.transfers.release_session(&s.id.0);
|
||||
debug!(session = %s.id, "purged session on browser disconnect");
|
||||
}
|
||||
} else {
|
||||
@@ -503,6 +505,7 @@ fn handle_session_window_closed(
|
||||
&state.session_interrupts,
|
||||
&session_id,
|
||||
) {
|
||||
state.transfers.release_session(&session_id.0);
|
||||
info!(session = %session_id, "session removed: user closed Agent Window");
|
||||
} else {
|
||||
debug!(session = %session_id, "session.window_closed for unknown session id");
|
||||
|
||||
@@ -97,6 +97,8 @@ fn dispatch(cli: Cli, format: Format) -> Result<(), CliError> {
|
||||
Command::Fill(args) => cli::interaction::dispatch_fill(args, format),
|
||||
Command::Press(args) => cli::interaction::dispatch_press(args, format),
|
||||
Command::Select(args) => cli::interaction::dispatch_select(args, format),
|
||||
Command::Upload(args) => cli::upload::dispatch(args, format),
|
||||
Command::Download(args) => cli::download::dispatch(args, format),
|
||||
Command::Evaluate(args) => cli::evaluate::dispatch(args, format),
|
||||
Command::WaitForNavigation(args) => cli::waits::dispatch_wait_for_navigation(args, format),
|
||||
Command::WaitMs(args) => cli::waits::dispatch_wait_ms(args, format),
|
||||
|
||||
@@ -213,6 +213,45 @@ fn rejects_zero_click_count() {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_upload_with_repeated_files() {
|
||||
let cli = parse(&[
|
||||
"bsk",
|
||||
"upload",
|
||||
"@e3",
|
||||
"--file",
|
||||
"one.png",
|
||||
"--file",
|
||||
"two.png",
|
||||
"--session",
|
||||
"s1",
|
||||
]);
|
||||
let Command::Upload(args) = cli.command else {
|
||||
panic!("expected upload command");
|
||||
};
|
||||
assert_eq!(args.target.as_deref(), Some("@e3"));
|
||||
assert_eq!(args.files.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_download_with_exact_output_policy() {
|
||||
let cli = parse(&[
|
||||
"bsk",
|
||||
"download",
|
||||
"#export",
|
||||
"--out",
|
||||
"result.zip",
|
||||
"--session",
|
||||
"s1",
|
||||
"--overwrite",
|
||||
]);
|
||||
let Command::Download(args) = cli.command else {
|
||||
panic!("expected download command");
|
||||
};
|
||||
assert_eq!(args.target.as_deref(), Some("#export"));
|
||||
assert!(args.overwrite);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parses_record_start_with_browser_and_url() {
|
||||
let cli = parse(&[
|
||||
|
||||
@@ -478,6 +478,7 @@ async fn observe_returns_semantic_text_and_ref_count() {
|
||||
tab_id: 13,
|
||||
truncated: false,
|
||||
dialogs: vec![],
|
||||
hover_probe: None,
|
||||
debug: None,
|
||||
})
|
||||
.unwrap(),
|
||||
@@ -494,6 +495,7 @@ async fn observe_returns_semantic_text_and_ref_count() {
|
||||
max_depth: None,
|
||||
max_tokens: None,
|
||||
debug_surfaces: false,
|
||||
probe_hover: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -0,0 +1,56 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "DownloadParams",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"session_id"
|
||||
],
|
||||
"properties": {
|
||||
"browser_relative_dir": {
|
||||
"description": "Daemon-injected relative directory beneath Chrome's Downloads root.",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"max_byte_size": {
|
||||
"description": "Daemon-injected authoritative transfer size limit. The extension uses it only for early cancellation; daemon import remains the final check.",
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "uint64",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"ref": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"selector": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"session_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"tab_id": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "int64"
|
||||
},
|
||||
"timeout_ms": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "DownloadResult",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"byte_size",
|
||||
"suggested_filename",
|
||||
"tab_id"
|
||||
],
|
||||
"properties": {
|
||||
"browser_path": {
|
||||
"description": "Extension-internal completed Chrome download path, stripped by the daemon.",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"byte_size": {
|
||||
"type": "integer",
|
||||
"format": "uint64",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"danger": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"mime": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"suggested_filename": {
|
||||
"type": "string"
|
||||
},
|
||||
"tab_id": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
},
|
||||
"transfer_id": {
|
||||
"description": "Opaque id returned by the daemon to the CLI.",
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"used_ref": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"used_selector": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "ObserveParams",
|
||||
"description": "Parameters for `tool.observe` — produce a semantic VOM observation for one tab plus a fresh `@e<N>` ref-store on the extension side.\n\nUnlike `tool.snapshot`, this path may run bounded perception probes such as hover-surface discovery. It must not submit input, click, navigate, or otherwise commit page state.",
|
||||
"description": "Parameters for `tool.observe` — produce a semantic VOM observation for one tab plus a fresh `@e<N>` ref-store on the extension side.\n\nUnlike `tool.snapshot`, this path may run bounded perception probes such as hover-surface discovery, but only when `probe_hover` asks for them. It must not submit input, click, navigate, or otherwise commit page state.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"session_id"
|
||||
@@ -30,6 +30,11 @@
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
},
|
||||
"probe_hover": {
|
||||
"description": "Opt in to active hover probing: the extension moves the real cursor over a bounded set of controls to discover hover-only menus and tooltips.\n\nOff by default. Probing costs seconds of wall clock, mutates the live page, and pays off only on pages whose content is genuinely hidden behind hover. Request it when a static observation looks like it is missing hover-revealed structure.",
|
||||
"default": false,
|
||||
"type": "boolean"
|
||||
},
|
||||
"session_id": {
|
||||
"type": "string"
|
||||
},
|
||||
|
||||
@@ -24,6 +24,17 @@
|
||||
"$ref": "#/definitions/JavaScriptDialogInfo"
|
||||
}
|
||||
},
|
||||
"hover_probe": {
|
||||
"description": "Present only when `probe_hover` was requested and probing actually ran.",
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/definitions/HoverProbeReport"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"ref_count": {
|
||||
"description": "Number of `@e<N>` refs registered for this session by this observation. Equivalent to the size of the new ref-store map.",
|
||||
"type": "integer",
|
||||
@@ -46,6 +57,24 @@
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"HoverProbeReport": {
|
||||
"description": "Reports what active hover probing did during an observation, so the agent can judge how far the live page may have drifted from the returned text.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"performed",
|
||||
"revealed_content"
|
||||
],
|
||||
"properties": {
|
||||
"performed": {
|
||||
"description": "The real cursor was moved over page content during this observation.",
|
||||
"type": "boolean"
|
||||
},
|
||||
"revealed_content": {
|
||||
"description": "Hovering surfaced content the static tree does not contain, so the page reacts to the cursor and may not have fully reverted.",
|
||||
"type": "boolean"
|
||||
}
|
||||
}
|
||||
},
|
||||
"JavaScriptDialogHandledAction": {
|
||||
"description": "How the extension resolved the dialog so CDP could continue.",
|
||||
"type": "string",
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "UploadParams",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"files",
|
||||
"session_id"
|
||||
],
|
||||
"properties": {
|
||||
"files": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/UploadFile"
|
||||
}
|
||||
},
|
||||
"ref": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"selector": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"session_id": {
|
||||
"type": "string"
|
||||
},
|
||||
"tab_id": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "int64"
|
||||
},
|
||||
"timeout_ms": {
|
||||
"type": [
|
||||
"integer",
|
||||
"null"
|
||||
],
|
||||
"format": "uint32",
|
||||
"minimum": 0.0
|
||||
}
|
||||
},
|
||||
"definitions": {
|
||||
"UploadFile": {
|
||||
"type": "object",
|
||||
"required": [
|
||||
"name",
|
||||
"transfer_id"
|
||||
],
|
||||
"properties": {
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"staged_path": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"transfer_id": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "UploadResult",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"file_names",
|
||||
"tab_id"
|
||||
],
|
||||
"properties": {
|
||||
"file_names": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"tab_id": {
|
||||
"type": "integer",
|
||||
"format": "int64"
|
||||
},
|
||||
"used_ref": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
},
|
||||
"used_selector": {
|
||||
"type": [
|
||||
"string",
|
||||
"null"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -85,6 +85,10 @@ fn main() {
|
||||
dump!(PressResult, "tool_press_result");
|
||||
dump!(SelectParams, "tool_select_params");
|
||||
dump!(SelectResult, "tool_select_result");
|
||||
dump!(UploadParams, "tool_upload_params");
|
||||
dump!(UploadResult, "tool_upload_result");
|
||||
dump!(DownloadParams, "tool_download_params");
|
||||
dump!(DownloadResult, "tool_download_result");
|
||||
|
||||
dump!(SnapshotParams, "tool_snapshot_params");
|
||||
dump!(SnapshotResult, "tool_snapshot_result");
|
||||
|
||||
@@ -78,6 +78,10 @@ pub enum Method {
|
||||
ToolPress,
|
||||
#[serde(rename = "tool.select")]
|
||||
ToolSelect,
|
||||
#[serde(rename = "tool.upload")]
|
||||
ToolUpload,
|
||||
#[serde(rename = "tool.download")]
|
||||
ToolDownload,
|
||||
#[serde(rename = "tool.snapshot")]
|
||||
ToolSnapshot,
|
||||
#[serde(rename = "tool.observe")]
|
||||
@@ -105,6 +109,17 @@ pub enum Method {
|
||||
#[serde(rename = "tool.record_await")]
|
||||
ToolRecordAwait,
|
||||
|
||||
#[serde(rename = "transfer.begin")]
|
||||
TransferBegin,
|
||||
#[serde(rename = "transfer.chunk")]
|
||||
TransferChunk,
|
||||
#[serde(rename = "transfer.finish")]
|
||||
TransferFinish,
|
||||
#[serde(rename = "transfer.read")]
|
||||
TransferRead,
|
||||
#[serde(rename = "transfer.release")]
|
||||
TransferRelease,
|
||||
|
||||
#[serde(rename = "cancel")]
|
||||
Cancel,
|
||||
}
|
||||
@@ -156,6 +171,8 @@ impl Method {
|
||||
| Method::ToolFill
|
||||
| Method::ToolPress
|
||||
| Method::ToolSelect
|
||||
| Method::ToolUpload
|
||||
| Method::ToolDownload
|
||||
| Method::ToolEvaluate
|
||||
// May navigate via optional `url` and changes Agent Window
|
||||
// chrome; gate behind pending-interrupt like other writes.
|
||||
@@ -194,6 +211,11 @@ impl Method {
|
||||
| Method::SystemPing
|
||||
| Method::SystemStatus
|
||||
| Method::BrowserList
|
||||
| Method::TransferBegin
|
||||
| Method::TransferChunk
|
||||
| Method::TransferFinish
|
||||
| Method::TransferRead
|
||||
| Method::TransferRelease
|
||||
| Method::Cancel => MethodEffect::ControlPlane,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
//! Content-transparent file transfer contracts.
|
||||
//!
|
||||
//! Public tool calls carry opaque transfer ids. Filesystem paths are injected
|
||||
//! by the daemon only on the daemon -> extension hop and are never accepted as
|
||||
//! an agent-facing source or destination.
|
||||
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct UploadFile {
|
||||
pub transfer_id: String,
|
||||
pub name: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub staged_path: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct UploadParams {
|
||||
pub session_id: String,
|
||||
#[serde(
|
||||
rename = "ref",
|
||||
alias = "ref_",
|
||||
default,
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub ref_: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub selector: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub tab_id: Option<i64>,
|
||||
pub files: Vec<UploadFile>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub timeout_ms: Option<u32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct UploadResult {
|
||||
pub tab_id: i64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub used_ref: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub used_selector: Option<String>,
|
||||
pub file_names: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct DownloadParams {
|
||||
pub session_id: String,
|
||||
#[serde(
|
||||
rename = "ref",
|
||||
alias = "ref_",
|
||||
default,
|
||||
skip_serializing_if = "Option::is_none"
|
||||
)]
|
||||
pub ref_: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub selector: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub tab_id: Option<i64>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub timeout_ms: Option<u32>,
|
||||
/// Daemon-injected relative directory beneath Chrome's Downloads root.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub browser_relative_dir: Option<String>,
|
||||
/// Daemon-injected authoritative transfer size limit. The extension uses
|
||||
/// it only for early cancellation; daemon import remains the final check.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub max_byte_size: Option<u64>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct DownloadResult {
|
||||
pub tab_id: i64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub used_ref: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub used_selector: Option<String>,
|
||||
pub suggested_filename: String,
|
||||
pub byte_size: u64,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub mime: Option<String>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub danger: Option<String>,
|
||||
/// Extension-internal completed Chrome download path, stripped by the daemon.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub browser_path: Option<String>,
|
||||
/// Opaque id returned by the daemon to the CLI.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub transfer_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct TransferBeginParams {
|
||||
pub session_id: String,
|
||||
pub name: String,
|
||||
pub byte_size: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct TransferBeginResult {
|
||||
pub transfer_id: String,
|
||||
pub chunk_size: u32,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct TransferChunkParams {
|
||||
pub transfer_id: String,
|
||||
pub offset: u64,
|
||||
pub data_base64: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct TransferChunkResult {
|
||||
pub next_offset: u64,
|
||||
#[serde(default)]
|
||||
pub eof: bool,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub data_base64: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct TransferIdParams {
|
||||
pub transfer_id: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct TransferReadyResult {
|
||||
pub transfer_id: String,
|
||||
pub byte_size: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct TransferReleaseResult {
|
||||
pub released: bool,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn upload_agent_shape_omits_daemon_path_and_uses_ref_wire_name() {
|
||||
let value = serde_json::to_value(UploadParams {
|
||||
session_id: "s1".into(),
|
||||
ref_: Some("@e3".into()),
|
||||
selector: None,
|
||||
tab_id: None,
|
||||
files: vec![UploadFile {
|
||||
transfer_id: "tr_1".into(),
|
||||
name: "image.png".into(),
|
||||
staged_path: None,
|
||||
}],
|
||||
timeout_ms: None,
|
||||
})
|
||||
.unwrap();
|
||||
assert_eq!(value["ref"], "@e3");
|
||||
assert!(value["files"][0].get("staged_path").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn download_agent_shape_omits_daemon_path() {
|
||||
let value = serde_json::to_value(DownloadParams {
|
||||
session_id: "s1".into(),
|
||||
ref_: None,
|
||||
selector: Some("#export".into()),
|
||||
tab_id: None,
|
||||
timeout_ms: None,
|
||||
browser_relative_dir: None,
|
||||
max_byte_size: None,
|
||||
})
|
||||
.unwrap();
|
||||
assert!(value.get("browser_relative_dir").is_none());
|
||||
assert!(value.get("max_byte_size").is_none());
|
||||
}
|
||||
}
|
||||
@@ -3,6 +3,7 @@
|
||||
pub mod console;
|
||||
pub mod dialog;
|
||||
pub mod emulate;
|
||||
pub mod file_transfer;
|
||||
pub mod human_loop;
|
||||
pub mod interaction;
|
||||
pub mod navigation;
|
||||
@@ -21,6 +22,7 @@ pub mod window;
|
||||
pub use console::*;
|
||||
pub use dialog::*;
|
||||
pub use emulate::*;
|
||||
pub use file_transfer::*;
|
||||
pub use human_loop::*;
|
||||
pub use interaction::*;
|
||||
pub use navigation::*;
|
||||
|
||||
@@ -58,8 +58,9 @@ pub struct SnapshotResult {
|
||||
/// for one tab plus a fresh `@e<N>` ref-store on the extension side.
|
||||
///
|
||||
/// Unlike `tool.snapshot`, this path may run bounded perception probes
|
||||
/// such as hover-surface discovery. It must not submit input, click,
|
||||
/// navigate, or otherwise commit page state.
|
||||
/// such as hover-surface discovery, but only when `probe_hover` asks for
|
||||
/// them. It must not submit input, click, navigate, or otherwise commit
|
||||
/// page state.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct ObserveParams {
|
||||
pub session_id: String,
|
||||
@@ -77,6 +78,16 @@ pub struct ObserveParams {
|
||||
/// Include implementation diagnostics for conditional surface probes.
|
||||
#[serde(default)]
|
||||
pub debug_surfaces: bool,
|
||||
/// Opt in to active hover probing: the extension moves the real cursor
|
||||
/// over a bounded set of controls to discover hover-only menus and
|
||||
/// tooltips.
|
||||
///
|
||||
/// Off by default. Probing costs seconds of wall clock, mutates the live
|
||||
/// page, and pays off only on pages whose content is genuinely hidden
|
||||
/// behind hover. Request it when a static observation looks like it is
|
||||
/// missing hover-revealed structure.
|
||||
#[serde(default)]
|
||||
pub probe_hover: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
@@ -102,6 +113,17 @@ pub struct ObserveDebug {
|
||||
pub surface_probes: Vec<SurfaceProbeDebug>,
|
||||
}
|
||||
|
||||
/// Reports what active hover probing did during an observation, so the agent
|
||||
/// can judge how far the live page may have drifted from the returned text.
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct HoverProbeReport {
|
||||
/// The real cursor was moved over page content during this observation.
|
||||
pub performed: bool,
|
||||
/// Hovering surfaced content the static tree does not contain, so the page
|
||||
/// reacts to the cursor and may not have fully reverted.
|
||||
pub revealed_content: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct ObserveResult {
|
||||
/// Semantic VOM observation text. Refs are rendered as `@e<N>` so
|
||||
@@ -118,6 +140,9 @@ pub struct ObserveResult {
|
||||
pub truncated: bool,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub dialogs: Vec<JavaScriptDialogInfo>,
|
||||
/// Present only when `probe_hover` was requested and probing actually ran.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub hover_probe: Option<HoverProbeReport>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub debug: Option<ObserveDebug>,
|
||||
}
|
||||
@@ -308,6 +333,7 @@ mod tests {
|
||||
tab_id: 42,
|
||||
truncated: false,
|
||||
dialogs: Vec::new(),
|
||||
hover_probe: None,
|
||||
debug: None,
|
||||
};
|
||||
let v = serde_json::to_value(&r).unwrap();
|
||||
@@ -315,4 +341,30 @@ mod tests {
|
||||
let round: ObserveResult = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, r);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn observe_params_default_to_no_hover_probing() {
|
||||
let params: ObserveParams =
|
||||
serde_json::from_value(serde_json::json!({ "session_id": "s1" })).unwrap();
|
||||
assert!(!params.probe_hover);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn observe_result_round_trips_with_hover_probe_report() {
|
||||
let r = ObserveResult {
|
||||
text: "@vom 1\n".into(),
|
||||
ref_count: 0,
|
||||
tab_id: 42,
|
||||
truncated: false,
|
||||
dialogs: Vec::new(),
|
||||
hover_probe: Some(HoverProbeReport {
|
||||
performed: true,
|
||||
revealed_content: true,
|
||||
}),
|
||||
debug: None,
|
||||
};
|
||||
let v = serde_json::to_value(&r).unwrap();
|
||||
let round: ObserveResult = serde_json::from_value(v).unwrap();
|
||||
assert_eq!(round, r);
|
||||
}
|
||||
}
|
||||
|
||||
+10
-1
@@ -43,7 +43,7 @@ Key modules:
|
||||
|
||||
### bsk daemon (same binary: `bsk daemon`)
|
||||
|
||||
- Listens on loopback WebSocket (default **52800**) for extensions.
|
||||
- Listens on loopback WebSocket (default **52800**, configurable in the extension popup) for extensions.
|
||||
- Validates `Origin: chrome-extension://…` on handshake.
|
||||
- Maintains `browsers` (connected extensions) and `sessions` (Agent Window bindings).
|
||||
- **Per-session queue** serializes tool calls targeting one session.
|
||||
@@ -143,6 +143,15 @@ flowchart LR
|
||||
- No credential storage in bsk — cookies stay in the user's browser profile.
|
||||
- `evaluate` restricted to Agent Window tabs in sandbox mode.
|
||||
|
||||
### File-transfer boundary
|
||||
|
||||
- The invoking agent/harness decides whether a transfer is authorized and supplies the task-local source or destination path.
|
||||
- The CLI is the only component that reads an upload source or writes the final download destination. Before browser dispatch it owns rollback of partially staged uploads; after dispatch, ownership moves to the session because a transport timeout cannot prove that Chrome did not attach the file. Download output becomes visible through one atomic commit, and replacement is opt-in without a pre-delete window. The extension never receives either agent-facing path.
|
||||
- The daemon is the authority for storage capabilities and limits. It issues opaque session-scoped transfer IDs, stages bounded chunks in a private runtime directory, and injects only private staged upload paths. For download it mints one relative Chrome directory capability. Only after validating the reported path, file type, symlink boundary, and authoritative byte limit does it take ownership of browser-file cleanup and import the bytes.
|
||||
- The extension owns only the browser transaction. Both tools resolve one `ResolvedActionTarget` and use that same target for protocol setup and the triggering click. Upload arms Chrome's chooser interception before clicking, then accepts either an exact `Page.fileChooserOpened` input node or an independent probe anchored in the trigger node's own document; one verified input is committed with `DOM.setFileInputFiles`. A non-input picker is rejected immediately as unsupported. Download correlates exact-target CDP intent and `chrome.downloads` filename candidates in either arrival order, claims only one unique match, and never cancels an unclaimed candidate.
|
||||
- Browser-side operations report `effect_state` (`none`, `committed`, or `unknown`), `phase`, and `cleanup_state`. Confirmed success wins over a late cancel; an unknown effect is preserved across timeout or transport loss and must not be retried blindly. A transfer deadline sends cancellation to the extension and keeps the session queue occupied for bounded compensation rather than abandoning an in-flight browser effect.
|
||||
- Download staging is released after CLI commit. Upload staging remains until session teardown because the page may read an attached file only on a later form submission. Remaining staging is released on session stop/browser disconnect and on daemon startup after a crash. BrowserSkill does not inspect content or decide whether a transfer is appropriate.
|
||||
|
||||
## Repository layout
|
||||
|
||||
```
|
||||
|
||||
@@ -0,0 +1,259 @@
|
||||
# Browser capability evaluation corpus
|
||||
|
||||
This directory is a deterministic, local browser evaluation environment for BrowserSkill, DSH,
|
||||
and other command-line agents. Cases, fixture pages, direct `bsk` workflows, and assertions are
|
||||
data-driven and discovered automatically. Adding a case does not require editing a central switch.
|
||||
|
||||
[中文说明](README.zh-CN.md)
|
||||
|
||||
## Design goals
|
||||
|
||||
- Agent-neutral: prompts describe observable page goals so the same cases can run through different
|
||||
agents and adapters.
|
||||
- Deterministic: pages run locally and use stable markers and run-scoped events.
|
||||
- Extensible: manifests, fixture modules, and workflows are auto-discovered.
|
||||
- Reproducible: generated cases record a seed and derive every DOM variation from it.
|
||||
- Honest verification: page-observable results, response markers, and adapter evidence are reported
|
||||
separately. Missing adapter evidence is `unverified`, never silently treated as passed.
|
||||
- Privacy-safe: user badcases must be reduced to synthetic DOM and anonymized data before commit.
|
||||
|
||||
The corpus currently has six stable `core` cases and one seeded `matrix` case. The operation
|
||||
inventory contains 28 browser operations; that count is capability coverage, not the number of
|
||||
cases. Cases can grow independently and many cases may cover the same operation under different
|
||||
page conditions.
|
||||
|
||||
## Quick start
|
||||
|
||||
These commands validate manifests and run unit/integration tests without launching a browser:
|
||||
|
||||
```sh
|
||||
pnpm eval:browser validate
|
||||
pnpm eval:browser list
|
||||
pnpm eval:browser coverage
|
||||
pnpm eval:browser:test
|
||||
```
|
||||
|
||||
Build the repository CLI and exercise the real CLI, daemon, extension, Agent Window, and pages:
|
||||
|
||||
```sh
|
||||
cargo build -p bsk
|
||||
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --suite core --bsk ./target/debug/bsk
|
||||
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --suite matrix --seed 4,7,14 --bsk ./target/debug/bsk
|
||||
```
|
||||
|
||||
Every smoke case starts a fresh browser session and stops it in a `finally` cleanup path. Reports
|
||||
and screenshots are written to the ignored `evals/browser/results/` directory.
|
||||
|
||||
`pnpm run eval:browser -- coverage` is also supported. The CLI intentionally accepts pnpm's leading
|
||||
`--` separator.
|
||||
|
||||
## Suites and selection
|
||||
|
||||
| Suite | Intended use | Default execution |
|
||||
| --- | --- | --- |
|
||||
| `core` | Small stable acceptance set for everyday smoke and CI | Yes |
|
||||
| `matrix` | Seeded DOM, timing, and layout variations | Explicit |
|
||||
| `regression` | Minimized user badcases tied to an issue or fix | Explicit |
|
||||
| `stress` | Higher volume or slower boundary testing | Explicit |
|
||||
| `manual` | Human-dependent capabilities that cannot be deterministic | Explicit |
|
||||
|
||||
`smoke` and `run-agent` default to `core`, so adding a stress or regression case cannot silently
|
||||
slow down existing CI. `list`, `coverage`, and `validate` inspect the full corpus by default.
|
||||
|
||||
```sh
|
||||
pnpm eval:browser list --suite core
|
||||
pnpm eval:browser list --tag form
|
||||
pnpm eval:browser smoke --case form-controls
|
||||
pnpm eval:browser smoke --case form-controls,generated-form
|
||||
pnpm eval:browser smoke --case all
|
||||
```
|
||||
|
||||
Filters from different dimensions are combined. Repeated/comma-separated suites or case ids are
|
||||
alternatives; every requested tag must be present. `--task` remains a backward-compatible alias for
|
||||
`--case`.
|
||||
|
||||
## Directory and discovery model
|
||||
|
||||
```text
|
||||
evals/browser/
|
||||
cases/
|
||||
core/*.case.json
|
||||
matrix/*.case.json
|
||||
regression/<case-id>/
|
||||
<case-id>.case.json
|
||||
<case-id>.fixture.mjs
|
||||
README.md
|
||||
fixtures/pages/*.fixture.mjs
|
||||
schemas/case.schema.json
|
||||
lib/
|
||||
tests/
|
||||
```
|
||||
|
||||
- Any `cases/**/*.case.json` file is loaded as a case manifest.
|
||||
- Any `fixtures/**/*.fixture.mjs` or `cases/**/*.fixture.mjs` file is registered as a fixture.
|
||||
- Manifests are ordered by suite, optional numeric `order`, then id.
|
||||
- `validate` rejects duplicate case ids, duplicate routes, missing fixture routes, unknown
|
||||
operations/actions, invalid workflow variables, and adapter assertions with no evidence producer.
|
||||
|
||||
## Case manifest
|
||||
|
||||
A case owns all information needed by both an agent evaluation and direct CLI smoke:
|
||||
|
||||
```json
|
||||
{
|
||||
"$schema": "../../schemas/case.schema.json",
|
||||
"schemaVersion": 1,
|
||||
"id": "example-case",
|
||||
"title": "Example browser behavior",
|
||||
"order": 10,
|
||||
"suite": "core",
|
||||
"tags": ["form", "keyboard"],
|
||||
"seed": 17,
|
||||
"fixture": { "startPath": "/example" },
|
||||
"prompts": { "en": "Open {url} ...", "zh-CN": "打开 {url}……" },
|
||||
"coverage": ["page.navigate", "interact.press"],
|
||||
"assertions": { "site": [], "response": [], "adapter": [] },
|
||||
"smoke": { "steps": [{ "action": "navigate" }] }
|
||||
}
|
||||
```
|
||||
|
||||
Prompt placeholders are `{url}`, `{baseUrl}`, `{runId}`, and `{seed}`. Agent adapter arguments also
|
||||
support `{prompt}`, `{caseId}` / `{taskId}`, and `{variant}`.
|
||||
|
||||
Assertions have three independent sources:
|
||||
|
||||
- `site`: events emitted by the local page, such as submitted values or a history navigation.
|
||||
- `response`: stable text that must appear in the agent or CLI output.
|
||||
- `adapter`: facts outside page JavaScript, such as a non-empty screenshot or confirmed session
|
||||
cleanup.
|
||||
|
||||
The declarative smoke workflow supports navigation/history, waits, observe/snapshot/HTML,
|
||||
screenshot/console/network, click/hover/fill/select/press, tabs, resize/emulation, and the manual
|
||||
ownership/help operations. A step may `saveAs` its JSON result and later reference values such as
|
||||
`{child.tab_id}`. A step may set `evidence` for an adapter assertion. See existing manifests for
|
||||
complete examples.
|
||||
|
||||
## Adding a normal case
|
||||
|
||||
1. Add a `*.fixture.mjs` module exporting `{ id, routes, render(context) }`. It can live in shared
|
||||
`fixtures/pages/` or beside a self-contained regression case.
|
||||
2. Add a `*.case.json` manifest with prompts, operation coverage, assertions, and smoke steps.
|
||||
3. Make all page events run-scoped using the shared `browserEval.send(...)` client helper.
|
||||
4. Run `pnpm eval:browser validate` and `pnpm eval:browser:test`.
|
||||
5. Run the new case against the real local build with `smoke --case <id>`.
|
||||
|
||||
Prefer one behavior or failure mechanism per case. Stable synthetic markers make failures easier to
|
||||
diagnose than assertions against incidental prose.
|
||||
|
||||
## Turning a user badcase into a regression
|
||||
|
||||
Create a self-contained starting point:
|
||||
|
||||
```sh
|
||||
pnpm eval:browser scaffold reported-timeout \
|
||||
--title "Navigation settles after a late frame" \
|
||||
--source "issue-123"
|
||||
```
|
||||
|
||||
The command creates a manifest, fixture, and reproduction notes under
|
||||
`cases/regression/reported-timeout/`. Then reduce the report to the smallest relevant mechanism:
|
||||
|
||||
- Keep DOM structure, timing, redirects, frames, or browser state that triggers the bug.
|
||||
- Replace names, account data, URLs, tokens, screenshots, and page copy with synthetic values.
|
||||
- Do not commit HAR files, cookies, credentials, production HTML, or proprietary assets.
|
||||
- Record the issue/PR reference, seed when relevant, expected result, and the fixing PR.
|
||||
- Confirm the new case fails on the known-bad build and passes on the fixed build when practical.
|
||||
|
||||
## Seeded matrix cases
|
||||
|
||||
`generated-form` varies label association, nesting depth, hydration delay, decoy controls, field
|
||||
order, and element ids while keeping the task and oracle stable. Inspect a seed without a browser:
|
||||
|
||||
```sh
|
||||
pnpm eval:browser generate --seed user-badcase-42
|
||||
pnpm eval:browser prompt generated-form --seed user-badcase-42 --run-id manual-42
|
||||
pnpm eval:browser smoke --case generated-form --seed user-badcase-42
|
||||
pnpm eval:browser smoke --case generated-form --seed 4,7,14
|
||||
```
|
||||
|
||||
Repeated/comma-separated seeds run as independent results. Reports store every normalized seed, so
|
||||
a failure can be replayed exactly and promoted into a named regression case if it represents a
|
||||
distinct bug.
|
||||
|
||||
## Run any command-line agent
|
||||
|
||||
Copy `agents.example.json` to the ignored `agents.local.json` and adapt its command. Arguments are
|
||||
passed directly to the executable without a shell.
|
||||
|
||||
```sh
|
||||
pnpm eval:browser run-agent \
|
||||
--config evals/browser/agents.local.json \
|
||||
--agent dsh-local \
|
||||
--suite core \
|
||||
--repeat 3
|
||||
```
|
||||
|
||||
An adapter may define `command`, `args`, `cwd`, `env`, `timeoutMs`, `variant`, free-form `metadata`,
|
||||
and optional `metrics.errorPattern` / `metrics.toolCallPattern` regular expressions.
|
||||
|
||||
To compare 28 granular tools with domain tools, create two isolated adapter entries and run both
|
||||
against the same selected cases and seeds:
|
||||
|
||||
```sh
|
||||
pnpm eval:browser run-agent \
|
||||
--config evals/browser/agents.local.json \
|
||||
--agent dsh-granular,dsh-domain \
|
||||
--suite core \
|
||||
--repeat 10
|
||||
```
|
||||
|
||||
Keep browser runs sequential to avoid session contention. Use fresh profiles and alternate adapter
|
||||
order in larger experiments to reduce warm-cache bias.
|
||||
|
||||
## Reading results
|
||||
|
||||
```sh
|
||||
pnpm eval:browser summarize evals/browser/results/*.json
|
||||
```
|
||||
|
||||
The summary groups by adapter and variant and reports pass rate, fully verified rate, process
|
||||
failures, matched error count, average tool calls, and average duration. The raw JSON retains case,
|
||||
suite, tags, seed, prompt, command output, page event count, and individual oracle checks.
|
||||
|
||||
A healthy run has:
|
||||
|
||||
- no execution timeout or non-zero exit;
|
||||
- no `executionError` in direct smoke;
|
||||
- `verification.status` equal to `passed` (or intentionally `passed-with-unverified` for an agent
|
||||
adapter that cannot expose external evidence);
|
||||
- every session cleanup assertion verified;
|
||||
- no unexpected increase in errors, tool calls, or duration versus the chosen baseline.
|
||||
|
||||
## Development baseline
|
||||
|
||||
Use these lanes as commit and PR gates:
|
||||
|
||||
| Change | Required checks |
|
||||
| --- | --- |
|
||||
| Any case, fixture, oracle, or harness change | `pnpm eval:browser:check` |
|
||||
| CLI, daemon, extension, DSH plugin, or browser-operation change | The check above, `cargo build -p bsk`, then real-browser `smoke --suite core` |
|
||||
| DOM discovery, form interaction, timing, or waiting change | The checks above, then `smoke --suite matrix --seed 4,7,14` |
|
||||
| A bug fix with a known reproduction | Add a `regression` case and run that case on the fixed build |
|
||||
|
||||
Before committing this harness, the complete acceptance sequence is:
|
||||
|
||||
```sh
|
||||
pnpm eval:browser:check
|
||||
cargo build -p bsk
|
||||
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --case all --bsk ./target/debug/bsk
|
||||
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --suite matrix --seed 4,7,14 --bsk ./target/debug/bsk
|
||||
```
|
||||
|
||||
All commands must exit zero. The smoke summary must have 100% pass and fully verified rates, zero
|
||||
execution failures/errors, and `bsk session list --json` must be `[]` afterward. Do not weaken an
|
||||
oracle or replace a stable marker merely to make a regression pass; behavior changes should update
|
||||
the case and expected baseline in the same reviewed change.
|
||||
|
||||
The direct smoke lane covers 25 of 28 operations. `tabs.borrow`, `tabs.return`, and
|
||||
`assist.request-help` stay manual because they require a real user tab or human interaction and
|
||||
would make the default suite destructive or non-deterministic.
|
||||
@@ -0,0 +1,197 @@
|
||||
# 浏览器能力测试语料库
|
||||
|
||||
这里是一套供 BrowserSkill、DSH 以及其他命令行 Agent 共用的本地确定性浏览器测试环境。
|
||||
case、测试页面、`bsk` 直连流程和断言都会自动发现;新增 case 不需要修改中央 `switch`。
|
||||
|
||||
[English](README.md)
|
||||
|
||||
## 核心原则
|
||||
|
||||
- 与 Agent 无关:prompt 只描述可观察的页面目标,同一套 case 可以被不同 Agent 和
|
||||
adapter 使用。
|
||||
- 本地且确定:页面只依赖本地服务,使用稳定 marker 和按 run 隔离的事件。
|
||||
- 可扩展:manifest、fixture 和 workflow 都从目录自动发现。
|
||||
- 可复现:生成式页面的所有变化由 seed 决定,报告会保留 seed。
|
||||
- 如实验证:页面事件、回答文本和 adapter 外部证据分别统计;拿不到的证据标为
|
||||
`unverified`,不会冒充通过。
|
||||
- 隐私安全:用户 badcase 必须先缩减为合成 DOM 和匿名数据,再进入仓库。
|
||||
|
||||
当前有 6 个稳定的 `core` case 和 1 个 seed 驱动的 `matrix` case。28 表示浏览器操作能力
|
||||
清单,并不是 case 数量;同一种操作可以在不同页面结构、时序和浏览器状态下由多个 case
|
||||
重复覆盖。
|
||||
|
||||
## 快速使用
|
||||
|
||||
不启动浏览器的校验:
|
||||
|
||||
```sh
|
||||
pnpm eval:browser validate
|
||||
pnpm eval:browser list
|
||||
pnpm eval:browser coverage
|
||||
pnpm eval:browser:test
|
||||
```
|
||||
|
||||
使用仓库当前代码编译 `bsk`,再跑真实 CLI、daemon、扩展、小窗和页面链路:
|
||||
|
||||
```sh
|
||||
cargo build -p bsk
|
||||
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --suite core --bsk ./target/debug/bsk
|
||||
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --suite matrix --seed 4,7,14 --bsk ./target/debug/bsk
|
||||
```
|
||||
|
||||
每个 smoke case 都会创建独立 session,并在 `finally` 中关闭。JSON 报告和截图输出到已被
|
||||
Git 忽略的 `evals/browser/results/`。
|
||||
|
||||
也可以写成 `pnpm run eval:browser -- coverage`;CLI 已兼容 pnpm 传入的前导 `--`。
|
||||
|
||||
## Suite 与筛选
|
||||
|
||||
| Suite | 用途 | 默认执行 |
|
||||
| --- | --- | --- |
|
||||
| `core` | 日常 smoke 和 CI 使用的小而稳定的验收集 | 是 |
|
||||
| `matrix` | seed 驱动的 DOM、时序、布局组合 | 显式选择 |
|
||||
| `regression` | 从用户 badcase 缩减并与 issue/修复关联的回归 | 显式选择 |
|
||||
| `stress` | 更高次数或更慢的边界测试 | 显式选择 |
|
||||
| `manual` | 必须依赖真人或真实用户标签页的能力 | 显式选择 |
|
||||
|
||||
`smoke` 和 `run-agent` 不带筛选时只跑 `core`,所以以后新增 stress/badcase 不会突然拖慢
|
||||
已有 CI。`list`、`coverage`、`validate` 默认检查全部语料。
|
||||
|
||||
```sh
|
||||
pnpm eval:browser list --suite core
|
||||
pnpm eval:browser list --tag form
|
||||
pnpm eval:browser smoke --case form-controls
|
||||
pnpm eval:browser smoke --case form-controls,generated-form
|
||||
pnpm eval:browser smoke --case all
|
||||
```
|
||||
|
||||
不同维度的筛选会叠加;逗号分隔的 suite/case 是“或”,多个 tag 必须全部命中。旧参数
|
||||
`--task` 仍作为 `--case` 的兼容别名。
|
||||
|
||||
## 自动发现结构
|
||||
|
||||
```text
|
||||
evals/browser/
|
||||
cases/
|
||||
core/*.case.json
|
||||
matrix/*.case.json
|
||||
regression/<case-id>/
|
||||
<case-id>.case.json
|
||||
<case-id>.fixture.mjs
|
||||
README.md
|
||||
fixtures/pages/*.fixture.mjs
|
||||
schemas/case.schema.json
|
||||
lib/
|
||||
tests/
|
||||
```
|
||||
|
||||
- `cases/**/*.case.json` 都会作为 case manifest 加载。
|
||||
- `fixtures/**/*.fixture.mjs` 和 `cases/**/*.fixture.mjs` 都会注册为页面 fixture。
|
||||
- 顺序按 suite、可选的数字 `order`、id 排列。
|
||||
- `validate` 会阻止重复 case id、重复路由、缺失 fixture、未知操作/action、无效 workflow
|
||||
变量,以及没有任何步骤产出的 adapter evidence。
|
||||
|
||||
一个 manifest 同时描述:中英文 prompt、能力 coverage、页面事件/回答/adapter 三类断言,以及
|
||||
`bsk` 直连 smoke 的声明式步骤。prompt 可用 `{url}`、`{baseUrl}`、`{runId}`、`{seed}`;
|
||||
workflow 可以用 `saveAs` 保存 JSON 结果,并用 `{child.tab_id}` 之类的变量引用后续步骤。
|
||||
完整写法可直接参考 `cases/core/` 和 `cases/matrix/`。
|
||||
|
||||
## 新增普通 case
|
||||
|
||||
1. 新增导出 `{ id, routes, render(context) }` 的 `*.fixture.mjs`,可放共享 fixture 目录,也可
|
||||
和 regression case 放在一起。
|
||||
2. 新增 `*.case.json`,写好 prompt、operation coverage、断言和 smoke steps。
|
||||
3. 页面使用共享的 `browserEval.send(...)` 上报按 run 隔离的事件。
|
||||
4. 执行 `pnpm eval:browser validate` 和 `pnpm eval:browser:test`。
|
||||
5. 用 `smoke --case <id>` 验证真实本地编译链路。
|
||||
|
||||
建议一个 case 只表达一种行为或故障机制,并使用稳定的合成 marker,避免断言偶然文案。
|
||||
|
||||
## 把用户 badcase 固化成回归
|
||||
|
||||
先生成一个自包含模板:
|
||||
|
||||
```sh
|
||||
pnpm eval:browser scaffold reported-timeout \
|
||||
--title "Navigation settles after a late frame" \
|
||||
--source "issue-123"
|
||||
```
|
||||
|
||||
它会在 `cases/regression/reported-timeout/` 下生成 manifest、fixture 和复现说明。随后只保留
|
||||
真正触发问题的 DOM 结构、时序、跳转、frame 或浏览器状态,并遵守:
|
||||
|
||||
- 所有姓名、账号、网址、token、截图和文案都替换为合成值。
|
||||
- 不提交 HAR、cookie、凭据、生产 HTML 或私有素材。
|
||||
- 记录 issue/PR 来源、相关 seed、期望行为和后续修复 PR。
|
||||
- 条件允许时,证明新 case 在旧版本失败、修复版本通过。
|
||||
|
||||
## Seed 矩阵
|
||||
|
||||
`generated-form` 会确定性改变 label 关联方式、DOM 嵌套、hydration 延迟、干扰控件、字段
|
||||
顺序和 element id,但任务和 oracle 保持不变:
|
||||
|
||||
```sh
|
||||
pnpm eval:browser generate --seed user-badcase-42
|
||||
pnpm eval:browser prompt generated-form --seed user-badcase-42 --run-id manual-42
|
||||
pnpm eval:browser smoke --case generated-form --seed user-badcase-42
|
||||
pnpm eval:browser smoke --case generated-form --seed 4,7,14
|
||||
```
|
||||
|
||||
逗号分隔或重复的 seed 会生成相互独立的结果。任何失败都能按报告中的规范化 seed 精确重放;
|
||||
若它代表独立 bug,再提升为命名 regression case。
|
||||
|
||||
## 测任意 Agent 与比较 28/6 工具
|
||||
|
||||
把 `agents.example.json` 复制为已被 Git 忽略的 `agents.local.json`,分别配置两个隔离的 Agent
|
||||
profile/build。参数以数组直接传给可执行程序,不经过 shell。
|
||||
|
||||
```sh
|
||||
pnpm eval:browser run-agent \
|
||||
--config evals/browser/agents.local.json \
|
||||
--agent dsh-granular,dsh-domain \
|
||||
--suite core \
|
||||
--repeat 10
|
||||
```
|
||||
|
||||
浏览器任务应串行运行,避免 session 竞争;更大实验中使用干净 profile 并轮换 adapter 顺序,
|
||||
降低缓存偏差。
|
||||
|
||||
## 如何看结果
|
||||
|
||||
```sh
|
||||
pnpm eval:browser summarize evals/browser/results/*.json
|
||||
```
|
||||
|
||||
汇总按 adapter/variant 展示通过率、完全验证率、进程失败、错误数、平均工具调用数和耗时。
|
||||
原始 JSON 会保留 case、suite、tag、seed、prompt、命令输出、页面事件数和每条 oracle 结果。
|
||||
|
||||
健康运行应满足:没有 timeout/非零退出,direct smoke 没有 `executionError`,验证状态为
|
||||
`passed`(无法暴露外部证据的 Agent adapter 可明确为 `passed-with-unverified`),session cleanup
|
||||
通过,并且错误数、调用数和耗时没有相对基线异常上涨。
|
||||
|
||||
## 开发基准线
|
||||
|
||||
后续开发统一按下面的门禁执行:
|
||||
|
||||
| 改动范围 | 必跑检查 |
|
||||
| --- | --- |
|
||||
| 任意 case、fixture、oracle 或 harness 改动 | `pnpm eval:browser:check` |
|
||||
| CLI、daemon、扩展、DSH 插件或浏览器操作改动 | 上述检查、`cargo build -p bsk`、真实浏览器 `smoke --suite core` |
|
||||
| DOM 发现、表单交互、时序或等待逻辑改动 | 上述检查,再跑 `smoke --suite matrix --seed 4,7,14` |
|
||||
| 修复已有明确复现的 bug | 新增 `regression` case,并在修复版本单独运行该 case |
|
||||
|
||||
提交这套 harness 前,完整验收命令为:
|
||||
|
||||
```sh
|
||||
pnpm eval:browser:check
|
||||
cargo build -p bsk
|
||||
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --case all --bsk ./target/debug/bsk
|
||||
BSK_AUTO_UPDATE=off pnpm eval:browser smoke --suite matrix --seed 4,7,14 --bsk ./target/debug/bsk
|
||||
```
|
||||
|
||||
所有命令必须以 0 退出;smoke 汇总必须达到 100% pass、100% fully verified、0 execution
|
||||
failure/error,结束后 `bsk session list --json` 必须为 `[]`。不能为了让回归通过而削弱 oracle
|
||||
或替换稳定 marker;如果产品行为确实改变,case 与预期基线必须在同一个受审查改动中更新。
|
||||
|
||||
直连 smoke 自动覆盖 28 项中的 25 项。`tabs.borrow`、`tabs.return`、`assist.request-help` 需要
|
||||
真实用户标签页或真人交互,因此保留在 manual lane,避免默认测试具有破坏性或不确定性。
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"agents": {
|
||||
"dsh-local": {
|
||||
"variant": "granular-28",
|
||||
"command": "dsh",
|
||||
"args": ["--profile", "headless", "{prompt}"],
|
||||
"cwd": "../..",
|
||||
"timeoutMs": 300000,
|
||||
"metadata": {
|
||||
"toolCount": 28
|
||||
},
|
||||
"metrics": {
|
||||
"toolCallPattern": "browser[_-][a-z_-]+"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
# Browser evaluation cases
|
||||
|
||||
Cases are discovered recursively from `*.case.json`; colocated `*.fixture.mjs` files are discovered
|
||||
as fixtures. Use `core` for small deterministic acceptance checks, `matrix` for seeded variations,
|
||||
and `regression/<case-id>/` for privacy-safe minimized user badcases.
|
||||
|
||||
Run `pnpm eval:browser validate` after every case change. See the parent
|
||||
[`README.md`](../README.md) for the manifest contract, workflow actions, scaffolding, and acceptance
|
||||
criteria.
|
||||
@@ -0,0 +1,48 @@
|
||||
{
|
||||
"$schema": "../../schemas/case.schema.json",
|
||||
"schemaVersion": 1,
|
||||
"id": "diagnostics",
|
||||
"title": "Console and network inspection",
|
||||
"order": 50,
|
||||
"suite": "core",
|
||||
"tags": ["console", "network", "diagnostics"],
|
||||
"fixture": { "startPath": "/diagnostics" },
|
||||
"prompts": {
|
||||
"en": "Open {url}. Use browser diagnostics to inspect the page console and recent network activity. Report the console marker and the token returned by the ping request, then close the browser session.",
|
||||
"zh-CN": "打开 {url}。使用浏览器诊断能力检查页面 console 和最近的 network 活动,报告 console marker 以及 ping 请求返回的 token,最后关闭浏览器会话。"
|
||||
},
|
||||
"coverage": [
|
||||
"session.start",
|
||||
"session.stop",
|
||||
"page.navigate",
|
||||
"inspect.console",
|
||||
"inspect.network"
|
||||
],
|
||||
"assertions": {
|
||||
"site": [
|
||||
{
|
||||
"label": "diagnostic ping completed",
|
||||
"type": "network.ping",
|
||||
"minCount": 1
|
||||
}
|
||||
],
|
||||
"response": [
|
||||
{ "label": "reported console marker", "includes": "CONSOLE-61" },
|
||||
{ "label": "reported network token", "includes": "NETWORK-73" }
|
||||
],
|
||||
"adapter": [
|
||||
{ "label": "console inspection was executed", "key": "consoleRead" },
|
||||
{ "label": "network inspection was executed", "key": "networkRead" },
|
||||
{ "label": "browser session was closed", "key": "sessionStopped" }
|
||||
]
|
||||
},
|
||||
"smoke": {
|
||||
"steps": [
|
||||
{ "action": "navigate" },
|
||||
{ "action": "wait", "duration": "300ms" },
|
||||
{ "action": "console", "evidence": "consoleRead" },
|
||||
{ "action": "network", "evidence": "networkRead" },
|
||||
{ "action": "observe" }
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
{
|
||||
"$schema": "../../schemas/case.schema.json",
|
||||
"schemaVersion": 1,
|
||||
"id": "form-controls",
|
||||
"title": "Form, select, keyboard, and screenshot",
|
||||
"order": 20,
|
||||
"suite": "core",
|
||||
"tags": ["form", "keyboard", "screenshot", "select"],
|
||||
"fixture": { "startPath": "/form" },
|
||||
"prompts": {
|
||||
"en": "Open {url}. Fill Text input with \"agent-parity\" and Textarea with \"BrowserSkill works\". Select the visible option \"Two\" from Dropdown. Capture a screenshot of the form, focus Submit, and press Enter to submit. Confirm that the result says \"Received!\", then close the browser session.",
|
||||
"zh-CN": "打开 {url}。把 Text input 填成“agent-parity”,Textarea 填成“BrowserSkill works”,Dropdown 选择可见选项“Two”。截取表单截图,然后聚焦 Submit 并按 Enter 提交。确认结果页出现“Received!”,最后关闭浏览器会话。"
|
||||
},
|
||||
"coverage": [
|
||||
"session.start",
|
||||
"session.stop",
|
||||
"page.navigate",
|
||||
"interact.fill",
|
||||
"interact.select",
|
||||
"interact.press",
|
||||
"inspect.screenshot"
|
||||
],
|
||||
"assertions": {
|
||||
"site": [
|
||||
{
|
||||
"label": "Enter was pressed on Submit",
|
||||
"type": "form.enter_pressed",
|
||||
"minCount": 1
|
||||
},
|
||||
{
|
||||
"label": "form values were submitted",
|
||||
"type": "form.submitted",
|
||||
"where": {
|
||||
"data.text": "agent-parity",
|
||||
"data.notes": "BrowserSkill works",
|
||||
"data.choice": "two"
|
||||
},
|
||||
"minCount": 1
|
||||
},
|
||||
{
|
||||
"label": "result page was displayed",
|
||||
"type": "page.shown",
|
||||
"where": { "data.path": "/result" },
|
||||
"minCount": 1
|
||||
}
|
||||
],
|
||||
"response": [{ "label": "confirmed result", "includes": "Received!" }],
|
||||
"adapter": [
|
||||
{ "label": "screenshot artifact was created", "key": "screenshotCreated" },
|
||||
{ "label": "browser session was closed", "key": "sessionStopped" }
|
||||
]
|
||||
},
|
||||
"smoke": {
|
||||
"steps": [
|
||||
{ "action": "navigate" },
|
||||
{
|
||||
"action": "fill",
|
||||
"selector": "#text-input",
|
||||
"value": "agent-parity"
|
||||
},
|
||||
{
|
||||
"action": "fill",
|
||||
"selector": "#notes",
|
||||
"value": "BrowserSkill works"
|
||||
},
|
||||
{ "action": "select", "selector": "#choice", "values": ["two"] },
|
||||
{
|
||||
"action": "screenshot",
|
||||
"artifactName": "form.png",
|
||||
"evidence": "screenshotCreated"
|
||||
},
|
||||
{ "action": "press", "key": "Enter", "selector": "#submit" },
|
||||
{
|
||||
"action": "wait-site-event",
|
||||
"type": "page.shown",
|
||||
"where": { "path": "/result" }
|
||||
},
|
||||
{ "action": "wait", "duration": "200ms" },
|
||||
{ "action": "observe" }
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
{
|
||||
"$schema": "../../schemas/case.schema.json",
|
||||
"schemaVersion": 1,
|
||||
"id": "hover-inspect",
|
||||
"title": "Hover, local HTML, and screenshot",
|
||||
"order": 30,
|
||||
"suite": "core",
|
||||
"tags": ["hover", "html", "screenshot"],
|
||||
"fixture": { "startPath": "/hover-inspect" },
|
||||
"prompts": {
|
||||
"en": "Open {url}. Hover over Account, then choose Profile beta from the revealed menu. Inspect the local HTML around the Inspection card and capture a screenshot. Report both marker codes you find, then close the browser session.",
|
||||
"zh-CN": "打开 {url}。把鼠标悬停在“Account”上,然后点击展开菜单中的“Profile beta”。检查 Inspection card 附近的局部 HTML,并截取页面截图。报告找到的两个标记代码,最后关闭浏览器会话。"
|
||||
},
|
||||
"coverage": [
|
||||
"session.start",
|
||||
"session.stop",
|
||||
"page.navigate",
|
||||
"interact.hover",
|
||||
"interact.click",
|
||||
"inspect.html",
|
||||
"inspect.screenshot"
|
||||
],
|
||||
"assertions": {
|
||||
"site": [
|
||||
{
|
||||
"label": "Account menu was hovered",
|
||||
"type": "hover.menu_opened",
|
||||
"minCount": 1
|
||||
},
|
||||
{
|
||||
"label": "Profile beta was selected",
|
||||
"type": "hover.profile_clicked",
|
||||
"minCount": 1
|
||||
}
|
||||
],
|
||||
"response": [
|
||||
{ "label": "reported hover marker", "includes": "HOVER-17" },
|
||||
{ "label": "reported HTML marker", "includes": "INSPECT-27" }
|
||||
],
|
||||
"adapter": [
|
||||
{ "label": "HTML inspection was executed", "key": "htmlInspected" },
|
||||
{ "label": "screenshot artifact was created", "key": "screenshotCreated" },
|
||||
{ "label": "browser session was closed", "key": "sessionStopped" }
|
||||
]
|
||||
},
|
||||
"smoke": {
|
||||
"steps": [
|
||||
{ "action": "navigate" },
|
||||
{ "action": "hover", "selector": "#account-menu" },
|
||||
{ "action": "click", "selector": "#profile-link" },
|
||||
{ "action": "html", "evidence": "htmlInspected" },
|
||||
{
|
||||
"action": "screenshot",
|
||||
"artifactName": "hover-inspect.png",
|
||||
"evidence": "screenshotCreated"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
{
|
||||
"$schema": "../../schemas/case.schema.json",
|
||||
"schemaVersion": 1,
|
||||
"id": "mobile-emulation",
|
||||
"title": "Mobile emulation",
|
||||
"order": 60,
|
||||
"suite": "core",
|
||||
"tags": ["emulation", "mobile", "viewport"],
|
||||
"fixture": { "startPath": "/responsive" },
|
||||
"prompts": {
|
||||
"en": "Open {url}, emulate an iPhone 14, and report the viewport marker shown on the page. Close the browser session when finished.",
|
||||
"zh-CN": "打开 {url},模拟 iPhone 14,并报告页面显示的 viewport marker。完成后关闭浏览器会话。"
|
||||
},
|
||||
"coverage": ["session.start", "session.stop", "page.navigate", "assist.emulate"],
|
||||
"assertions": {
|
||||
"site": [
|
||||
{
|
||||
"label": "iPhone-sized viewport was observed",
|
||||
"type": "viewport.sample",
|
||||
"where": { "data.width": 390, "data.height": 844 },
|
||||
"minCount": 1
|
||||
}
|
||||
],
|
||||
"response": [{ "label": "reported viewport marker", "includes": "VIEWPORT-84" }],
|
||||
"adapter": [
|
||||
{ "label": "device emulation was executed", "key": "emulatedDevice" },
|
||||
{ "label": "browser session was closed", "key": "sessionStopped" }
|
||||
]
|
||||
},
|
||||
"smoke": {
|
||||
"steps": [
|
||||
{ "action": "navigate" },
|
||||
{ "action": "resize", "width": 900, "height": 700 },
|
||||
{
|
||||
"action": "emulate",
|
||||
"device": "iphone-14",
|
||||
"evidence": "emulatedDevice"
|
||||
},
|
||||
{ "action": "wait", "duration": "300ms" },
|
||||
{ "action": "html" }
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
{
|
||||
"$schema": "../../schemas/case.schema.json",
|
||||
"schemaVersion": 1,
|
||||
"id": "navigation-history",
|
||||
"title": "Navigation, history, and reload",
|
||||
"order": 10,
|
||||
"suite": "core",
|
||||
"tags": ["navigation", "history", "reload"],
|
||||
"fixture": { "startPath": "/navigation/start" },
|
||||
"prompts": {
|
||||
"en": "Open {url}. Read the main heading, follow the Details link, and report the code shown there. Then go back, go forward, reload the page, and close the browser session. Use only the browser capability for page operations.",
|
||||
"zh-CN": "打开 {url}。读取页面主标题,点击“详情”,并报告详情页上的代码。然后依次后退、前进、刷新页面,最后关闭浏览器会话。页面操作只使用浏览器能力。"
|
||||
},
|
||||
"coverage": [
|
||||
"session.start",
|
||||
"session.stop",
|
||||
"page.navigate",
|
||||
"page.back",
|
||||
"page.forward",
|
||||
"page.reload",
|
||||
"interact.click"
|
||||
],
|
||||
"assertions": {
|
||||
"site": [
|
||||
{
|
||||
"label": "details link was clicked",
|
||||
"type": "navigation.detail.clicked",
|
||||
"minCount": 1
|
||||
},
|
||||
{
|
||||
"label": "back returned to the start page",
|
||||
"type": "page.shown",
|
||||
"where": { "data.path": "/navigation/start" },
|
||||
"minCount": 2
|
||||
},
|
||||
{
|
||||
"label": "forward returned to the detail page",
|
||||
"type": "page.shown",
|
||||
"where": { "data.path": "/navigation/detail" },
|
||||
"minCount": 2
|
||||
},
|
||||
{
|
||||
"label": "detail page was reloaded",
|
||||
"type": "page.shown",
|
||||
"where": {
|
||||
"data.path": "/navigation/detail",
|
||||
"data.navigationType": "reload"
|
||||
},
|
||||
"minCount": 1
|
||||
}
|
||||
],
|
||||
"response": [{ "label": "reported detail code", "includes": "NAV-42" }],
|
||||
"adapter": [{ "label": "browser session was closed", "key": "sessionStopped" }]
|
||||
},
|
||||
"smoke": {
|
||||
"steps": [
|
||||
{ "action": "navigate" },
|
||||
{ "action": "observe" },
|
||||
{ "action": "snapshot" },
|
||||
{ "action": "click", "selector": "#details-link" },
|
||||
{ "action": "html" },
|
||||
{ "action": "back" },
|
||||
{ "action": "forward" },
|
||||
{ "action": "reload" }
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
{
|
||||
"$schema": "../../schemas/case.schema.json",
|
||||
"schemaVersion": 1,
|
||||
"id": "tabs",
|
||||
"title": "Create, select, and close a tab",
|
||||
"order": 40,
|
||||
"suite": "core",
|
||||
"tags": ["tabs", "ownership", "lifecycle"],
|
||||
"fixture": { "startPath": "/tabs" },
|
||||
"prompts": {
|
||||
"en": "Open {url}. Create a new browser tab at {baseUrl}/tabs/child?run={runId}, read its token, click \"Mark child visited\", close the child tab, return to the original tab, and close the browser session.",
|
||||
"zh-CN": "打开 {url}。新建一个浏览器标签页并访问 {baseUrl}/tabs/child?run={runId},读取其中的 token,点击“Mark child visited”,关闭子标签页,回到原标签页,最后关闭浏览器会话。"
|
||||
},
|
||||
"coverage": [
|
||||
"session.start",
|
||||
"session.stop",
|
||||
"page.navigate",
|
||||
"interact.click",
|
||||
"tabs.create",
|
||||
"tabs.select",
|
||||
"tabs.close"
|
||||
],
|
||||
"assertions": {
|
||||
"site": [
|
||||
{
|
||||
"label": "child tab page was displayed",
|
||||
"type": "page.shown",
|
||||
"where": { "data.path": "/tabs/child" },
|
||||
"minCount": 1
|
||||
},
|
||||
{
|
||||
"label": "child tab action ran",
|
||||
"type": "tabs.child_marked",
|
||||
"minCount": 1
|
||||
}
|
||||
],
|
||||
"response": [{ "label": "reported child token", "includes": "TAB-29" }],
|
||||
"adapter": [
|
||||
{ "label": "child tab was closed", "key": "childTabClosed" },
|
||||
{ "label": "browser session was closed", "key": "sessionStopped" }
|
||||
]
|
||||
},
|
||||
"smoke": {
|
||||
"steps": [
|
||||
{ "action": "navigate", "saveAs": "original" },
|
||||
{ "action": "tab-list" },
|
||||
{
|
||||
"action": "tab-create",
|
||||
"url": "{baseUrl}/tabs/child?run={runId}",
|
||||
"saveAs": "child"
|
||||
},
|
||||
{
|
||||
"action": "navigate",
|
||||
"url": "{baseUrl}/tabs/child?run={runId}",
|
||||
"tabId": "{child.tab_id}"
|
||||
},
|
||||
{
|
||||
"action": "click",
|
||||
"selector": "#mark-child",
|
||||
"tabId": "{child.tab_id}"
|
||||
},
|
||||
{ "action": "html", "tabId": "{child.tab_id}" },
|
||||
{
|
||||
"action": "tab-close",
|
||||
"tabId": "{child.tab_id}",
|
||||
"evidence": "childTabClosed"
|
||||
},
|
||||
{ "action": "tab-select", "tabId": "{original.tab_id}" }
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,5 @@
|
||||
# Seeded matrix cases
|
||||
|
||||
Matrix cases vary page structure or timing deterministically from a recorded seed. The task,
|
||||
assertions, and success marker remain stable so a failed seed can be replayed exactly. A seed that
|
||||
exposes a distinct product bug should be minimized and promoted to a named regression case.
|
||||
@@ -0,0 +1,77 @@
|
||||
{
|
||||
"$schema": "../../schemas/case.schema.json",
|
||||
"schemaVersion": 1,
|
||||
"id": "generated-form",
|
||||
"title": "Seeded form structure and hydration variants",
|
||||
"order": 10,
|
||||
"suite": "matrix",
|
||||
"tags": ["dynamic-dom", "form", "hydration", "seeded"],
|
||||
"seed": 20260827,
|
||||
"fixture": { "startPath": "/matrix/generated-form" },
|
||||
"prompts": {
|
||||
"en": "Open {url}. Wait for the seeded form to finish hydrating. Fill Text input with \"seeded-agent\", select the visible option \"Two\", submit with Enter, and report the result marker and seed. Close the browser session.",
|
||||
"zh-CN": "打开 {url}。等待 seeded form 完成 hydration,把 Text input 填成“seeded-agent”,选择可见选项“Two”,用 Enter 提交,并报告结果 marker 和 seed。最后关闭浏览器会话。"
|
||||
},
|
||||
"coverage": [
|
||||
"session.start",
|
||||
"session.stop",
|
||||
"page.navigate",
|
||||
"page.wait",
|
||||
"inspect.observe",
|
||||
"interact.fill",
|
||||
"interact.select",
|
||||
"interact.press"
|
||||
],
|
||||
"assertions": {
|
||||
"site": [
|
||||
{ "label": "generated form hydrated", "type": "matrix.ready", "minCount": 1 },
|
||||
{
|
||||
"label": "Enter was pressed on generated form",
|
||||
"type": "matrix.enter_pressed",
|
||||
"minCount": 1
|
||||
},
|
||||
{
|
||||
"label": "generated values were submitted",
|
||||
"type": "matrix.submitted",
|
||||
"where": { "data.text": "seeded-agent", "data.choice": "two" },
|
||||
"minCount": 1
|
||||
},
|
||||
{
|
||||
"label": "matrix result page was displayed",
|
||||
"type": "page.shown",
|
||||
"where": { "data.path": "/matrix/generated-result" },
|
||||
"minCount": 1
|
||||
}
|
||||
],
|
||||
"response": [
|
||||
{ "label": "reported matrix result", "includes": "MATRIX-OK" },
|
||||
{ "label": "reported selected seed", "includes": "SEED-" }
|
||||
],
|
||||
"adapter": [{ "label": "browser session was closed", "key": "sessionStopped" }]
|
||||
},
|
||||
"smoke": {
|
||||
"steps": [
|
||||
{ "action": "navigate" },
|
||||
{
|
||||
"action": "wait-site-event",
|
||||
"type": "matrix.ready",
|
||||
"timeoutMs": 5000
|
||||
},
|
||||
{ "action": "observe" },
|
||||
{ "action": "fill", "selector": "[name=text]", "value": "seeded-agent" },
|
||||
{ "action": "select", "selector": "[name=choice]", "values": ["two"] },
|
||||
{
|
||||
"action": "press",
|
||||
"key": "Enter",
|
||||
"selector": "button[type=submit]"
|
||||
},
|
||||
{
|
||||
"action": "wait-site-event",
|
||||
"type": "page.shown",
|
||||
"where": { "path": "/matrix/generated-result" }
|
||||
},
|
||||
{ "action": "wait", "duration": "200ms" },
|
||||
{ "action": "observe" }
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
# Regression cases
|
||||
|
||||
Each user-reported badcase should live in its own directory with a manifest, synthetic fixture, and
|
||||
README describing the symptom, minimal reproduction, expected behavior, source reference, and fix.
|
||||
Never commit credentials, cookies, HAR files, production HTML, private screenshots, or proprietary
|
||||
assets.
|
||||
|
||||
Create a starting point with:
|
||||
|
||||
```sh
|
||||
pnpm eval:browser scaffold <case-id> --title "..." --source "issue-or-pr"
|
||||
```
|
||||
@@ -0,0 +1,467 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
|
||||
import { loadAgentConfig, runCommandTask } from "./lib/agent-runner.mjs";
|
||||
import { runBskSmokeTask } from "./lib/bsk-runner.mjs";
|
||||
import { buildOperationCoverage } from "./lib/coverage.mjs";
|
||||
import { loadFixtureRegistry } from "./lib/fixture-registry.mjs";
|
||||
import { verifyTask } from "./lib/oracle.mjs";
|
||||
import { resultsDirectory } from "./lib/paths.mjs";
|
||||
import { repositorySummary, validateRepositoryCases } from "./lib/repository-validation.mjs";
|
||||
import { scaffoldCase } from "./lib/scaffold-case.mjs";
|
||||
import { describeSeed, normalizeSeed } from "./lib/seeded-generator.mjs";
|
||||
import { createEvalServer } from "./lib/server.mjs";
|
||||
import { printSummary, summarizeReports } from "./lib/summary.mjs";
|
||||
import { allTasks, getTask, listTasks, renderPrompt } from "./lib/tasks.mjs";
|
||||
|
||||
const booleanOptions = new Set(["help", "json"]);
|
||||
|
||||
function parseArguments(argv) {
|
||||
const normalized = argv[0] === "--" ? argv.slice(1) : argv;
|
||||
let [command, ...rest] = normalized;
|
||||
const options = new Map();
|
||||
const positionals = [];
|
||||
if (command === "--help" || command === "-h") {
|
||||
command = undefined;
|
||||
options.set("help", [true]);
|
||||
}
|
||||
for (let index = 0; index < rest.length; index += 1) {
|
||||
const argument = rest[index];
|
||||
if (!argument.startsWith("--")) {
|
||||
positionals.push(argument);
|
||||
continue;
|
||||
}
|
||||
const equals = argument.indexOf("=");
|
||||
const name = argument.slice(2, equals === -1 ? undefined : equals);
|
||||
let value = equals === -1 ? undefined : argument.slice(equals + 1);
|
||||
if (value === undefined && !booleanOptions.has(name)) {
|
||||
const candidate = rest[index + 1];
|
||||
if (candidate === undefined || candidate.startsWith("--")) {
|
||||
throw new Error(`--${name} requires a value`);
|
||||
}
|
||||
value = candidate;
|
||||
index += 1;
|
||||
}
|
||||
const values = options.get(name) ?? [];
|
||||
values.push(value ?? true);
|
||||
options.set(name, values);
|
||||
}
|
||||
return { command, options, positionals };
|
||||
}
|
||||
|
||||
function option(parsed, name, fallback) {
|
||||
return parsed.options.get(name)?.at(-1) ?? fallback;
|
||||
}
|
||||
|
||||
function optionList(parsed, name) {
|
||||
return (parsed.options.get(name) ?? [])
|
||||
.flatMap((value) => String(value).split(","))
|
||||
.map((value) => value.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function numericOption(parsed, name, fallback) {
|
||||
const value = Number(option(parsed, name, fallback));
|
||||
if (!Number.isFinite(value) || value < 0) throw new Error(`--${name} must be non-negative`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function seedOptions(parsed, fallback) {
|
||||
const requested = optionList(parsed, "seed");
|
||||
return requested.length > 0 ? unique(requested.map(normalizeSeed)) : [fallback];
|
||||
}
|
||||
|
||||
function seedOption(parsed, fallback) {
|
||||
const seeds = seedOptions(parsed, fallback);
|
||||
if (seeds.length > 1) throw new Error("this command accepts only one --seed value");
|
||||
return seeds[0];
|
||||
}
|
||||
|
||||
function unique(values) {
|
||||
return [...new Set(values)];
|
||||
}
|
||||
|
||||
function selectCases(cases, parsed, { defaultSuite } = {}) {
|
||||
const requestedIds = unique([...optionList(parsed, "case"), ...optionList(parsed, "task")]);
|
||||
const requestedSuites = unique(optionList(parsed, "suite"));
|
||||
const requestedTags = unique(optionList(parsed, "tag"));
|
||||
const hasSelector =
|
||||
requestedIds.length > 0 || requestedSuites.length > 0 || requestedTags.length > 0;
|
||||
|
||||
if (requestedIds.includes("all") && requestedIds.length > 1) {
|
||||
throw new Error("--case all cannot be combined with named cases");
|
||||
}
|
||||
if (requestedSuites.includes("all") && requestedSuites.length > 1) {
|
||||
throw new Error("--suite all cannot be combined with named suites");
|
||||
}
|
||||
|
||||
const caseIds = requestedIds.includes("all") ? [] : requestedIds;
|
||||
const suites = requestedSuites.includes("all")
|
||||
? []
|
||||
: requestedSuites.length > 0
|
||||
? requestedSuites
|
||||
: !hasSelector && defaultSuite
|
||||
? [defaultSuite]
|
||||
: [];
|
||||
|
||||
const knownIds = new Set(cases.map(({ id }) => id));
|
||||
const missingIds = caseIds.filter((id) => !knownIds.has(id));
|
||||
if (missingIds.length > 0) throw new Error(`unknown case(s): ${missingIds.join(", ")}`);
|
||||
const knownSuites = new Set(cases.map(({ suite }) => suite));
|
||||
const missingSuites = suites.filter((suite) => !knownSuites.has(suite));
|
||||
if (missingSuites.length > 0) throw new Error(`unknown suite(s): ${missingSuites.join(", ")}`);
|
||||
const knownTags = new Set(cases.flatMap(({ tags }) => tags));
|
||||
const missingTags = requestedTags.filter((tag) => !knownTags.has(tag));
|
||||
if (missingTags.length > 0) throw new Error(`unknown tag(s): ${missingTags.join(", ")}`);
|
||||
|
||||
const selected = cases.filter(
|
||||
(testCase) =>
|
||||
(caseIds.length === 0 || caseIds.includes(testCase.id)) &&
|
||||
(suites.length === 0 || suites.includes(testCase.suite)) &&
|
||||
requestedTags.every((tag) => testCase.tags.includes(tag)),
|
||||
);
|
||||
if (selected.length === 0) throw new Error("case selectors matched no cases");
|
||||
return selected;
|
||||
}
|
||||
|
||||
function reportFilename(mode) {
|
||||
return `${new Date().toISOString().replaceAll(/[:.]/g, "-")}-${mode}.json`;
|
||||
}
|
||||
|
||||
async function saveReport(report, outputDirectory) {
|
||||
await mkdir(outputDirectory, { recursive: true });
|
||||
const path = join(outputDirectory, reportFilename(report.mode));
|
||||
await writeFile(path, `${JSON.stringify(report, null, 2)}\n`);
|
||||
return path;
|
||||
}
|
||||
|
||||
async function loadValidatedCorpus() {
|
||||
const [cases, fixtureRegistry] = await Promise.all([allTasks(), loadFixtureRegistry()]);
|
||||
const errors = validateRepositoryCases(cases, fixtureRegistry);
|
||||
if (errors.length > 0) throw new Error(`browser eval validation failed:\n${errors.join("\n")}`);
|
||||
return { cases, fixtureRegistry };
|
||||
}
|
||||
|
||||
function selectionMetadata(cases) {
|
||||
return {
|
||||
cases: cases.map(({ id }) => id),
|
||||
suites: unique(cases.map(({ suite }) => suite)),
|
||||
tags: unique(cases.flatMap(({ tags }) => tags)).sort(),
|
||||
};
|
||||
}
|
||||
|
||||
function usage() {
|
||||
console.log(`BrowserSkill local browser evaluation
|
||||
|
||||
Usage:
|
||||
node evals/browser/cli.mjs validate [--json]
|
||||
node evals/browser/cli.mjs list [--suite NAME] [--tag NAME] [--case ID] [--json]
|
||||
node evals/browser/cli.mjs coverage [--suite NAME] [--tag NAME] [--case ID] [--json]
|
||||
node evals/browser/cli.mjs serve [--host 127.0.0.1] [--port 4173]
|
||||
node evals/browser/cli.mjs prompt <case> [--base-url URL] [--run-id ID] [--seed VALUE]
|
||||
node evals/browser/cli.mjs verify <case> --base-url URL --run-id ID [--response TEXT]
|
||||
node evals/browser/cli.mjs smoke [--bsk bsk] [--suite core] [--case ID] [--seed VALUE]
|
||||
node evals/browser/cli.mjs run-agent --config FILE --agent NAME [--suite core] [--case ID]
|
||||
node evals/browser/cli.mjs generate --seed VALUE [--json]
|
||||
node evals/browser/cli.mjs scaffold <case-id> [--suite regression] [--title TEXT]
|
||||
node evals/browser/cli.mjs summarize <report.json> [more reports...]
|
||||
|
||||
Selection:
|
||||
--case ID[,ID] Select case ids; --task is a backward-compatible alias
|
||||
--suite NAME[,NAME] Select suites
|
||||
--tag NAME[,NAME] Require every selected tag
|
||||
--case all Explicitly include every suite (run commands default to core)
|
||||
--seed VALUE[,VALUE] Run one or more reproducible matrix seeds
|
||||
|
||||
run-agent options:
|
||||
--agent NAME[,NAME] One or more entries from the config file
|
||||
--variant NAME Override the report label (does not switch plugin code)
|
||||
--locale en|zh-CN Prompt language (default zh-CN)
|
||||
--repeat N Repeat every selected case (default 1)
|
||||
--timeout MS Per-agent timeout (default 300000)
|
||||
--out DIRECTORY Report/artifact directory
|
||||
`);
|
||||
}
|
||||
|
||||
async function commandValidate(parsed) {
|
||||
const { cases, fixtureRegistry } = await loadValidatedCorpus();
|
||||
const summary = { ok: true, ...repositorySummary(cases, fixtureRegistry) };
|
||||
if (option(parsed, "json", false)) console.log(JSON.stringify(summary, null, 2));
|
||||
else {
|
||||
console.log(`Validated ${summary.cases} cases and ${summary.fixtureRoutes} fixture routes.`);
|
||||
console.log(
|
||||
`Suites: ${Object.entries(summary.suites)
|
||||
.map(([suite, count]) => `${suite}=${count}`)
|
||||
.join(", ")}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function commandList(parsed) {
|
||||
const cases = selectCases(await allTasks(), parsed);
|
||||
const tasks = listTasks(cases);
|
||||
if (option(parsed, "json", false)) console.log(JSON.stringify(tasks, null, 2));
|
||||
else {
|
||||
for (const task of tasks) {
|
||||
console.log(`${task.id}\t${task.suite}\t${task.tags.join(",")}\t${task.title}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function commandCoverage(parsed) {
|
||||
const cases = selectCases(await allTasks(), parsed);
|
||||
const coverage = buildOperationCoverage(cases);
|
||||
if (option(parsed, "json", false)) {
|
||||
console.log(JSON.stringify(coverage, null, 2));
|
||||
} else {
|
||||
console.table(
|
||||
coverage.map(({ operation, agentCases, smokeCases, directSmoke, note }) => ({
|
||||
operation,
|
||||
agent_cases: agentCases.join(", ") || "-",
|
||||
smoke_cases: smokeCases.join(", ") || "-",
|
||||
direct_smoke: directSmoke ? "yes" : "manual",
|
||||
note: note ?? "",
|
||||
})),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function commandServe(parsed) {
|
||||
const { fixtureRegistry } = await loadValidatedCorpus();
|
||||
const server = createEvalServer({
|
||||
host: option(parsed, "host", "127.0.0.1"),
|
||||
port: numericOption(parsed, "port", 4173),
|
||||
fixtureRegistry,
|
||||
});
|
||||
const info = await server.start();
|
||||
console.log(`Browser eval server listening at ${info.baseUrl}`);
|
||||
console.log("Press Ctrl-C to stop.");
|
||||
await new Promise((resolveSignal) => {
|
||||
process.once("SIGINT", resolveSignal);
|
||||
process.once("SIGTERM", resolveSignal);
|
||||
});
|
||||
await server.stop();
|
||||
}
|
||||
|
||||
async function commandPrompt(parsed) {
|
||||
const cases = await allTasks();
|
||||
const task = getTask(parsed.positionals[0], cases);
|
||||
const prompt = renderPrompt(task, {
|
||||
baseUrl: option(parsed, "base-url", "http://127.0.0.1:4173"),
|
||||
runId: option(parsed, "run-id", randomUUID()),
|
||||
locale: option(parsed, "locale", "zh-CN"),
|
||||
seed: seedOption(parsed, task.seed),
|
||||
});
|
||||
console.log(prompt);
|
||||
}
|
||||
|
||||
async function commandVerify(parsed) {
|
||||
const task = getTask(parsed.positionals[0], await allTasks());
|
||||
const baseUrl = option(parsed, "base-url");
|
||||
const runId = option(parsed, "run-id");
|
||||
if (!baseUrl || !runId) throw new Error("verify requires --base-url and --run-id");
|
||||
let responseText = option(parsed, "response", "");
|
||||
const responseFile = option(parsed, "response-file");
|
||||
if (responseFile) responseText = await readFile(resolve(responseFile), "utf8");
|
||||
const stateResponse = await fetch(`${baseUrl}/api/runs/${encodeURIComponent(runId)}`);
|
||||
if (!stateResponse.ok) throw new Error(`oracle request failed with ${stateResponse.status}`);
|
||||
const state = await stateResponse.json();
|
||||
const verification = verifyTask(task, { events: state.events, responseText });
|
||||
console.log(JSON.stringify(verification, null, 2));
|
||||
if (verification.status === "failed") process.exitCode = 1;
|
||||
}
|
||||
|
||||
async function commandRunAgent(parsed) {
|
||||
const configPath = option(parsed, "config");
|
||||
const agentNames = optionList(parsed, "agent");
|
||||
if (!configPath || agentNames.length === 0) {
|
||||
throw new Error("run-agent requires --config and at least one --agent");
|
||||
}
|
||||
const { cases, fixtureRegistry } = await loadValidatedCorpus();
|
||||
const selected = selectCases(cases, parsed, { defaultSuite: "core" });
|
||||
const repeat = numericOption(parsed, "repeat", 1);
|
||||
if (!Number.isInteger(repeat) || repeat < 1) throw new Error("--repeat must be at least 1");
|
||||
const outputDirectory = resolve(option(parsed, "out", resultsDirectory));
|
||||
const server = createEvalServer({ fixtureRegistry });
|
||||
const serverInfo = await server.start();
|
||||
const results = [];
|
||||
const startedAt = new Date().toISOString();
|
||||
try {
|
||||
for (const agentName of agentNames) {
|
||||
const { adapter, configDirectory } = await loadAgentConfig(configPath, agentName);
|
||||
for (let iteration = 1; iteration <= repeat; iteration += 1) {
|
||||
for (const task of selected) {
|
||||
for (const seed of seedOptions(parsed, task.seed)) {
|
||||
const seedLabel = seed === undefined ? "" : ` seed=${seed}`;
|
||||
console.log(`[run] ${agentName} ${task.id}${seedLabel} (${iteration}/${repeat})`);
|
||||
const result = await runCommandTask({
|
||||
server,
|
||||
serverInfo,
|
||||
task,
|
||||
adapter,
|
||||
adapterName: agentName,
|
||||
configDirectory,
|
||||
iteration,
|
||||
locale: option(parsed, "locale", "zh-CN"),
|
||||
seed,
|
||||
timeoutMs: numericOption(parsed, "timeout", 300_000),
|
||||
variantOverride: option(parsed, "variant"),
|
||||
});
|
||||
results.push(result);
|
||||
console.log(` ${result.verification.status} in ${result.execution.durationMs}ms`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await server.stop();
|
||||
}
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
mode: "agent",
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
selection: selectionMetadata(selected),
|
||||
seeds: unique(results.map(({ seed }) => seed).filter((seed) => seed !== undefined)),
|
||||
results,
|
||||
};
|
||||
const path = await saveReport(report, outputDirectory);
|
||||
printSummary(summarizeReports([report]));
|
||||
console.log(`Report: ${path}`);
|
||||
if (
|
||||
results.some(
|
||||
(result) =>
|
||||
result.verification.status === "failed" ||
|
||||
result.execution.timedOut ||
|
||||
result.execution.exitCode !== 0,
|
||||
)
|
||||
) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
async function commandSmoke(parsed) {
|
||||
const { cases, fixtureRegistry } = await loadValidatedCorpus();
|
||||
const selected = selectCases(cases, parsed, { defaultSuite: "core" });
|
||||
const outputDirectory = resolve(option(parsed, "out", resultsDirectory));
|
||||
const server = createEvalServer({ fixtureRegistry });
|
||||
const serverInfo = await server.start();
|
||||
const results = [];
|
||||
const startedAt = new Date().toISOString();
|
||||
try {
|
||||
for (const task of selected) {
|
||||
for (const seed of seedOptions(parsed, task.seed)) {
|
||||
const runId = `bsk-${task.id}-${randomUUID().slice(0, 8)}`;
|
||||
const seedLabel = seed === undefined ? "" : ` seed=${seed}`;
|
||||
console.log(`[smoke] ${task.id}${seedLabel}`);
|
||||
const result = await runBskSmokeTask({
|
||||
bskCommand: option(parsed, "bsk", "bsk"),
|
||||
outputDirectory,
|
||||
server,
|
||||
serverInfo,
|
||||
task,
|
||||
runId,
|
||||
seed,
|
||||
timeoutMs: numericOption(parsed, "timeout", 60_000),
|
||||
});
|
||||
results.push(result);
|
||||
console.log(
|
||||
` ${result.verification.status}${result.executionError ? `: ${result.executionError}` : ""}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
await server.stop();
|
||||
}
|
||||
const report = {
|
||||
schemaVersion: 1,
|
||||
mode: "bsk-smoke",
|
||||
startedAt,
|
||||
finishedAt: new Date().toISOString(),
|
||||
selection: selectionMetadata(selected),
|
||||
seeds: unique(results.map(({ seed }) => seed).filter((seed) => seed !== undefined)),
|
||||
results,
|
||||
};
|
||||
const path = await saveReport(report, outputDirectory);
|
||||
printSummary(summarizeReports([report]));
|
||||
console.log(`Report: ${path}`);
|
||||
if (results.some((result) => result.executionError || result.verification.status === "failed")) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
async function commandGenerate(parsed) {
|
||||
if (!parsed.options.has("seed")) throw new Error("generate requires --seed");
|
||||
const dimensions = unique(optionList(parsed, "seed").map(normalizeSeed)).map(describeSeed);
|
||||
const output = dimensions.length === 1 ? dimensions[0] : dimensions;
|
||||
if (option(parsed, "json", false)) console.log(JSON.stringify(output, null, 2));
|
||||
else console.table(dimensions);
|
||||
}
|
||||
|
||||
async function commandScaffold(parsed) {
|
||||
const id = parsed.positionals[0];
|
||||
if (!id) throw new Error("scaffold requires a case id");
|
||||
const result = await scaffoldCase({
|
||||
id,
|
||||
title: option(parsed, "title", id),
|
||||
suite: option(parsed, "suite", "regression"),
|
||||
source: option(parsed, "source"),
|
||||
});
|
||||
if (option(parsed, "json", false)) console.log(JSON.stringify(result, null, 2));
|
||||
else {
|
||||
console.log(`Created ${result.directory}`);
|
||||
console.log(
|
||||
"Edit the fixture, prompt, workflow, and assertions, then run eval:browser validate.",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
async function commandSummarize(parsed) {
|
||||
const paths = [...parsed.positionals, ...optionList(parsed, "input")];
|
||||
if (paths.length === 0) throw new Error("summarize requires at least one report path");
|
||||
const reports = await Promise.all(
|
||||
paths.map(async (path) => JSON.parse(await readFile(resolve(path), "utf8"))),
|
||||
);
|
||||
const rows = summarizeReports(reports);
|
||||
if (option(parsed, "json", false)) console.log(JSON.stringify(rows, null, 2));
|
||||
else printSummary(rows);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const parsed = parseArguments(process.argv.slice(2));
|
||||
if (!parsed.command || option(parsed, "help", false)) return usage();
|
||||
switch (parsed.command) {
|
||||
case "validate":
|
||||
return commandValidate(parsed);
|
||||
case "list":
|
||||
return commandList(parsed);
|
||||
case "coverage":
|
||||
return commandCoverage(parsed);
|
||||
case "serve":
|
||||
return commandServe(parsed);
|
||||
case "prompt":
|
||||
return commandPrompt(parsed);
|
||||
case "verify":
|
||||
return commandVerify(parsed);
|
||||
case "run-agent":
|
||||
return commandRunAgent(parsed);
|
||||
case "smoke":
|
||||
return commandSmoke(parsed);
|
||||
case "generate":
|
||||
return commandGenerate(parsed);
|
||||
case "scaffold":
|
||||
return commandScaffold(parsed);
|
||||
case "summarize":
|
||||
return commandSummarize(parsed);
|
||||
default:
|
||||
throw new Error(`unknown command ${JSON.stringify(parsed.command)}`);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(`browser eval: ${error instanceof Error ? error.message : String(error)}`);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -0,0 +1,18 @@
|
||||
import { page } from "../../lib/fixtures.mjs";
|
||||
|
||||
export default {
|
||||
id: "diagnostics",
|
||||
routes: ["/diagnostics"],
|
||||
render() {
|
||||
return page({
|
||||
title: "Diagnostics Lab",
|
||||
body: `<section class="card"><h1>Diagnostics Lab</h1><p>Inspect console and network buffers.</p><p id="network-status" class="muted">Ping pending</p></section>`,
|
||||
script: `
|
||||
console.log("Browser eval console marker: CONSOLE-61");
|
||||
fetch("/api/ping?run=" + encodeURIComponent(browserEval.runId))
|
||||
.then((response) => response.json())
|
||||
.then(({ token }) => { document.querySelector("#network-status").textContent = "Ping completed: " + token; });
|
||||
`,
|
||||
});
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,25 @@
|
||||
import { escapeHtml, page } from "../../lib/fixtures.mjs";
|
||||
|
||||
export default {
|
||||
id: "form",
|
||||
routes: ["/form", "/result"],
|
||||
render({ pathname, runId, query, record }) {
|
||||
if (pathname === "/form") {
|
||||
return page({
|
||||
title: "Browser Eval Form",
|
||||
body: `<section class="card"><h1>Web form</h1><form action="/result" method="get"><input type="hidden" name="run" value="${escapeHtml(runId)}"><label>Text input<input id="text-input" name="text" autocomplete="off"></label><label>Textarea<textarea id="notes" name="notes"></textarea></label><label>Dropdown<select id="choice" name="choice"><option value="one">One</option><option value="two">Two</option><option value="three">Three</option></select></label><button id="submit" type="submit">Submit</button></form></section>`,
|
||||
script: `document.querySelector("#submit").addEventListener("keydown", (event) => { if (event.key === "Enter") browserEval.send("form.enter_pressed"); });`,
|
||||
});
|
||||
}
|
||||
const values = {
|
||||
text: query.get("text") ?? "",
|
||||
notes: query.get("notes") ?? "",
|
||||
choice: query.get("choice") ?? "",
|
||||
};
|
||||
record("form.submitted", values, { path: pathname });
|
||||
return page({
|
||||
title: "Form Result",
|
||||
body: `<section class="card"><h1>Received!</h1><p id="submitted-values" class="marker">${escapeHtml(values.text)} | ${escapeHtml(values.notes)} | ${escapeHtml(values.choice)}</p></section>`,
|
||||
});
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,71 @@
|
||||
import { escapeHtml, page } from "../../lib/fixtures.mjs";
|
||||
import { describeSeed } from "../../lib/seeded-generator.mjs";
|
||||
|
||||
function nest(content, depth) {
|
||||
let nested = content;
|
||||
for (let index = 0; index < depth; index += 1) {
|
||||
nested = `<div data-depth="${index + 1}">${nested}</div>`;
|
||||
}
|
||||
return nested;
|
||||
}
|
||||
|
||||
function textField(dimensions) {
|
||||
const id = `matrix-text-${dimensions.idSuffix}`;
|
||||
const input = `<input id="${id}" name="text" autocomplete="off">`;
|
||||
if (dimensions.labelMode === "wrapped") return `<label>Text input${input}</label>`;
|
||||
if (dimensions.labelMode === "for") return `<label for="${id}">Text input</label>${input}`;
|
||||
return `<span id="${id}-label">Text input</span><input id="${id}" name="text" aria-labelledby="${id}-label" autocomplete="off">`;
|
||||
}
|
||||
|
||||
function formMarkup(dimensions, runId) {
|
||||
const text = textField(dimensions);
|
||||
const select = `<label>Dropdown<select name="choice"><option value="one">One</option><option value="two">Two</option><option value="three">Three</option></select></label>`;
|
||||
const fields = dimensions.fieldOrder === "text-first" ? `${text}${select}` : `${select}${text}`;
|
||||
const decoy =
|
||||
dimensions.decoy === "disabled-input"
|
||||
? `<label>Text input<input disabled value="decoy"></label>`
|
||||
: dimensions.decoy === "duplicate-text"
|
||||
? `<p class="muted">Text input is intentionally repeated in helper copy.</p>`
|
||||
: "";
|
||||
return nest(
|
||||
`<form action="/matrix/generated-result" method="get"><input type="hidden" name="run" value="${escapeHtml(runId)}"><input type="hidden" name="seed" value="${dimensions.seed}">${decoy}${fields}<button type="submit">Submit matrix form</button></form>`,
|
||||
dimensions.nestingDepth,
|
||||
);
|
||||
}
|
||||
|
||||
export default {
|
||||
id: "generated-form",
|
||||
routes: ["/matrix/generated-form", "/matrix/generated-result"],
|
||||
render({ pathname, runId, query, record }) {
|
||||
const dimensions = describeSeed(query.get("seed") ?? 1);
|
||||
if (pathname === "/matrix/generated-form") {
|
||||
const markup = JSON.stringify(formMarkup(dimensions, runId));
|
||||
const dimensionsJson = JSON.stringify(dimensions);
|
||||
return page({
|
||||
title: "Seeded Matrix Form",
|
||||
body: `<section class="card"><h1>Seeded Matrix Form</h1><p class="marker">SEED-${dimensions.seed}</p><div id="matrix-root">Hydrating…</div></section>`,
|
||||
script: `
|
||||
const hydrate = () => {
|
||||
const root = document.querySelector("#matrix-root");
|
||||
root.innerHTML = ${markup};
|
||||
root.querySelector("button[type=submit]").addEventListener("keydown", (event) => {
|
||||
if (event.key === "Enter") browserEval.send("matrix.enter_pressed");
|
||||
});
|
||||
browserEval.send("matrix.ready", ${dimensionsJson});
|
||||
};
|
||||
setTimeout(hydrate, ${dimensions.hydrationDelayMs});
|
||||
`,
|
||||
});
|
||||
}
|
||||
const values = {
|
||||
seed: dimensions.seed,
|
||||
text: query.get("text") ?? "",
|
||||
choice: query.get("choice") ?? "",
|
||||
};
|
||||
record("matrix.submitted", values, { path: pathname });
|
||||
return page({
|
||||
title: "Matrix Result",
|
||||
body: `<section class="card"><h1>Matrix Result</h1><p class="marker">MATRIX-OK</p><p>${escapeHtml(values.text)} | ${escapeHtml(values.choice)} | seed ${values.seed}</p></section>`,
|
||||
});
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,17 @@
|
||||
import { page } from "../../lib/fixtures.mjs";
|
||||
|
||||
export default {
|
||||
id: "hover-inspect",
|
||||
routes: ["/hover-inspect"],
|
||||
render() {
|
||||
return page({
|
||||
title: "Hover and Inspect",
|
||||
body: `<section class="card"><h1>Hover and Inspect</h1><div id="account-menu" tabindex="0"><button id="account-button" type="button">Account</button><div id="hover-panel"><button id="profile-link" type="button">Profile beta</button></div></div><p id="hover-result" class="marker" hidden>HOVER-17</p><article id="inspect-card" class="card" data-build="2026.08" data-marker="INSPECT-27"><h2>Inspection card</h2><p>Local HTML marker: <code>INSPECT-27</code></p></article></section>`,
|
||||
script: `
|
||||
const menu = document.querySelector("#account-menu");
|
||||
menu.addEventListener("mouseenter", () => { menu.classList.add("open"); browserEval.send("hover.menu_opened"); });
|
||||
document.querySelector("#profile-link").addEventListener("click", () => { document.querySelector("#hover-result").hidden = false; browserEval.send("hover.profile_clicked"); });
|
||||
`,
|
||||
});
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,12 @@
|
||||
import { page } from "../../lib/fixtures.mjs";
|
||||
|
||||
export default {
|
||||
id: "fixture-index",
|
||||
routes: ["/"],
|
||||
render() {
|
||||
return page({
|
||||
title: "Browser Eval Fixtures",
|
||||
body: `<section class="card"><h1>Browser Eval Fixtures</h1><p class="muted">Deterministic local pages for agent-neutral browser evaluation.</p></section>`,
|
||||
});
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,19 @@
|
||||
import { page, withRun } from "../../lib/fixtures.mjs";
|
||||
|
||||
export default {
|
||||
id: "navigation",
|
||||
routes: ["/navigation/start", "/navigation/detail"],
|
||||
render({ pathname, runId }) {
|
||||
if (pathname === "/navigation/start") {
|
||||
return page({
|
||||
title: "Navigation Start",
|
||||
body: `<section class="card"><h1>Navigation Start</h1><p>Use the link to open the deterministic detail page.</p><a id="details-link" class="button" href="${withRun("/navigation/detail", runId)}">Details</a></section>`,
|
||||
script: `document.querySelector("#details-link").addEventListener("click", () => browserEval.send("navigation.detail.clicked"));`,
|
||||
});
|
||||
}
|
||||
return page({
|
||||
title: "Navigation Detail",
|
||||
body: `<section class="card"><h1>Navigation Detail</h1><p>Stable detail code:</p><p id="navigation-code" class="marker">NAV-42</p></section>`,
|
||||
});
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,21 @@
|
||||
import { page } from "../../lib/fixtures.mjs";
|
||||
|
||||
export default {
|
||||
id: "responsive",
|
||||
routes: ["/responsive"],
|
||||
render() {
|
||||
return page({
|
||||
title: "Responsive Lab",
|
||||
body: `<section class="card"><h1>Responsive Lab</h1><p id="viewport-marker" class="marker">VIEWPORT-84</p><p id="viewport-value"></p></section>`,
|
||||
script: `
|
||||
const sample = () => {
|
||||
const data = { width: innerWidth, height: innerHeight, touchPoints: navigator.maxTouchPoints, userAgent: navigator.userAgent };
|
||||
document.querySelector("#viewport-value").textContent = data.width + "x" + data.height;
|
||||
browserEval.send("viewport.sample", data);
|
||||
};
|
||||
window.addEventListener("resize", sample);
|
||||
sample();
|
||||
`,
|
||||
});
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,19 @@
|
||||
import { escapeHtml, page, withRun } from "../../lib/fixtures.mjs";
|
||||
|
||||
export default {
|
||||
id: "tabs",
|
||||
routes: ["/tabs", "/tabs/child"],
|
||||
render({ pathname, runId }) {
|
||||
if (pathname === "/tabs") {
|
||||
return page({
|
||||
title: "Tab Lab",
|
||||
body: `<section class="card"><h1>Tab Lab</h1><p>Keep this original tab open while visiting the child page in a new tab.</p><p class="muted">Child URL: <code>${escapeHtml(withRun("/tabs/child", runId))}</code></p></section>`,
|
||||
});
|
||||
}
|
||||
return page({
|
||||
title: "Child Tab",
|
||||
body: `<section class="card"><h1>Child Tab</h1><p id="tab-token" class="marker">TAB-29</p><button id="mark-child" type="button">Mark child visited</button><p id="child-status" hidden>Marked</p></section>`,
|
||||
script: `document.querySelector("#mark-child").addEventListener("click", () => { document.querySelector("#child-status").hidden = false; browserEval.send("tabs.child_marked"); });`,
|
||||
});
|
||||
},
|
||||
};
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user