mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
61
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d05279f991 | ||
|
|
2719e76438 | ||
|
|
dd9fc9edba | ||
|
|
005a6fd534 | ||
|
|
20554508db | ||
|
|
8352c3c67a | ||
|
|
4eedd98c1d | ||
|
|
6aa931b60a | ||
|
|
3df2738280 | ||
|
|
f86e506d47 | ||
|
|
45a246c41b | ||
|
|
6b78211e79 | ||
|
|
eebf334b40 | ||
|
|
a284ffc5ae | ||
|
|
e720f1ffe5 | ||
|
|
2b7e10ca2b | ||
|
|
1ef93463dd | ||
|
|
c7252529f4 | ||
|
|
0b65b4b295 | ||
|
|
94ec7e73fc | ||
|
|
42403e55f2 | ||
|
|
22019cdf07 | ||
|
|
8d7c0aa66c | ||
|
|
a9489282fa | ||
|
|
d4cddd6224 | ||
|
|
8e4de6cf4b | ||
|
|
fea66ce404 | ||
|
|
8bae85127a | ||
|
|
edb149611b | ||
|
|
14c5884cbc | ||
|
|
e7aad9bc16 | ||
|
|
3fb9e9f581 | ||
|
|
9854ee2f45 | ||
|
|
7035e46024 | ||
|
|
51085b960b | ||
|
|
51df868008 | ||
|
|
43ebfab7ba | ||
|
|
ce11325de2 | ||
|
|
4d1dcd122f | ||
|
|
a4e668542d | ||
|
|
52aa356153 | ||
|
|
d1001e14b2 | ||
|
|
eb71ddc1da | ||
|
|
b07b8fec65 | ||
|
|
47070fdc14 | ||
|
|
916229dc62 | ||
|
|
dde707b558 | ||
|
|
e2d07574d7 | ||
|
|
461d204ca3 | ||
|
|
bc5d9610a3 | ||
|
|
2955260d11 | ||
|
|
ce89e95211 | ||
|
|
24faa10659 | ||
|
|
a7bc8bacf6 | ||
|
|
3d050e0f26 | ||
|
|
afdc398730 | ||
|
|
6b96fcca32 | ||
|
|
0771292336 | ||
|
|
54869cabd0 | ||
|
|
2de7c5c6ff | ||
|
|
703fc64399 |
@@ -0,0 +1,105 @@
|
||||
name: Release dsh plugin
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- dsh-plugin-v*
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: release-dsh-plugin-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
PACKAGE_NAME: "@wxg-prc-cpg/browser-skill-dsh-plugin"
|
||||
PACKAGE_DIR: packages/dsh-plugin-browserskill
|
||||
|
||||
jobs:
|
||||
resolve:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
tag: ${{ steps.version.outputs.tag }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Resolve plugin version
|
||||
id: version
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
VERSION="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').version")"
|
||||
else
|
||||
VERSION="${GITHUB_REF_NAME#dsh-plugin-v}"
|
||||
fi
|
||||
VERSION="${VERSION#v}"
|
||||
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
echo "tag=dsh-plugin-v${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
guard:
|
||||
needs: resolve
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Verify tag matches package.json name and version
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
PKG_NAME="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').name")"
|
||||
PKG_VERSION="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').version")"
|
||||
RELEASE_VERSION="${{ needs.resolve.outputs.version }}"
|
||||
if [ "$PKG_NAME" != "$PACKAGE_NAME" ]; then
|
||||
echo "package.json name (${PKG_NAME}) does not match expected ${PACKAGE_NAME}"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$PKG_VERSION" != "$RELEASE_VERSION" ]; then
|
||||
echo "${PACKAGE_DIR}/package.json version (${PKG_VERSION}) does not match release version (${RELEASE_VERSION})"
|
||||
exit 1
|
||||
fi
|
||||
echo "Version guard passed: ${PKG_NAME}@${RELEASE_VERSION}"
|
||||
|
||||
publish:
|
||||
needs: [resolve, guard]
|
||||
runs-on: ubuntu-latest
|
||||
# Secret NPM_TOKEN is stored on the "npm-publish" GitHub Environment.
|
||||
environment: npm-publish
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: pnpm/action-setup@v6
|
||||
|
||||
- uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 22
|
||||
cache: pnpm
|
||||
registry-url: https://registry.npmjs.org
|
||||
scope: "@wxg-prc-cpg"
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Typecheck plugin
|
||||
run: pnpm --filter "${{ env.PACKAGE_NAME }}" typecheck
|
||||
|
||||
- name: Test plugin
|
||||
run: pnpm --filter "${{ env.PACKAGE_NAME }}" test
|
||||
|
||||
- name: Publish to npm
|
||||
working-directory: ${{ env.PACKAGE_DIR }}
|
||||
env:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${NODE_AUTH_TOKEN:-}" ]; then
|
||||
echo "::error::Environment secret NPM_TOKEN is not configured"
|
||||
exit 1
|
||||
fi
|
||||
pnpm publish --access public --no-git-checks
|
||||
echo "Published ${PACKAGE_NAME}@${{ needs.resolve.outputs.version }}"
|
||||
@@ -3,3 +3,6 @@
|
||||
**/.output/**
|
||||
**/target/**
|
||||
**/node_modules/**
|
||||
# Generated client CSS (tailwind compile output); built by
|
||||
# packages/dsh-plugin-browserskill/scripts/build-client-css.mjs.
|
||||
**/dsh-plugin-browserskill/src/client/bsk-ui.nomodule.css
|
||||
|
||||
+9
-5
@@ -34,13 +34,17 @@ where only `extension connected` fails is expected; go to Step 3.
|
||||
## 3. Open the extension install page
|
||||
|
||||
If `extension connected` is `FAIL` (`0 browsers connected`), the user likely
|
||||
has not installed or enabled the browser extension yet. Open the
|
||||
[Chrome Web Store install page](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||
in the user's browser, then tell the user:
|
||||
has not installed or enabled the browser extension yet. Open the install page
|
||||
matching the user's browser —
|
||||
[Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||
for Chrome and other Chromium browsers,
|
||||
[Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg)
|
||||
for Microsoft Edge — then tell the user:
|
||||
|
||||
> Install **BrowserSkill** from the
|
||||
> [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi),
|
||||
> open the popup, and wait until it turns green. Reply when done.
|
||||
> [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||
> (or [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg)
|
||||
> on Edge), open the popup, and wait until it turns green. Reply when done.
|
||||
|
||||
If opening the page fails, give the user the same link. Then run `bsk doctor`
|
||||
once more. All `ok`/`na` → tell the user it's ready.
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
</p>
|
||||
|
||||
**BrowserSkill** connects Cursor, Claude Code, Codex, OpenClaw, CodeBuddy,
|
||||
WorkBuddy, Pi, Hermes Agent, and other shell-capable AI agents to your already logged-in
|
||||
WorkBuddy, Pi, Hermes Agent, DeepSeek Harness, and other AI agents to your already logged-in
|
||||
browser.
|
||||
|
||||
Need the agent to touch a tab you already have open? It must borrow that tab
|
||||
@@ -67,7 +67,8 @@ Set up browser-skill on this machine by following https://raw.githubusercontent.
|
||||
|
||||
<br>
|
||||
|
||||
Install the CLI, then install the extension from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi).
|
||||
Install the CLI, then install the extension from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||
or [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg).
|
||||
|
||||
#### 1. Install the `bsk` CLI
|
||||
|
||||
@@ -91,7 +92,14 @@ bsk --version
|
||||
|
||||
#### 2. Install the browser extension
|
||||
|
||||
Install BrowserSkill from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi).
|
||||
Install BrowserSkill from your browser's store:
|
||||
|
||||
| Browser | Store listing |
|
||||
| --- | --- |
|
||||
| Chrome | [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) |
|
||||
| Microsoft Edge | [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) |
|
||||
|
||||
On other Chromium-based browsers, install the Chrome Web Store build.
|
||||
|
||||
#### 3. Install the skill
|
||||
|
||||
@@ -123,7 +131,8 @@ internal variants and install paths.
|
||||
|
||||
Other shell-capable agent harnesses are supported too. Copy
|
||||
[`skill/SKILL.md`](skill/SKILL.md) into your harness's skills directory as
|
||||
`browser-skill/SKILL.md` to install the skill manually.
|
||||
`browser-skill/SKILL.md` to install the skill manually. DeepSeek Harness uses a
|
||||
dedicated plugin instead — see [DeepSeek Harness plugin](#deepseek-harness-plugin).
|
||||
|
||||
</details>
|
||||
|
||||
@@ -133,6 +142,26 @@ Start a new Agent session and write a prompt that needs the browser, for example
|
||||
/browser-skill open example.com and summarize what is on the page.
|
||||
```
|
||||
|
||||
## DeepSeek Harness plugin
|
||||
|
||||
Using [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`dsh`)?
|
||||
BrowserSkill ships a first-class dsh plugin on npm as
|
||||
[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin).
|
||||
It injects native `browser_*` tools (no shelling out to `bsk`) and a live Web UI
|
||||
overlay of each Agent Window.
|
||||
|
||||
Add it to a dsh profile, then start that profile:
|
||||
|
||||
```sh
|
||||
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
|
||||
dsh --profile web
|
||||
```
|
||||
|
||||
The plugin carries its own copy of the skill, so `bsk install-skill` is not needed
|
||||
for dsh — but the `bsk` CLI and the browser extension are still prerequisites. See
|
||||
the [plugin README](packages/dsh-plugin-browserskill/README.md) for the tool list,
|
||||
configuration, and the observation overlay.
|
||||
|
||||
## How It Works
|
||||
|
||||
BrowserSkill is a local bridge between your agent harness and your browser.
|
||||
@@ -166,7 +195,9 @@ flowchart TB
|
||||
|
||||
The agent never talks to the browser directly. It asks the `bsk` CLI to perform a
|
||||
browser task; the local daemon routes that request to the extension; the
|
||||
extension runs it in an Agent Window.
|
||||
extension runs it in an Agent Window. DeepSeek Harness takes the same path
|
||||
through the [plugin](#deepseek-harness-plugin): the agent calls injected
|
||||
`browser_*` tools, and the plugin invokes `bsk` on its behalf.
|
||||
|
||||
## For Developers
|
||||
|
||||
@@ -176,6 +207,7 @@ The repository is a Cargo + pnpm workspace:
|
||||
- `crates/bsk-protocol` — shared wire types and JSON schemas
|
||||
- `apps/extension` — browser extension
|
||||
- `packages/ui` and `packages/i18n` — shared extension UI support
|
||||
- `packages/dsh-plugin-browserskill` — DeepSeek Harness plugin (`@wxg-prc-cpg/browser-skill-dsh-plugin`)
|
||||
|
||||
## License
|
||||
|
||||
|
||||
+32
-7
@@ -12,7 +12,7 @@
|
||||
<a href="README.md">English</a> · 中文
|
||||
</p>
|
||||
|
||||
**BrowserSkill** 把 Cursor、Claude Code、Codex、OpenClaw、CodeBuddy、WorkBuddy、Pi、Hermes Agent 等支持 Shell 的 AI Agent 连接到你已登录的浏览器。
|
||||
**BrowserSkill** 把 Cursor、Claude Code、Codex、OpenClaw、CodeBuddy、WorkBuddy、Pi、Hermes Agent、DeepSeek Harness 等 AI Agent 连接到你已登录的浏览器。
|
||||
|
||||
需要 Agent 操作你已打开的标签页?必须显式借用该标签,任务结束后归还,其余浏览器窗口不受影响。
|
||||
|
||||
@@ -54,7 +54,8 @@ BrowserSkill 由两个本地运行组件组成:`bsk` CLI/daemon 和浏览器
|
||||
|
||||
<br>
|
||||
|
||||
先安装 CLI,再从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) 安装浏览器扩展。
|
||||
先安装 CLI,再从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||
或 [Edge 加载项商店](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) 安装浏览器扩展。
|
||||
|
||||
#### 1. 安装 `bsk` CLI
|
||||
|
||||
@@ -64,8 +65,11 @@ BrowserSkill 由两个本地运行组件组成:`bsk` CLI/daemon 和浏览器
|
||||
curl -fsSL https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.sh | sh
|
||||
```
|
||||
|
||||
**Windows**:从 [最新 CLI release](https://github.com/Tencent/BrowserSkill/releases/latest)
|
||||
下载 `bsk-v<version>-x86_64-pc-windows-msvc.zip`,解压后将 `bsk.exe` 加入 `PATH`。
|
||||
**Windows**(PowerShell,安装到 `~/.local/bin`):
|
||||
|
||||
```powershell
|
||||
irm https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.ps1 | iex
|
||||
```
|
||||
|
||||
验证二进制:
|
||||
|
||||
@@ -75,7 +79,14 @@ bsk --version
|
||||
|
||||
#### 2. 安装浏览器扩展
|
||||
|
||||
从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) 安装 BrowserSkill。
|
||||
在对应浏览器的商店安装 BrowserSkill:
|
||||
|
||||
| 浏览器 | 商店页面 |
|
||||
| --- | --- |
|
||||
| Chrome | [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) |
|
||||
| Microsoft Edge | [Edge 加载项商店](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) |
|
||||
|
||||
其他基于 Chromium 的浏览器,安装 Chrome Web Store 版本即可。
|
||||
|
||||
#### 3. 安装 skill
|
||||
|
||||
@@ -102,7 +113,7 @@ bsk install-skill
|
||||
|
||||
用 <kbd>Space</kbd> 选择需要安装的 Agent harness,然后按 <kbd>Enter</kbd> 安装 skill。运行 `bsk install-skill --list` 可查看 internal 变体及安装路径。
|
||||
|
||||
其他支持 Shell 的 Agent harness 也可使用 BrowserSkill,但需手动将 [`skill/SKILL.md`](skill/SKILL.md) 复制到对应 skills 目录下的 `browser-skill/SKILL.md`。
|
||||
其他支持 Shell 的 Agent harness 也可使用 BrowserSkill,但需手动将 [`skill/SKILL.md`](skill/SKILL.md) 复制到对应 skills 目录下的 `browser-skill/SKILL.md`。DeepSeek Harness 走独立插件,见 [DeepSeek Harness 插件](#deepseek-harness-插件)。
|
||||
|
||||
</details>
|
||||
|
||||
@@ -112,6 +123,19 @@ bsk install-skill
|
||||
/browser-skill open example.com and summarize what is on the page.
|
||||
```
|
||||
|
||||
## DeepSeek Harness 插件
|
||||
|
||||
在用 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness)(`dsh`)?BrowserSkill 提供了官方 dsh 插件,已发布到 npm:[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)。它会注入原生 `browser_*` 工具(无需再通过 Shell 调用 `bsk`),并在 Web UI 中实时观察每个 Agent Window。
|
||||
|
||||
把它装进某个 dsh profile,然后启动该 profile:
|
||||
|
||||
```sh
|
||||
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
|
||||
dsh --profile web
|
||||
```
|
||||
|
||||
插件自带 skill,所以在 dsh 下无需执行 `bsk install-skill`;但 `bsk` CLI 和浏览器扩展仍是前置条件。工具清单、配置项与观察浮层见[插件 README](packages/dsh-plugin-browserskill/README.md)。
|
||||
|
||||
## 工作原理
|
||||
|
||||
BrowserSkill 是 Agent 运行时与浏览器之间的本地桥接层。
|
||||
@@ -143,7 +167,7 @@ flowchart TB
|
||||
style UserWindows fill:#f8fafc,stroke:#cbd5e1,color:#334155
|
||||
```
|
||||
|
||||
Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;本地 daemon 把请求路由到扩展;扩展在 Agent Window 中执行。
|
||||
Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;本地 daemon 把请求路由到扩展;扩展在 Agent Window 中执行。DeepSeek Harness 走同一条链路,只是经由 [插件](#deepseek-harness-插件):Agent 调用注入的 `browser_*` 工具,由插件代为执行 `bsk`。
|
||||
|
||||
## 面向开发者
|
||||
|
||||
@@ -153,6 +177,7 @@ Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;
|
||||
- `crates/bsk-protocol` — 共享协议类型与 JSON Schema
|
||||
- `apps/extension` — 浏览器扩展
|
||||
- `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持
|
||||
- `packages/dsh-plugin-browserskill` — DeepSeek Harness 插件(`@wxg-prc-cpg/browser-skill-dsh-plugin`)
|
||||
|
||||
## 许可证
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { type CdpDebuggerApi, ChromiumCdp } from "../chromium-cdp";
|
||||
import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "../chromium-cdp";
|
||||
|
||||
function fakeChromeEvent<TArgs extends unknown[]>() {
|
||||
const listeners = new Set<(...args: TArgs) => void>();
|
||||
@@ -14,7 +14,7 @@ function fakeChromeEvent<TArgs extends unknown[]>() {
|
||||
}
|
||||
|
||||
function fakeApi() {
|
||||
const onEvent = fakeChromeEvent<[chrome.debugger.Debuggee, string, unknown]>();
|
||||
const onEvent = fakeChromeEvent<[CdpDebuggee, string, unknown]>();
|
||||
const onDetach = fakeChromeEvent<[chrome.debugger.Debuggee, string]>();
|
||||
const api: CdpDebuggerApi = {
|
||||
attach: vi.fn(async () => {}),
|
||||
@@ -29,6 +29,135 @@ function fakeApi() {
|
||||
}
|
||||
|
||||
describe("ChromiumCdp", () => {
|
||||
it("discovers multiple iframe targets and recursively routes nested OOPIF commands", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
async (
|
||||
target: chrome.debugger.Debuggee & { sessionId?: string },
|
||||
method: string,
|
||||
params?: { frameId?: string },
|
||||
) => {
|
||||
if (method === "DOM.getFrameOwner") {
|
||||
return { backendNodeId: params?.frameId === "nested" ? 300 : 200 };
|
||||
}
|
||||
if (method !== "Page.getFrameTree") return {};
|
||||
if (target.sessionId === "right-session") {
|
||||
return {
|
||||
frameTree: {
|
||||
frame: { id: "right", url: "https://right.test" },
|
||||
childFrames: [
|
||||
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
if (target.sessionId === "nested-session") {
|
||||
return { frameTree: { frame: { id: "nested", url: "https://nested.test" } } };
|
||||
}
|
||||
return {
|
||||
frameTree: {
|
||||
frame: { id: "main", url: "https://app.test" },
|
||||
childFrames: [
|
||||
{ frame: { id: "left", parentId: "main", url: "https://left.test" } },
|
||||
{
|
||||
frame: { id: "right", parentId: "main", url: "https://right.test" },
|
||||
childFrames: [
|
||||
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
},
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
await cdp.ensureAttached(4);
|
||||
onEvent.fire({ tabId: 4 }, "Target.attachedToTarget", { sessionId: "right-session" });
|
||||
onEvent.fire({ tabId: 4 } as chrome.debugger.Debuggee, "Target.attachedToTarget", {
|
||||
sessionId: "nested-session",
|
||||
});
|
||||
|
||||
const graph = await cdp.getFrameGraph(4);
|
||||
|
||||
expect(graph.frames).toHaveLength(4);
|
||||
expect(graph.frames.find((frame) => frame.frameId === "left")?.target).toEqual({ tabId: 4 });
|
||||
expect(graph.frames.find((frame) => frame.frameId === "right")?.target.sessionId).toBe(
|
||||
"right-session",
|
||||
);
|
||||
expect(graph.frames.find((frame) => frame.frameId === "nested")?.target.sessionId).toBe(
|
||||
"nested-session",
|
||||
);
|
||||
expect(graph.frames.find((frame) => frame.frameId === "right")?.ownerBackendNodeId).toBe(200);
|
||||
expect(graph.frames.find((frame) => frame.frameId === "nested")?.ownerBackendNodeId).toBe(300);
|
||||
expect(api.sendCommand).toHaveBeenCalledWith(
|
||||
{ tabId: 4, sessionId: "right-session" },
|
||||
"DOM.getFrameOwner",
|
||||
{ frameId: "nested" },
|
||||
);
|
||||
expect(api.sendCommand).toHaveBeenCalledWith(
|
||||
{ tabId: 4, sessionId: "nested-session" },
|
||||
"Target.setAutoAttach",
|
||||
expect.objectContaining({ flatten: true }),
|
||||
);
|
||||
});
|
||||
|
||||
it("waits for recursively attached iframe targets without a depth limit", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
const depth = 6;
|
||||
const scheduled = new Set<string>();
|
||||
const nestedTree = (index: number): Record<string, unknown> => ({
|
||||
frame: {
|
||||
id: index === 0 ? "main" : `frame-${index}`,
|
||||
...(index > 0 ? { parentId: index === 1 ? "main" : `frame-${index - 1}` } : {}),
|
||||
},
|
||||
...(index < depth ? { childFrames: [nestedTree(index + 1)] } : {}),
|
||||
});
|
||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
async (target: CdpDebuggee, method: string, params?: { frameId?: string }) => {
|
||||
if (method === "Target.setAutoAttach") {
|
||||
const parentIndex = target.sessionId
|
||||
? Number(target.sessionId.replace("session-", ""))
|
||||
: 0;
|
||||
const nextIndex = parentIndex + 1;
|
||||
const key = `${target.sessionId ?? "root"}:${nextIndex}`;
|
||||
if (nextIndex <= depth && !scheduled.has(key)) {
|
||||
scheduled.add(key);
|
||||
setTimeout(() => {
|
||||
onEvent.fire(
|
||||
{ tabId: 4, ...(target.sessionId ? { sessionId: target.sessionId } : {}) },
|
||||
"Target.attachedToTarget",
|
||||
{
|
||||
sessionId: `session-${nextIndex}`,
|
||||
targetInfo: { type: "iframe" },
|
||||
},
|
||||
);
|
||||
}, 5);
|
||||
}
|
||||
return {};
|
||||
}
|
||||
if (method === "Page.getFrameTree") {
|
||||
if (!target.sessionId) return { frameTree: nestedTree(0) };
|
||||
const index = Number(target.sessionId.replace("session-", ""));
|
||||
return { frameTree: nestedTree(index) };
|
||||
}
|
||||
if (method === "DOM.getFrameOwner") {
|
||||
return { backendNodeId: Number(params?.frameId?.replace("frame-", "")) + 100 };
|
||||
}
|
||||
return {};
|
||||
},
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
|
||||
const graph = await cdp.getFrameGraph(4);
|
||||
|
||||
expect(graph.frames).toHaveLength(depth + 1);
|
||||
for (let index = 1; index <= depth; index += 1) {
|
||||
expect(
|
||||
graph.frames.find((frame) => frame.frameId === `frame-${index}`)?.target.sessionId,
|
||||
).toBe(`session-${index}`);
|
||||
}
|
||||
});
|
||||
|
||||
it("coalesces concurrent attach calls for the same tab", async () => {
|
||||
const { api } = fakeApi();
|
||||
let releaseAttach!: () => void;
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildFrameGraph, type CdpFrameTreeNode, type CdpFrameTreeSource } from "../frame-graph";
|
||||
|
||||
describe("buildFrameGraph", () => {
|
||||
it("keeps sibling frames distinct and routes nested OOPIFs to their child sessions", () => {
|
||||
const graph = buildFrameGraph([
|
||||
{
|
||||
target: { tabId: 4 },
|
||||
tree: {
|
||||
frame: { id: "main", url: "https://app.test" },
|
||||
childFrames: [
|
||||
{ frame: { id: "left", parentId: "main", url: "https://left.test" } },
|
||||
{
|
||||
frame: { id: "right", parentId: "main", url: "https://right.test" },
|
||||
childFrames: [
|
||||
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
target: { tabId: 4, sessionId: "right-session" },
|
||||
tree: {
|
||||
frame: { id: "right", url: "https://right.test" },
|
||||
childFrames: [{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } }],
|
||||
},
|
||||
},
|
||||
{
|
||||
target: { tabId: 4, sessionId: "nested-session" },
|
||||
tree: { frame: { id: "nested", url: "https://nested.test" } },
|
||||
},
|
||||
]);
|
||||
|
||||
expect(graph?.frames).toHaveLength(4);
|
||||
expect(graph?.frames.find((frame) => frame.frameId === "left")?.target).toEqual({ tabId: 4 });
|
||||
expect(graph?.frames.find((frame) => frame.frameId === "right")).toMatchObject({
|
||||
parentFrameId: "main",
|
||||
target: { tabId: 4, sessionId: "right-session" },
|
||||
});
|
||||
expect(graph?.frames.find((frame) => frame.frameId === "nested")).toMatchObject({
|
||||
parentFrameId: "right",
|
||||
target: { tabId: 4, sessionId: "nested-session" },
|
||||
});
|
||||
});
|
||||
|
||||
it("propagates each target boundary to its same-process descendants", () => {
|
||||
const sources: CdpFrameTreeSource[] = [
|
||||
{
|
||||
target: { tabId: 4 },
|
||||
tree: {
|
||||
frame: { id: "main" },
|
||||
childFrames: [
|
||||
{
|
||||
frame: { id: "oopif-a", parentId: "main" },
|
||||
childFrames: [
|
||||
{
|
||||
frame: { id: "same-process-b", parentId: "oopif-a" },
|
||||
childFrames: [
|
||||
{
|
||||
frame: { id: "oopif-c", parentId: "same-process-b" },
|
||||
childFrames: [{ frame: { id: "same-process-d", parentId: "oopif-c" } }],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
target: { tabId: 4, sessionId: "session-a" },
|
||||
tree: {
|
||||
frame: { id: "oopif-a" },
|
||||
childFrames: [
|
||||
{
|
||||
frame: { id: "same-process-b", parentId: "oopif-a" },
|
||||
childFrames: [{ frame: { id: "oopif-c", parentId: "same-process-b" } }],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
target: { tabId: 4, sessionId: "session-c" },
|
||||
tree: {
|
||||
frame: { id: "oopif-c" },
|
||||
childFrames: [{ frame: { id: "same-process-d", parentId: "oopif-c" } }],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const graph = buildFrameGraph(sources);
|
||||
|
||||
expect(graph?.frames.map((frame) => [frame.frameId, frame.target.sessionId])).toEqual([
|
||||
["main", undefined],
|
||||
["oopif-a", "session-a"],
|
||||
["same-process-b", "session-a"],
|
||||
["oopif-c", "session-c"],
|
||||
["same-process-d", "session-c"],
|
||||
]);
|
||||
expect(graph?.frames.find((frame) => frame.frameId === "oopif-a")?.parentFrameId).toBe("main");
|
||||
expect(graph?.frames.find((frame) => frame.frameId === "oopif-c")?.parentFrameId).toBe(
|
||||
"same-process-b",
|
||||
);
|
||||
|
||||
const reordered = buildFrameGraph([sources[2], sources[1], sources[0]]);
|
||||
expect(reordered).toEqual(graph);
|
||||
});
|
||||
|
||||
it("walks deeply nested frame trees without consuming the JavaScript call stack", () => {
|
||||
const depth = 5_000;
|
||||
const root: CdpFrameTreeNode = { frame: { id: "frame-0" } };
|
||||
let parent = root;
|
||||
for (let index = 1; index <= depth; index += 1) {
|
||||
const child: CdpFrameTreeNode = {
|
||||
frame: { id: `frame-${index}`, parentId: `frame-${index - 1}` },
|
||||
};
|
||||
parent.childFrames = [child];
|
||||
parent = child;
|
||||
}
|
||||
|
||||
const graph = buildFrameGraph([{ target: { tabId: 4 }, tree: root }]);
|
||||
|
||||
expect(graph?.frames).toHaveLength(depth + 1);
|
||||
expect(graph?.frames.at(-1)?.frameId).toBe(`frame-${depth}`);
|
||||
});
|
||||
|
||||
it("does not loop when an in-memory frame tree contains an object cycle", () => {
|
||||
const root: CdpFrameTreeNode = { frame: { id: "main" } };
|
||||
root.childFrames = [root];
|
||||
|
||||
expect(buildFrameGraph([{ target: { tabId: 4 }, tree: root }])?.frames).toEqual([
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -31,6 +31,15 @@ import type {
|
||||
NetworkEntryKind,
|
||||
NetworkResult,
|
||||
} from "@/transport/types";
|
||||
import {
|
||||
buildFrameGraph,
|
||||
type CdpFrameGraph,
|
||||
type CdpFrameTreeNode,
|
||||
type CdpFrameTreeSource,
|
||||
type CdpTarget,
|
||||
} from "./frame-graph";
|
||||
|
||||
export type CdpDebuggee = chrome.debugger.Debuggee & { sessionId?: string };
|
||||
|
||||
/**
|
||||
* Minimal slice of `chrome.debugger` the rest of the extension
|
||||
@@ -38,21 +47,15 @@ import type {
|
||||
* fake without monkey-patching the real `chrome` global.
|
||||
*/
|
||||
export interface CdpDebuggerApi {
|
||||
attach(target: chrome.debugger.Debuggee, requiredVersion: string): Promise<void>;
|
||||
detach(target: chrome.debugger.Debuggee): Promise<void>;
|
||||
sendCommand(
|
||||
target: chrome.debugger.Debuggee,
|
||||
method: string,
|
||||
commandParams?: object,
|
||||
): Promise<unknown>;
|
||||
attach(target: CdpDebuggee, requiredVersion: string): Promise<void>;
|
||||
detach(target: CdpDebuggee): Promise<void>;
|
||||
sendCommand(target: CdpDebuggee, method: string, commandParams?: object): Promise<unknown>;
|
||||
/**
|
||||
* Fires for every CDP event (`Page.lifecycleEvent`, `DOM.documentUpdated`,
|
||||
* …). The first callback argument is the source debuggee; the second
|
||||
* is the CDP method name; the third is the payload.
|
||||
*/
|
||||
onEvent: chrome.events.Event<
|
||||
(source: chrome.debugger.Debuggee, method: string, params: unknown) => void
|
||||
>;
|
||||
onEvent: chrome.events.Event<(source: CdpDebuggee, method: string, params: unknown) => void>;
|
||||
/**
|
||||
* Fires when Chrome unilaterally detaches us — most commonly because
|
||||
* the tab navigated to a chrome:// URL or the user clicked
|
||||
@@ -110,6 +113,8 @@ const MAX_CONSOLE_STACK_FRAMES = 20;
|
||||
const MAX_NETWORK_BUFFER = 200;
|
||||
const MAX_NETWORK_FIELD_LENGTH = 4096;
|
||||
const MAX_NETWORK_REQUEST_META = 1024;
|
||||
const FRAME_DISCOVERY_TIMEOUT_MS = 1000;
|
||||
const FRAME_DISCOVERY_QUIET_MS = 20;
|
||||
|
||||
interface ParsedDialogOpening {
|
||||
type: JavaScriptDialogType;
|
||||
@@ -131,6 +136,27 @@ interface NetworkRequestMeta {
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
interface FrameDiscoveryState {
|
||||
sessions: Set<string>;
|
||||
pending: Set<Promise<void>>;
|
||||
generation: number;
|
||||
}
|
||||
|
||||
async function settleBeforeDeadline(promises: Promise<void>[], deadline: number): Promise<boolean> {
|
||||
if (promises.length === 0) return true;
|
||||
const remaining = deadline - Date.now();
|
||||
if (remaining <= 0) return false;
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const timedOut = await Promise.race([
|
||||
Promise.allSettled(promises).then(() => false),
|
||||
new Promise<boolean>((resolve) => {
|
||||
timer = setTimeout(() => resolve(true), remaining);
|
||||
}),
|
||||
]);
|
||||
if (timer) clearTimeout(timer);
|
||||
return !timedOut;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrapper around `chrome.debugger` that owns the "attach once per
|
||||
* tabId" cache and exposes typed `send<T>()`.
|
||||
@@ -149,10 +175,12 @@ export class ChromiumCdp {
|
||||
private readonly networkSequences = new Map<number, number>();
|
||||
private readonly networkDomainsEnabledTabs = new Set<number>();
|
||||
private readonly networkRequestMeta = new Map<number, Map<string, NetworkRequestMeta>>();
|
||||
private readonly frameDiscovery = new Map<number, FrameDiscoveryState>();
|
||||
private detachSubscription: { dispose(): void } | null = null;
|
||||
private dialogSubscription: { dispose(): void } | null = null;
|
||||
private consoleSubscription: { dispose(): void } | null = null;
|
||||
private networkSubscription: { dispose(): void } | null = null;
|
||||
private frameTargetSubscription: { dispose(): void } | null = null;
|
||||
|
||||
constructor(api: CdpDebuggerApi = chromeDebuggerApi) {
|
||||
this.api = api;
|
||||
@@ -160,6 +188,7 @@ export class ChromiumCdp {
|
||||
this.bindDialogHandler();
|
||||
this.bindConsoleHandler();
|
||||
this.bindNetworkHandler();
|
||||
this.bindFrameTargetHandler();
|
||||
}
|
||||
|
||||
/** Attach to `tabId` if we haven't already in this driver. */
|
||||
@@ -177,6 +206,9 @@ export class ChromiumCdp {
|
||||
await this.enableConsoleDomains(tabId);
|
||||
await this.enableNetworkDomainBestEffort(tabId);
|
||||
this.attachedTabs.add(tabId);
|
||||
await this.enableFrameDiscovery({ tabId }).catch((err) => {
|
||||
console.debug("[bsk cdp] frame discovery unavailable", { tabId, err });
|
||||
});
|
||||
} catch (err) {
|
||||
// A CDP domain enable failed after the raw attach succeeded
|
||||
// (e.g. `Page.enable` rejects because the tab just navigated to
|
||||
@@ -222,6 +254,73 @@ export class ChromiumCdp {
|
||||
}
|
||||
}
|
||||
|
||||
async sendToTarget<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T> {
|
||||
if (!this.attachedTabs.has(target.tabId)) {
|
||||
await this.ensureAttached(target.tabId);
|
||||
}
|
||||
try {
|
||||
return (await this.api.sendCommand(target, method, params ?? {})) as T;
|
||||
} catch (err) {
|
||||
throw normalizeError(err);
|
||||
}
|
||||
}
|
||||
|
||||
async getFrameGraph(tabId: number): Promise<CdpFrameGraph> {
|
||||
await this.ensureAttached(tabId);
|
||||
await this.enableFrameDiscovery({ tabId }).catch(() => {});
|
||||
await this.drainFrameAttachTasks(tabId);
|
||||
|
||||
const sources: CdpFrameTreeSource[] = [];
|
||||
const root = await this.sendToTarget<{ frameTree?: CdpFrameTreeNode }>(
|
||||
{ tabId },
|
||||
"Page.getFrameTree",
|
||||
{},
|
||||
);
|
||||
if (root.frameTree) sources.push({ target: { tabId }, tree: root.frameTree });
|
||||
|
||||
const sessions = [...(this.frameDiscovery.get(tabId)?.sessions ?? [])];
|
||||
const childTrees = await Promise.all(
|
||||
sessions.map(async (sessionId): Promise<CdpFrameTreeSource | null> => {
|
||||
const target = { tabId, sessionId };
|
||||
try {
|
||||
const reply = await this.sendToTarget<{ frameTree?: CdpFrameTreeNode }>(
|
||||
target,
|
||||
"Page.getFrameTree",
|
||||
{},
|
||||
);
|
||||
return reply.frameTree ? { target, tree: reply.frameTree } : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}),
|
||||
);
|
||||
for (const source of childTrees) {
|
||||
if (source) sources.push(source);
|
||||
}
|
||||
|
||||
const graph = buildFrameGraph(sources);
|
||||
if (!graph) throw new Error("Page.getFrameTree returned no root frame");
|
||||
const frameById = new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
||||
await Promise.all(
|
||||
graph.frames.map(async (frame) => {
|
||||
if (!frame.parentFrameId) return;
|
||||
const parent = frameById.get(frame.parentFrameId);
|
||||
if (!parent) return;
|
||||
try {
|
||||
const owner = await this.sendToTarget<{ backendNodeId?: number }>(
|
||||
parent.target,
|
||||
"DOM.getFrameOwner",
|
||||
{ frameId: frame.frameId },
|
||||
);
|
||||
if (owner.backendNodeId !== undefined) frame.ownerBackendNodeId = owner.backendNodeId;
|
||||
} catch {
|
||||
// The frame may have navigated between tree capture and owner lookup.
|
||||
}
|
||||
}),
|
||||
);
|
||||
return graph;
|
||||
}
|
||||
|
||||
/** Return a cursor marking the current dialog sequence for `tabId`. */
|
||||
dialogCursor(tabId: number): DialogCursor {
|
||||
return this.dialogSequences.get(tabId) ?? 0;
|
||||
@@ -332,6 +431,7 @@ export class ChromiumCdp {
|
||||
this.clearDialogState(tabId);
|
||||
this.clearConsoleState(tabId);
|
||||
this.clearNetworkState(tabId);
|
||||
this.clearFrameState(tabId);
|
||||
try {
|
||||
await this.api.detach({ tabId });
|
||||
} catch (err) {
|
||||
@@ -354,7 +454,7 @@ export class ChromiumCdp {
|
||||
}
|
||||
|
||||
/** Subscribe to all CDP events. Returned disposable removes the listener. */
|
||||
onEvent(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): {
|
||||
onEvent(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
|
||||
dispose(): void;
|
||||
} {
|
||||
this.api.onEvent.addListener(handler);
|
||||
@@ -378,6 +478,7 @@ export class ChromiumCdp {
|
||||
this.networkSequences.clear();
|
||||
this.networkDomainsEnabledTabs.clear();
|
||||
this.networkRequestMeta.clear();
|
||||
this.frameDiscovery.clear();
|
||||
await Promise.all(
|
||||
tabs.map(async (tabId) => {
|
||||
try {
|
||||
@@ -428,9 +529,104 @@ export class ChromiumCdp {
|
||||
}
|
||||
}
|
||||
|
||||
private async enableFrameDiscovery(target: CdpTarget): Promise<void> {
|
||||
await this.api.sendCommand(target, "Target.setAutoAttach", {
|
||||
autoAttach: true,
|
||||
waitForDebuggerOnStart: false,
|
||||
flatten: true,
|
||||
filter: [{ type: "iframe", exclude: false }],
|
||||
});
|
||||
}
|
||||
|
||||
private bindFrameTargetHandler(): void {
|
||||
if (this.frameTargetSubscription) return;
|
||||
const listener = (source: CdpDebuggee, method: string, params: unknown) => {
|
||||
const tabId = source.tabId;
|
||||
if (typeof tabId !== "number") return;
|
||||
const raw = (params ?? {}) as Record<string, unknown>;
|
||||
const sessionId = typeof raw.sessionId === "string" ? raw.sessionId : undefined;
|
||||
if (!sessionId) return;
|
||||
|
||||
if (method === "Target.detachedFromTarget") {
|
||||
const state = this.frameDiscovery.get(tabId);
|
||||
if (state?.sessions.delete(sessionId)) state.generation += 1;
|
||||
return;
|
||||
}
|
||||
if (method !== "Target.attachedToTarget") return;
|
||||
const targetInfo = raw.targetInfo as { type?: string } | undefined;
|
||||
if (targetInfo?.type && targetInfo.type !== "iframe") return;
|
||||
|
||||
const state = this.frameDiscoveryState(tabId);
|
||||
if (state.sessions.has(sessionId)) return;
|
||||
state.sessions.add(sessionId);
|
||||
state.generation += 1;
|
||||
|
||||
const task = this.initializeFrameTarget({ tabId, sessionId });
|
||||
state.pending.add(task);
|
||||
void task.finally(() => {
|
||||
state.pending.delete(task);
|
||||
});
|
||||
};
|
||||
this.api.onEvent.addListener(listener);
|
||||
this.frameTargetSubscription = {
|
||||
dispose: () => this.api.onEvent.removeListener(listener),
|
||||
};
|
||||
}
|
||||
|
||||
private async initializeFrameTarget(target: CdpTarget): Promise<void> {
|
||||
try {
|
||||
await this.enableFrameDiscovery(target);
|
||||
} catch (err) {
|
||||
const state = this.frameDiscovery.get(target.tabId);
|
||||
if (state?.sessions.delete(target.sessionId as string)) state.generation += 1;
|
||||
console.debug("[bsk cdp] child frame target initialization failed", { target, err });
|
||||
}
|
||||
}
|
||||
|
||||
private async drainFrameAttachTasks(tabId: number): Promise<void> {
|
||||
const deadline = Date.now() + FRAME_DISCOVERY_TIMEOUT_MS;
|
||||
while (Date.now() < deadline) {
|
||||
const state = this.frameDiscovery.get(tabId);
|
||||
const generation = state?.generation ?? 0;
|
||||
if (state?.pending.size && !(await settleBeforeDeadline([...state.pending], deadline))) break;
|
||||
|
||||
// Target.setAutoAttach may enqueue the next attachedToTarget event after
|
||||
// its command promise settles. Wait for a short quiet window, then finish
|
||||
// only if neither the state object nor its generation changed.
|
||||
const quietTime = Math.min(FRAME_DISCOVERY_QUIET_MS, deadline - Date.now());
|
||||
if (quietTime <= 0) break;
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, quietTime));
|
||||
const current = this.frameDiscovery.get(tabId);
|
||||
if (
|
||||
current === state &&
|
||||
(current?.generation ?? 0) === generation &&
|
||||
!current?.pending.size
|
||||
) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
console.debug("[bsk cdp] frame discovery did not reach quiescence before timeout", { tabId });
|
||||
}
|
||||
|
||||
private frameDiscoveryState(tabId: number): FrameDiscoveryState {
|
||||
const existing = this.frameDiscovery.get(tabId);
|
||||
if (existing) return existing;
|
||||
const created: FrameDiscoveryState = {
|
||||
sessions: new Set(),
|
||||
pending: new Set(),
|
||||
generation: 0,
|
||||
};
|
||||
this.frameDiscovery.set(tabId, created);
|
||||
return created;
|
||||
}
|
||||
|
||||
private clearFrameState(tabId: number): void {
|
||||
this.frameDiscovery.delete(tabId);
|
||||
}
|
||||
|
||||
private bindDialogHandler(): void {
|
||||
if (this.dialogSubscription) return;
|
||||
const listener = (source: chrome.debugger.Debuggee, method: string, params: unknown) => {
|
||||
const listener = (source: CdpDebuggee, method: string, params: unknown) => {
|
||||
if (method !== "Page.javascriptDialogOpening") return;
|
||||
const tabId = source.tabId;
|
||||
if (typeof tabId !== "number") return;
|
||||
@@ -614,6 +810,7 @@ export class ChromiumCdp {
|
||||
this.clearDialogState(source.tabId);
|
||||
this.clearConsoleState(source.tabId);
|
||||
this.clearNetworkState(source.tabId);
|
||||
this.clearFrameState(source.tabId);
|
||||
}
|
||||
};
|
||||
this.api.onDetach.addListener(listener);
|
||||
@@ -632,6 +829,8 @@ export class ChromiumCdp {
|
||||
this.consoleSubscription = null;
|
||||
this.networkSubscription?.dispose();
|
||||
this.networkSubscription = null;
|
||||
this.frameTargetSubscription?.dispose();
|
||||
this.frameTargetSubscription = null;
|
||||
}
|
||||
|
||||
/** Detach tabs only when no other live session has claimed them. */
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
export interface CdpTarget {
|
||||
tabId: number;
|
||||
sessionId?: string;
|
||||
}
|
||||
|
||||
export interface CdpFrame {
|
||||
frameId: string;
|
||||
parentFrameId?: string;
|
||||
ownerBackendNodeId?: number;
|
||||
url?: string;
|
||||
target: CdpTarget;
|
||||
}
|
||||
|
||||
export interface CdpFrameGraph {
|
||||
rootFrameId: string;
|
||||
frames: CdpFrame[];
|
||||
}
|
||||
|
||||
export interface CdpFrameTreeNode {
|
||||
frame: {
|
||||
id: string;
|
||||
parentId?: string;
|
||||
url?: string;
|
||||
};
|
||||
childFrames?: CdpFrameTreeNode[];
|
||||
}
|
||||
|
||||
export interface CdpFrameTreeSource {
|
||||
target: CdpTarget;
|
||||
tree: CdpFrameTreeNode;
|
||||
}
|
||||
|
||||
export function cdpTargetKey(target: CdpTarget): string {
|
||||
return `${target.tabId}:${target.sessionId ?? "root"}`;
|
||||
}
|
||||
|
||||
function mergeFrame(
|
||||
frames: Map<string, CdpFrame>,
|
||||
order: string[],
|
||||
node: CdpFrameTreeNode,
|
||||
target: CdpTarget,
|
||||
): void {
|
||||
const existing = frames.get(node.frame.id);
|
||||
if (!existing) order.push(node.frame.id);
|
||||
const parentFrameId = node.frame.parentId || existing?.parentFrameId;
|
||||
const url = node.frame.url || existing?.url;
|
||||
frames.set(node.frame.id, {
|
||||
frameId: node.frame.id,
|
||||
...(parentFrameId ? { parentFrameId } : {}),
|
||||
...(url ? { url } : {}),
|
||||
target,
|
||||
});
|
||||
}
|
||||
|
||||
function walkFrameTree(
|
||||
source: CdpFrameTreeSource,
|
||||
targetByRootFrameId: ReadonlyMap<string, CdpTarget>,
|
||||
frames: Map<string, CdpFrame>,
|
||||
order: string[],
|
||||
): void {
|
||||
const stack: Array<{ node: CdpFrameTreeNode; inheritedTarget: CdpTarget }> = [
|
||||
{ node: source.tree, inheritedTarget: source.target },
|
||||
];
|
||||
const expanded = new WeakSet<CdpFrameTreeNode>();
|
||||
|
||||
while (stack.length > 0) {
|
||||
const current = stack.pop();
|
||||
if (!current || expanded.has(current.node)) continue;
|
||||
expanded.add(current.node);
|
||||
|
||||
const target = targetByRootFrameId.get(current.node.frame.id) ?? current.inheritedTarget;
|
||||
mergeFrame(frames, order, current.node, target);
|
||||
|
||||
const children = current.node.childFrames ?? [];
|
||||
for (let index = children.length - 1; index >= 0; index -= 1) {
|
||||
stack.push({ node: children[index], inheritedTarget: target });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function buildFrameGraph(sources: CdpFrameTreeSource[]): CdpFrameGraph | null {
|
||||
const rootSource = sources.find((source) => source.target.sessionId === undefined) ?? sources[0];
|
||||
if (!rootSource) return null;
|
||||
|
||||
const frames = new Map<string, CdpFrame>();
|
||||
const order: string[] = [];
|
||||
const targetByRootFrameId = new Map(
|
||||
sources.map((source) => [source.tree.frame.id, source.target] as const),
|
||||
);
|
||||
// The top-level source is authoritative if malformed input reports the same
|
||||
// frame as both a root target and a child target.
|
||||
targetByRootFrameId.set(rootSource.tree.frame.id, rootSource.target);
|
||||
|
||||
walkFrameTree(rootSource, targetByRootFrameId, frames, order);
|
||||
for (const source of sources) {
|
||||
if (source === rootSource) continue;
|
||||
walkFrameTree(source, targetByRootFrameId, frames, order);
|
||||
}
|
||||
|
||||
return {
|
||||
rootFrameId: rootSource.tree.frame.id,
|
||||
frames: order.flatMap((frameId) => {
|
||||
const frame = frames.get(frameId);
|
||||
return frame ? [frame] : [];
|
||||
}),
|
||||
};
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
useRef,
|
||||
useState,
|
||||
} from "react";
|
||||
import type { HelpHighlightRect } from "@/lib/help-bridge";
|
||||
import logoUrl from "../../assets/logo.png";
|
||||
|
||||
export interface HelpRequestData {
|
||||
@@ -19,6 +20,10 @@ export interface HelpRequestData {
|
||||
displayMode?: "full" | "compact";
|
||||
/** CSS selectors to scroll to + flash-highlight. */
|
||||
selectors: string[];
|
||||
/** Pre-resolved top-viewport rectangles, used for cross-frame targets. */
|
||||
rects?: HelpHighlightRect[];
|
||||
/** Re-resolve cross-frame rectangles after scroll, resize, or layout changes. */
|
||||
refreshRects?: () => Promise<HelpHighlightRect[] | undefined>;
|
||||
onContinue: (note: string) => void;
|
||||
onCancel: () => void;
|
||||
}
|
||||
@@ -49,6 +54,7 @@ const PANEL_WIDTH = 420;
|
||||
const FALLBACK_PANEL_H = 180;
|
||||
const FALLBACK_PANEL_H_COLLAPSED = 44;
|
||||
const EMPTY_SELECTORS: string[] = [];
|
||||
const EMPTY_RECTS: HelpHighlightRect[] = [];
|
||||
|
||||
function clamp(value: number, min: number, max: number): number {
|
||||
return Math.max(min, Math.min(max, value));
|
||||
@@ -179,6 +185,8 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
const { t } = useTranslation("extension");
|
||||
const isCompact = request?.displayMode === "compact";
|
||||
const effectiveSelectors = isCompact ? EMPTY_SELECTORS : (request?.selectors ?? EMPTY_SELECTORS);
|
||||
const [liveRects, setLiveRects] = useState<HelpHighlightRect[] | null>(null);
|
||||
const effectiveRects = isCompact ? EMPTY_RECTS : (liveRects ?? request?.rects ?? EMPTY_RECTS);
|
||||
const [note, setNote] = useState("");
|
||||
const [collapsed, setCollapsed] = useState(false);
|
||||
const [boxes, setBoxes] = useState<Box[]>([]);
|
||||
@@ -201,6 +209,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
// Reset the note and collapse state whenever a new request appears.
|
||||
useEffect(() => {
|
||||
setNote("");
|
||||
setLiveRects(null);
|
||||
setCollapsed(false);
|
||||
placementRef.current = null;
|
||||
setDragPos(null);
|
||||
@@ -209,6 +218,32 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
dragStartRef.current = null;
|
||||
}, [request?.id]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!request || isCompact || !request.refreshRects) return;
|
||||
let disposed = false;
|
||||
let resolving = false;
|
||||
const refresh = async () => {
|
||||
if (resolving) return;
|
||||
resolving = true;
|
||||
try {
|
||||
const rects = await request.refreshRects?.();
|
||||
if (!disposed && rects) setLiveRects(rects);
|
||||
} finally {
|
||||
resolving = false;
|
||||
}
|
||||
};
|
||||
const onViewportChange = () => void refresh();
|
||||
window.addEventListener("scroll", onViewportChange, true);
|
||||
window.addEventListener("resize", onViewportChange);
|
||||
const interval = window.setInterval(onViewportChange, 500);
|
||||
return () => {
|
||||
disposed = true;
|
||||
window.removeEventListener("scroll", onViewportChange, true);
|
||||
window.removeEventListener("resize", onViewportChange);
|
||||
window.clearInterval(interval);
|
||||
};
|
||||
}, [request, isCompact]);
|
||||
|
||||
// Scroll the first matched target into view once per distinct request id.
|
||||
useLayoutEffect(() => {
|
||||
if (!request) return;
|
||||
@@ -231,7 +266,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
return;
|
||||
}
|
||||
const update = () => {
|
||||
const nextBoxes = measure(effectiveSelectors);
|
||||
const nextBoxes = [...measure(effectiveSelectors), ...effectiveRects];
|
||||
setBoxes((prev) => (boxesEqual(prev, nextBoxes) ? prev : nextBoxes));
|
||||
};
|
||||
update();
|
||||
@@ -243,7 +278,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
window.removeEventListener("resize", update);
|
||||
window.clearInterval(interval);
|
||||
};
|
||||
}, [request, effectiveSelectors]);
|
||||
}, [request, effectiveSelectors, effectiveRects]);
|
||||
|
||||
const onHeaderPointerDown = useCallback(
|
||||
(e: ReactPointerEvent<HTMLDivElement>) => {
|
||||
@@ -305,7 +340,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
return;
|
||||
}
|
||||
if (userMovedRef.current) return;
|
||||
const measured = boxes.length > 0 ? boxes : measure(effectiveSelectors);
|
||||
const measured = boxes.length > 0 ? boxes : [...measure(effectiveSelectors), ...effectiveRects];
|
||||
const anchor = unionBox(measured);
|
||||
const banner = bannerRef.current;
|
||||
if (!anchor || !banner) {
|
||||
@@ -329,7 +364,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
}
|
||||
const nextPos = placePanel(anchor, panelW, panelH, placementRef.current.placement);
|
||||
setPanelPos((prev) => (panelPosEqual(prev, nextPos) ? prev : nextPos));
|
||||
}, [request, boxes, collapsed, isCompact, effectiveSelectors]);
|
||||
}, [request, boxes, collapsed, isCompact, effectiveSelectors, effectiveRects]);
|
||||
|
||||
if (!request) return null;
|
||||
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { i18n } from "@browser-skill/i18n";
|
||||
import { I18nextProvider } from "@browser-skill/i18n/react";
|
||||
import { cleanup, fireEvent, render, screen } from "@testing-library/react";
|
||||
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||
import { createElement } from "react";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { type HelpRequestData, HelpRequestOverlay } from "../HelpRequestOverlay";
|
||||
@@ -38,6 +38,36 @@ describe("HelpRequestOverlay", () => {
|
||||
expect(screen.getByText("Please complete the captcha")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("renders explicit viewport rectangles for cross-frame targets", () => {
|
||||
const { container } = renderOverlay(
|
||||
baseRequest({ rects: [{ top: 30, left: 40, width: 120, height: 50 }] }),
|
||||
);
|
||||
const highlight = container.querySelector<HTMLElement>("[data-slot='help-highlight']");
|
||||
expect(highlight?.style.top).toBe("30px");
|
||||
expect(highlight?.style.left).toBe("40px");
|
||||
expect(highlight?.style.width).toBe("120px");
|
||||
expect(highlight?.style.height).toBe("50px");
|
||||
});
|
||||
|
||||
it("re-resolves cross-frame rectangles after the viewport changes", async () => {
|
||||
const refreshRects = vi.fn(async () => [{ top: 80, left: 90, width: 140, height: 60 }]);
|
||||
const { container } = renderOverlay(
|
||||
baseRequest({
|
||||
rects: [{ top: 30, left: 40, width: 120, height: 50 }],
|
||||
refreshRects,
|
||||
}),
|
||||
);
|
||||
|
||||
window.dispatchEvent(new Event("scroll"));
|
||||
|
||||
await waitFor(() => {
|
||||
const highlight = container.querySelector<HTMLElement>("[data-slot='help-highlight']");
|
||||
expect(highlight?.style.top).toBe("80px");
|
||||
expect(highlight?.style.left).toBe("90px");
|
||||
});
|
||||
expect(refreshRects).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("keeps the inactive render stable", () => {
|
||||
const { container, rerender } = renderOverlay(null);
|
||||
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createHelpRequestData } from "../help-request";
|
||||
|
||||
describe("createHelpRequestData", () => {
|
||||
it("gives live delivery and recovery the same rectangle refresh mechanism", async () => {
|
||||
const finish = vi.fn();
|
||||
const query = vi.fn(async () => ({
|
||||
active: true,
|
||||
request: {
|
||||
requestId: "help-1",
|
||||
prompt: "complete the challenge",
|
||||
selectors: [],
|
||||
rects: [{ top: 80, left: 90, width: 140, height: 60 }],
|
||||
timeoutMs: 1_000,
|
||||
},
|
||||
}));
|
||||
const request = createHelpRequestData(
|
||||
{
|
||||
requestId: "help-1",
|
||||
prompt: "complete the challenge",
|
||||
selectors: [],
|
||||
rects: [{ top: 10, left: 20, width: 30, height: 40 }],
|
||||
timeoutMs: 1_000,
|
||||
},
|
||||
{ finish, query },
|
||||
);
|
||||
|
||||
await expect(request.refreshRects?.()).resolves.toEqual([
|
||||
{ top: 80, left: 90, width: 140, height: 60 },
|
||||
]);
|
||||
request.onContinue(" done ");
|
||||
request.onCancel();
|
||||
|
||||
expect(query).toHaveBeenCalledOnce();
|
||||
expect(finish).toHaveBeenNthCalledWith(1, "help-1", "continued", " done ");
|
||||
expect(finish).toHaveBeenNthCalledWith(2, "help-1", "cancelled");
|
||||
});
|
||||
|
||||
it("ignores rectangle replies for a replaced request", async () => {
|
||||
const request = createHelpRequestData(
|
||||
{
|
||||
requestId: "help-1",
|
||||
prompt: "complete the challenge",
|
||||
selectors: [],
|
||||
timeoutMs: 1_000,
|
||||
},
|
||||
{
|
||||
finish: vi.fn(),
|
||||
query: vi.fn(async () => ({
|
||||
active: true,
|
||||
request: {
|
||||
requestId: "help-2",
|
||||
prompt: "new challenge",
|
||||
selectors: [],
|
||||
rects: [{ top: 1, left: 2, width: 3, height: 4 }],
|
||||
timeoutMs: 1_000,
|
||||
},
|
||||
})),
|
||||
},
|
||||
);
|
||||
|
||||
await expect(request.refreshRects?.()).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,31 @@
|
||||
import type { HelpQueryResponse, HelpRequestMessage } from "@/lib/help-bridge";
|
||||
import type { HelpRequestData } from "./HelpRequestOverlay";
|
||||
|
||||
export interface HelpRequestDataDeps {
|
||||
finish(requestId: string, outcome: "continued" | "cancelled", note?: string): void;
|
||||
query(): Promise<HelpQueryResponse | undefined>;
|
||||
}
|
||||
|
||||
/** Build the content-side representation shared by live delivery and recovery. */
|
||||
export function createHelpRequestData(
|
||||
message: Omit<HelpRequestMessage, "type">,
|
||||
deps: HelpRequestDataDeps,
|
||||
): HelpRequestData {
|
||||
return {
|
||||
id: message.requestId,
|
||||
prompt: message.prompt,
|
||||
...(message.title ? { title: message.title } : {}),
|
||||
...(message.displayMode ? { displayMode: message.displayMode } : {}),
|
||||
selectors: message.selectors,
|
||||
rects: message.rects,
|
||||
refreshRects: async () => {
|
||||
const response = await deps.query();
|
||||
return response?.active && response.request?.requestId === message.requestId
|
||||
? response.request.rects
|
||||
: undefined;
|
||||
},
|
||||
onContinue: (note: string) =>
|
||||
deps.finish(message.requestId, "continued", note.trim() ? note : undefined),
|
||||
onCancel: () => deps.finish(message.requestId, "cancelled"),
|
||||
};
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import { BorrowConfirmationOverlay } from "@/content/BorrowConfirmationOverlay";
|
||||
import { ControlOverlay } from "@/content/ControlOverlay";
|
||||
import { createCaptureSuppressController } from "@/content/capture-suppress";
|
||||
import { HelpRequestOverlay } from "@/content/HelpRequestOverlay";
|
||||
import { createHelpRequestData } from "@/content/help-request";
|
||||
import overlayCss from "@/content/overlay.css?inline";
|
||||
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
|
||||
import { RecordOverlay } from "@/content/RecordOverlay";
|
||||
@@ -344,16 +345,7 @@ export default defineContentScript({
|
||||
|
||||
if (isHelpRequestMessage(message)) {
|
||||
const helpMsg = message as HelpRequestMessage;
|
||||
const previousHelp = overlays.setAgentHelpRequest({
|
||||
id: helpMsg.requestId,
|
||||
prompt: helpMsg.prompt,
|
||||
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
||||
...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}),
|
||||
selectors: helpMsg.selectors,
|
||||
onContinue: (note: string) =>
|
||||
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
|
||||
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
|
||||
});
|
||||
const previousHelp = mountHelpRequest(helpMsg);
|
||||
if (previousHelp && previousHelp.id !== helpMsg.requestId) {
|
||||
void sendHelpFinish(previousHelp.id, "cancelled");
|
||||
}
|
||||
@@ -406,17 +398,18 @@ export default defineContentScript({
|
||||
});
|
||||
}
|
||||
|
||||
function mountHelpRequest(helpMsg: Omit<HelpRequestMessage, "type">): void {
|
||||
overlays.setAgentHelpRequest({
|
||||
id: helpMsg.requestId,
|
||||
prompt: helpMsg.prompt,
|
||||
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
||||
...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}),
|
||||
selectors: helpMsg.selectors,
|
||||
onContinue: (note: string) =>
|
||||
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
|
||||
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
|
||||
});
|
||||
function mountHelpRequest(helpMsg: Omit<HelpRequestMessage, "type">) {
|
||||
return overlays.setAgentHelpRequest(
|
||||
createHelpRequestData(helpMsg, {
|
||||
finish: (requestId, outcome, note) => {
|
||||
void sendHelpFinish(requestId, outcome, note);
|
||||
},
|
||||
query: () =>
|
||||
chrome.runtime.sendMessage({ type: HELP_QUERY }) as Promise<
|
||||
HelpQueryResponse | undefined
|
||||
>,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function queryActiveHelpWithRetry(): Promise<void> {
|
||||
|
||||
@@ -32,6 +32,7 @@ describe("help-bridge", () => {
|
||||
prompt: "log in",
|
||||
displayMode: "compact",
|
||||
selectors: ["#login"],
|
||||
rects: [{ top: 10, left: 20, width: 100, height: 40 }],
|
||||
timeoutMs: 1000,
|
||||
}),
|
||||
).toBe(true);
|
||||
|
||||
@@ -16,6 +16,13 @@ export const HELP_ACK = "bsk-help-ack";
|
||||
export const HELP_QUERY = "bsk-help-query";
|
||||
export const HELP_FINISH = "bsk-help-finish";
|
||||
|
||||
export interface HelpHighlightRect {
|
||||
top: number;
|
||||
left: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface HelpRequestMessage {
|
||||
type: typeof HELP_REQUEST;
|
||||
requestId: string;
|
||||
@@ -26,6 +33,8 @@ export interface HelpRequestMessage {
|
||||
displayMode?: "full" | "compact";
|
||||
/** CSS selectors to scroll to + flash-highlight (may be empty). */
|
||||
selectors: string[];
|
||||
/** Top-viewport rectangles for targets that cannot be represented by a root selector. */
|
||||
rects?: HelpHighlightRect[];
|
||||
timeoutMs: number;
|
||||
}
|
||||
|
||||
@@ -74,6 +83,16 @@ export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage {
|
||||
(m.displayMode === undefined || m.displayMode === "full" || m.displayMode === "compact") &&
|
||||
Array.isArray(m.selectors) &&
|
||||
m.selectors.every((selector) => typeof selector === "string") &&
|
||||
(m.rects === undefined ||
|
||||
(Array.isArray(m.rects) &&
|
||||
m.rects.every(
|
||||
(rect) =>
|
||||
typeof rect === "object" &&
|
||||
rect !== null &&
|
||||
["top", "left", "width", "height"].every((key) =>
|
||||
Number.isFinite((rect as Record<string, unknown>)[key]),
|
||||
),
|
||||
))) &&
|
||||
typeof m.timeoutMs === "number"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -33,4 +33,19 @@ describe("RefStore", () => {
|
||||
expect(s.resolve("@e11")).toBe(11);
|
||||
expect(s.size()).toBe(2);
|
||||
});
|
||||
|
||||
it("preserves the child CDP session as part of ref identity", () => {
|
||||
const s = new RefStore();
|
||||
s.set("e1", 42, {
|
||||
tabId: 7,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
expect(s.resolveEntry("e1")).toMatchObject({
|
||||
backendNodeId: 42,
|
||||
tabId: 7,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
/**
|
||||
* Per-session map from `@e<N>` snapshot refs to CDP `backendNodeId`.
|
||||
* Per-session map from `@e<N>` snapshot refs to a CDP node address.
|
||||
*
|
||||
* Each fresh `tool.snapshot` resets the store: M6 will call
|
||||
* `replace(...)` with the new ref → backendNodeId pairs. Tools like
|
||||
* `tool.click` consume the store via `resolve("@e1")`.
|
||||
* `replace(...)` with the new ref → node address pairs. A node address
|
||||
* includes the owning flat CDP session/frame when the element lives in
|
||||
* an OOPIF; tools resolve live geometry from that identity at call time.
|
||||
*
|
||||
* Refs are session-scoped (§7): looking up a ref in the wrong session
|
||||
* returns `null`, never silently leaks. Storing values in different
|
||||
@@ -15,10 +16,19 @@ export type BackendNodeId = number;
|
||||
export interface RefEntry {
|
||||
backendNodeId: BackendNodeId;
|
||||
tabId: number | null;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
generation: number;
|
||||
}
|
||||
|
||||
type RefInput = BackendNodeId | { backendNodeId: BackendNodeId; tabId: number };
|
||||
export type RefInput =
|
||||
| BackendNodeId
|
||||
| {
|
||||
backendNodeId: BackendNodeId;
|
||||
tabId: number;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
};
|
||||
|
||||
export class RefStore {
|
||||
private readonly map = new Map<string, RefEntry>();
|
||||
@@ -44,7 +54,7 @@ export class RefStore {
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace the entire store with a new ref → backendNodeId mapping.
|
||||
* Replace the entire store with a new ref → CDP node identity mapping.
|
||||
* Used after every fresh `tool.snapshot`.
|
||||
*/
|
||||
replace(entries: Iterable<readonly [string, RefInput]>): void {
|
||||
@@ -53,10 +63,20 @@ export class RefStore {
|
||||
for (const [ref, input] of entries) this.map.set(normaliseRef(ref), this.entry(input));
|
||||
}
|
||||
|
||||
set(ref: string, id: BackendNodeId, opts: { tabId?: number } = {}): void {
|
||||
set(
|
||||
ref: string,
|
||||
id: BackendNodeId,
|
||||
opts: {
|
||||
tabId?: number;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
} = {},
|
||||
): void {
|
||||
this.map.set(normaliseRef(ref), {
|
||||
backendNodeId: id,
|
||||
tabId: opts.tabId ?? null,
|
||||
...(opts.frameId ? { frameId: opts.frameId } : {}),
|
||||
...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}),
|
||||
generation: this.generation,
|
||||
});
|
||||
}
|
||||
@@ -80,6 +100,8 @@ export class RefStore {
|
||||
return {
|
||||
backendNodeId: input.backendNodeId,
|
||||
tabId: input.tabId,
|
||||
...(input.frameId ? { frameId: input.frameId } : {}),
|
||||
...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
|
||||
generation: this.generation,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -612,4 +612,14 @@ describe("isInjectableContentScriptUrl", () => {
|
||||
).toBe(false);
|
||||
expect(isInjectableContentScriptUrl("https://chromewebstore.google.com/")).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects the Edge Add-ons store", () => {
|
||||
expect(
|
||||
isInjectableContentScriptUrl(
|
||||
"https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg",
|
||||
),
|
||||
).toBe(false);
|
||||
// Non-store paths on the same host stay injectable.
|
||||
expect(isInjectableContentScriptUrl("https://microsoftedge.microsoft.com/")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -203,6 +203,55 @@ describe("ToolDispatcher", () => {
|
||||
expect(sent[0]).toEqual({ id: "r-1", result: {} });
|
||||
});
|
||||
|
||||
it("forwards daemon cancellation into an in-flight session stop", async () => {
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const remove = vi.fn(async () => {});
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove,
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
let finishDetach: () => void = () => {};
|
||||
const detach = new Promise<void>((resolve) => {
|
||||
finishDetach = resolve;
|
||||
});
|
||||
const cdp = {
|
||||
send: vi.fn(),
|
||||
detachSession: vi.fn(() => detach),
|
||||
ensureNetworkCapture: vi.fn(async () => {}),
|
||||
networkEntriesSince: vi.fn(() => ({
|
||||
tab_id: 7,
|
||||
entries: [],
|
||||
next_since: 0,
|
||||
truncated: false,
|
||||
})),
|
||||
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
setUserAgentOverride: vi.fn(async () => {}),
|
||||
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||
dispatcher.start();
|
||||
|
||||
deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
|
||||
await vi.waitFor(() => expect(cdp.detachSession).toHaveBeenCalledWith("aa11"));
|
||||
deliver({ id: "cancel-stop", method: "cancel", params: { rpc_id: "r-1" } });
|
||||
await flushMicrotasks();
|
||||
finishDetach();
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(sent).toContainEqual({ id: "cancel-stop", result: { cancelled: true } });
|
||||
expect(sent).toContainEqual({
|
||||
id: "r-1",
|
||||
error: expect.objectContaining({ code: "cancelled" }),
|
||||
});
|
||||
expect(sessions.has("aa11")).toBe(true);
|
||||
expect(remove).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns not_found when stopping an unknown session", async () => {
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
|
||||
@@ -1,24 +1,54 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { nodeBoundingRect, quadBoundingRect } from "../element-geometry";
|
||||
import { nodeContentRegion } from "../element-geometry";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
describe("quadBoundingRect", () => {
|
||||
it("computes axis-aligned bounds from an 8-double quad", () => {
|
||||
expect(quadBoundingRect([10, 20, 110, 20, 110, 60, 10, 60])).toEqual({
|
||||
x: 10,
|
||||
y: 20,
|
||||
width: 100,
|
||||
height: 40,
|
||||
describe("nodeContentRegion", () => {
|
||||
it("rejects invalid content quads before using the box model fallback", async () => {
|
||||
const send = async (_tabId: number, method: string) => {
|
||||
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 0, 0, 0, 0, 0, 0]] };
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [5, 10, 25, 10, 25, 30, 5, 30] } };
|
||||
}
|
||||
throw new Error(`unexpected CDP call ${method}`);
|
||||
};
|
||||
|
||||
await expect(nodeContentRegion({ send: send as CdpRunner["send"] }, 7, 555)).resolves.toEqual([
|
||||
[
|
||||
{ x: 5, y: 10 },
|
||||
{ x: 25, y: 10 },
|
||||
{ x: 25, y: 30 },
|
||||
{ x: 5, y: 30 },
|
||||
],
|
||||
]);
|
||||
});
|
||||
|
||||
it("preserves every visible content fragment", async () => {
|
||||
const send = async () => ({
|
||||
quads: [
|
||||
[10, 20, 60, 20, 60, 40, 10, 40],
|
||||
[15, 50, 90, 50, 90, 80, 15, 80],
|
||||
],
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as CdpRunner["send"],
|
||||
};
|
||||
|
||||
await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([
|
||||
[
|
||||
{ x: 10, y: 20 },
|
||||
{ x: 60, y: 20 },
|
||||
{ x: 60, y: 40 },
|
||||
{ x: 10, y: 40 },
|
||||
],
|
||||
[
|
||||
{ x: 15, y: 50 },
|
||||
{ x: 90, y: 50 },
|
||||
{ x: 90, y: 80 },
|
||||
{ x: 15, y: 80 },
|
||||
],
|
||||
]);
|
||||
});
|
||||
|
||||
it("returns null for degenerate quads", () => {
|
||||
expect(quadBoundingRect([0, 0, 0, 0, 0, 0, 0, 0])).toBeNull();
|
||||
expect(quadBoundingRect([1, 2, 3])).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("nodeBoundingRect", () => {
|
||||
it("falls back to visible descendant bounds for zero-size containers", async () => {
|
||||
const calls: Array<{ method: string; params?: object }> = [];
|
||||
const send = async (_tabId: number, method: string, params?: object) => {
|
||||
@@ -38,12 +68,14 @@ describe("nodeBoundingRect", () => {
|
||||
};
|
||||
const cdp = { send: send as CdpRunner["send"] };
|
||||
|
||||
await expect(nodeBoundingRect(cdp, 7, 555)).resolves.toEqual({
|
||||
x: 242,
|
||||
y: 468,
|
||||
width: 769,
|
||||
height: 180,
|
||||
});
|
||||
await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([
|
||||
[
|
||||
{ x: 242, y: 468 },
|
||||
{ x: 1011, y: 468 },
|
||||
{ x: 1011, y: 648 },
|
||||
{ x: 242, y: 648 },
|
||||
],
|
||||
]);
|
||||
expect(calls.map((c) => c.method)).toEqual([
|
||||
"DOM.getContentQuads",
|
||||
"DOM.getBoxModel",
|
||||
|
||||
@@ -0,0 +1,286 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry";
|
||||
import {
|
||||
clipPolygon,
|
||||
polygonArea,
|
||||
polygonBounds,
|
||||
projectAndClipRegion,
|
||||
projectUnitPoint,
|
||||
type Quad,
|
||||
rectPolygon,
|
||||
regionBounds,
|
||||
} from "../geometry";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
describe("frame geometry projection", () => {
|
||||
it("keeps region bounds separate from polygon area", () => {
|
||||
const region = [
|
||||
rectPolygon({ x: 0, y: 0, w: 10, h: 10 }),
|
||||
rectPolygon({ x: 20, y: 5, w: 5, h: 5 }),
|
||||
];
|
||||
expect(regionBounds(region)).toEqual({ x: 0, y: 0, width: 25, height: 10 });
|
||||
expect(polygonArea(region[0])).toBe(100);
|
||||
});
|
||||
|
||||
it("maps through the iframe content box instead of its border box", () => {
|
||||
const contentQuad: Quad = [
|
||||
{ x: 204, y: 306 },
|
||||
{ x: 604, y: 306 },
|
||||
{ x: 604, y: 506 },
|
||||
{ x: 204, y: 506 },
|
||||
];
|
||||
const projected = projectAndClipRegion(
|
||||
[rectPolygon({ x: 10, y: 20, w: 100, h: 40 })],
|
||||
[{ sourceViewport: { width: 200, height: 100 }, destinationQuad: contentQuad }],
|
||||
{ width: 1000, height: 800 },
|
||||
);
|
||||
expect(polygonBounds(projected.flat())).toEqual({ x: 224, y: 346, width: 200, height: 80 });
|
||||
});
|
||||
|
||||
it("uses a projective mapping for perspective-transformed iframe quads", () => {
|
||||
const quad: Quad = [
|
||||
{ x: 0, y: 0 },
|
||||
{ x: 200, y: 20 },
|
||||
{ x: 180, y: 140 },
|
||||
{ x: 20, y: 100 },
|
||||
];
|
||||
expect(projectUnitPoint({ x: 0, y: 0 }, quad)).toEqual(quad[0]);
|
||||
expect(projectUnitPoint({ x: 1, y: 0 }, quad)).toEqual(quad[1]);
|
||||
expect(projectUnitPoint({ x: 1, y: 1 }, quad)).toEqual(quad[2]);
|
||||
expect(projectUnitPoint({ x: 0, y: 1 }, quad)).toEqual(quad[3]);
|
||||
});
|
||||
|
||||
it("clips against the iframe polygon rather than its bounding box", () => {
|
||||
const frameQuad: Quad = [
|
||||
{ x: 50, y: 0 },
|
||||
{ x: 100, y: 50 },
|
||||
{ x: 50, y: 100 },
|
||||
{ x: 0, y: 50 },
|
||||
];
|
||||
expect(clipPolygon(rectPolygon({ x: 0, y: 0, w: 10, h: 10 }), frameQuad)).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not reapply same-target frame transforms around an OOPIF", async () => {
|
||||
const send = vi.fn(async (_tabId, method, params) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||
}
|
||||
if (method === "DOM.getBoxModel") {
|
||||
const backendNodeId = (params as { backendNodeId?: number }).backendNodeId;
|
||||
return backendNodeId === 20
|
||||
? { model: { content: [110, 70, 310, 70, 310, 170, 110, 170] } }
|
||||
: { model: { content: [100, 50, 400, 50, 400, 250, 100, 250] } };
|
||||
}
|
||||
throw new Error(`unexpected root command ${method}`);
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
throw new Error(`unexpected child command ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
};
|
||||
const projection = await resolveFrameProjection(
|
||||
cdp,
|
||||
{
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "same-process-parent",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4 },
|
||||
},
|
||||
{
|
||||
frameId: "oopif",
|
||||
parentFrameId: "same-process-parent",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 4, sessionId: "oopif-session" },
|
||||
},
|
||||
],
|
||||
},
|
||||
"oopif",
|
||||
);
|
||||
|
||||
expect(projection?.edges).toEqual([
|
||||
{
|
||||
sourceViewport: { width: 200, height: 100 },
|
||||
destinationQuad: [
|
||||
{ x: 110, y: 70 },
|
||||
{ x: 310, y: 70 },
|
||||
{ x: 310, y: 170 },
|
||||
{ x: 110, y: 170 },
|
||||
],
|
||||
destinationClips: [
|
||||
[
|
||||
{ x: 100, y: 50 },
|
||||
{ x: 400, y: 50 },
|
||||
{ x: 400, y: 250 },
|
||||
{ x: 100, y: 250 },
|
||||
],
|
||||
],
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("clips live OOPIF geometry at every frame boundary", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||
}
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||
}
|
||||
throw new Error(`unexpected root command ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[-50, -20, 250, -20, 250, 120, -50, 120]] };
|
||||
}
|
||||
throw new Error(`unexpected child command ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const geometry = await resolveNodeGeometry(cdp, 4, {
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
frameId: "child",
|
||||
backendNodeId: 101,
|
||||
});
|
||||
|
||||
expect(geometry).toMatchObject({
|
||||
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
||||
actionPoint: { x: 200, y: 150 },
|
||||
});
|
||||
});
|
||||
|
||||
it("reuses one frame graph while scrolling and projecting live geometry", async () => {
|
||||
const getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
}));
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||
}
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||
}
|
||||
throw new Error(`unexpected root command ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||
}
|
||||
throw new Error(`unexpected child command ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
getFrameGraph,
|
||||
};
|
||||
|
||||
await expect(
|
||||
resolveNodeGeometry(
|
||||
cdp,
|
||||
4,
|
||||
{
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
frameId: "child",
|
||||
backendNodeId: 101,
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
),
|
||||
).resolves.not.toMatchObject({ code: expect.any(String) });
|
||||
expect(getFrameGraph).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("clips live geometry to same-target iframe ancestors", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||
}
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||
}
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[50, 50, 350, 50, 350, 250, 50, 250]] };
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4 },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const geometry = await resolveNodeGeometry(cdp, 4, {
|
||||
target: { tabId: 4 },
|
||||
frameId: "child",
|
||||
backendNodeId: 101,
|
||||
});
|
||||
|
||||
expect(geometry).toMatchObject({
|
||||
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
||||
actionPoint: { x: 200, y: 150 },
|
||||
});
|
||||
});
|
||||
|
||||
it("fails closed when an OOPIF frame graph is unavailable", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(),
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
};
|
||||
|
||||
await expect(
|
||||
resolveNodeGeometry(cdp, 4, {
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
frameId: "child",
|
||||
backendNodeId: 101,
|
||||
}),
|
||||
).resolves.toMatchObject({ code: "cdp_failed" });
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { RefStore } from "@/session-manager/ref-store";
|
||||
import type { RequestHelpParams } from "@/transport/types";
|
||||
import { handleRequestHelp, type RequestHelpDeps, resetHelpLifecycleForTests } from "../human-loop";
|
||||
|
||||
@@ -50,11 +51,10 @@ function installHelpLifecycleChrome() {
|
||||
}
|
||||
|
||||
function fakeManager(sessionId: string, agentWindowId: number, tabId: number) {
|
||||
const refStore = new RefStore();
|
||||
const mgr = {
|
||||
get: (id: string) =>
|
||||
id === sessionId
|
||||
? { sessionId, agentWindowId, refStore: { resolve: () => null }, borrowedTabs: new Map() }
|
||||
: null,
|
||||
id === sessionId ? { sessionId, agentWindowId, refStore, borrowedTabs: new Map() } : null,
|
||||
findByWindowId: (wid: number) => (wid === agentWindowId ? { sessionId } : null),
|
||||
} as unknown as SessionManager;
|
||||
return mgr;
|
||||
@@ -339,26 +339,23 @@ describe("handleRequestHelp", () => {
|
||||
|
||||
ac.abort();
|
||||
await expect(pending).resolves.toMatchObject({ code: "cancelled" });
|
||||
expect(cdpSend).toHaveBeenCalledWith(
|
||||
5,
|
||||
"DOM.querySelectorAll",
|
||||
expect.objectContaining({ selector: "[data-bsk-help]" }),
|
||||
);
|
||||
expect(cdpSend).toHaveBeenCalledWith(
|
||||
6,
|
||||
"DOM.querySelectorAll",
|
||||
expect.objectContaining({ selector: "[data-bsk-help]" }),
|
||||
expect(cdpSend).not.toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
"DOM.removeAttribute",
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
|
||||
it("tags ref targets via CDP and reports them matched", async () => {
|
||||
it("resolves ref targets to explicit viewport rectangles without mutating the page", async () => {
|
||||
const refStore = new RefStore();
|
||||
refStore.set("e1", 42, { tabId: 5 });
|
||||
const mgr = {
|
||||
get: (id: string) =>
|
||||
id === "abcd"
|
||||
? {
|
||||
sessionId: "abcd",
|
||||
agentWindowId: 99,
|
||||
refStore: { resolve: () => 42 },
|
||||
refStore,
|
||||
borrowedTabs: new Map(),
|
||||
}
|
||||
: null,
|
||||
@@ -366,7 +363,17 @@ describe("handleRequestHelp", () => {
|
||||
} as unknown as SessionManager;
|
||||
const deps = baseDeps({
|
||||
cdp: {
|
||||
send: vi.fn(async () => ({ object: { objectId: "obj-1" } })),
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}),
|
||||
} as unknown as RequestHelpDeps["cdp"],
|
||||
});
|
||||
const res = await handleRequestHelp(
|
||||
@@ -375,14 +382,97 @@ describe("handleRequestHelp", () => {
|
||||
deps,
|
||||
);
|
||||
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
|
||||
expect(sentMsg.selectors).toContain('[data-bsk-help="0"]');
|
||||
expect(sentMsg.selectors).toEqual([]);
|
||||
expect(sentMsg.rects).toEqual([{ top: 20, left: 10, width: 100, height: 40 }]);
|
||||
expect(res).toMatchObject({
|
||||
outcome: "continued",
|
||||
tab_id: 5,
|
||||
resolved_targets: [{ matched: true, ref: "@e1" }],
|
||||
resolved_targets: [
|
||||
{
|
||||
matched: true,
|
||||
ref: "@e1",
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("projects an OOPIF ref through its live content quad for the help overlay", async () => {
|
||||
const mgr = {
|
||||
get: (id: string) =>
|
||||
id === "abcd"
|
||||
? {
|
||||
sessionId: "abcd",
|
||||
agentWindowId: 99,
|
||||
refStore: {
|
||||
resolveEntry: () => ({
|
||||
backendNodeId: 42,
|
||||
tabId: 5,
|
||||
frameId: "child",
|
||||
cdpSessionId: "child-session",
|
||||
generation: 1,
|
||||
}),
|
||||
},
|
||||
borrowedTabs: new Map(),
|
||||
}
|
||||
: null,
|
||||
findByWindowId: (wid: number) => (wid === 99 ? { sessionId: "abcd" } : null),
|
||||
} as unknown as SessionManager;
|
||||
const send = vi.fn(async (_tabId, method) => {
|
||||
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [204, 306, 604, 306, 604, 506, 204, 506] } };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
throw new Error(`unexpected root ${method}`);
|
||||
});
|
||||
const sendToTarget = vi.fn(async (_target, method) => {
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
throw new Error(`unexpected child ${method}`);
|
||||
});
|
||||
const deps = baseDeps({
|
||||
cdp: {
|
||||
send,
|
||||
sendToTarget,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 5 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 99,
|
||||
target: { tabId: 5, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
} as unknown as NonNullable<RequestHelpDeps["cdp"]>,
|
||||
});
|
||||
|
||||
const res = await handleRequestHelp(
|
||||
mgr,
|
||||
baseParams({ tab_id: 5, targets: [{ ref: "@e1" }] }),
|
||||
deps,
|
||||
);
|
||||
|
||||
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
|
||||
expect(sentMsg.rects).toEqual([{ top: 346, left: 224, width: 200, height: 80 }]);
|
||||
expect(res).toMatchObject({ resolved_targets: [{ matched: true, ref: "@e1" }] });
|
||||
expect(sendToTarget).toHaveBeenCalledWith(
|
||||
{ tabId: 5, sessionId: "child-session" },
|
||||
"DOM.getContentQuads",
|
||||
{ backendNodeId: 42 },
|
||||
);
|
||||
});
|
||||
|
||||
it("reports ref target unmatched when ref does not resolve", async () => {
|
||||
const deps = baseDeps();
|
||||
const res = await handleRequestHelp(
|
||||
@@ -398,16 +488,15 @@ describe("handleRequestHelp", () => {
|
||||
});
|
||||
|
||||
it("reports ref target unmatched when ref is for another tab", async () => {
|
||||
const refStore = new RefStore();
|
||||
refStore.set("e1", 42, { tabId: 4 });
|
||||
const mgr = {
|
||||
get: (id: string) =>
|
||||
id === "abcd"
|
||||
? {
|
||||
sessionId: "abcd",
|
||||
agentWindowId: 99,
|
||||
refStore: {
|
||||
resolve: (ref: string, opts: { tabId?: number }) =>
|
||||
ref === "e1" && opts.tabId === 4 ? 42 : null,
|
||||
},
|
||||
refStore,
|
||||
borrowedTabs: new Map(),
|
||||
}
|
||||
: null,
|
||||
|
||||
@@ -30,6 +30,9 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
||||
const sendImpl = async (tabId: number, method: string, params?: object) => {
|
||||
sent.push({ tabId, method, params });
|
||||
const h = handlers[method];
|
||||
if (!h && method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
if (!h) throw new Error(`unexpected CDP call ${method}`);
|
||||
return h(params);
|
||||
};
|
||||
@@ -164,6 +167,63 @@ describe("handleClick", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves frame refs in their CDP session and dispatches input in top coordinates", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
const fake = makeFakeCdp({
|
||||
"Input.dispatchMouseEvent": () => ({}),
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.getBoxModel": () => ({
|
||||
model: { content: [204, 306, 604, 306, 604, 506, 204, 506] },
|
||||
}),
|
||||
});
|
||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child-frame",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 99,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
}));
|
||||
const targetCalls: Array<{ sessionId?: string; method: string }> = [];
|
||||
fake.cdp.sendToTarget = vi.fn(async (target, method) => {
|
||||
targetCalls.push({ sessionId: target.sessionId, method });
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
throw new Error(`unexpected child CDP call ${method}`);
|
||||
}) as CdpRunner["sendToTarget"];
|
||||
|
||||
const res = await handleClick(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e3" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
// The iframe content box starts at (204, 306) after its border and is scaled 2x.
|
||||
expect(res).toMatchObject({ x: 324, y: 386 });
|
||||
expect(targetCalls).toEqual([
|
||||
{ sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" },
|
||||
{ sessionId: "child-session", method: "DOM.getContentQuads" },
|
||||
{ sessionId: "child-session", method: "Page.getLayoutMetrics" },
|
||||
]);
|
||||
expect(fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength(3);
|
||||
});
|
||||
|
||||
it("enables overlay bypass before mouse events when overlay blocks the click point", async () => {
|
||||
const order: string[] = [];
|
||||
const bypassOverlay = vi.fn(async (_tabId: number, enabled: boolean) => {
|
||||
|
||||
@@ -5,8 +5,10 @@ import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "@/lib/overlay-bridg
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import type { CdpRunner } from "@/tools/shared";
|
||||
import {
|
||||
buildFrameVomScene,
|
||||
buildVomScene,
|
||||
type CdpAxNode,
|
||||
captureVomObservation,
|
||||
handleGetHtml,
|
||||
handleObserve,
|
||||
handleScreenshot,
|
||||
@@ -62,6 +64,9 @@ function makeFakeCdp(handlers: Record<string, (params?: object) => unknown>) {
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
sent.push({ method, params });
|
||||
const handler = handlers[method];
|
||||
if (!handler && method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
if (!handler) throw new Error(`unexpected CDP call ${method}`);
|
||||
return handler(params);
|
||||
});
|
||||
@@ -508,6 +513,292 @@ describe("handleScreenshot overlay suppression", () => {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("buildVomScene", () => {
|
||||
it("merges multiple sibling and nested frames without mixing their targets", () => {
|
||||
const node = (
|
||||
backendNodeId: number,
|
||||
parentBackendNodeId: number | null,
|
||||
frameId: string,
|
||||
tag: string,
|
||||
attrs: Record<string, string> = {},
|
||||
): CapturedNode => ({
|
||||
backendNodeId,
|
||||
parentBackendNodeId,
|
||||
frameId,
|
||||
tag,
|
||||
attrs,
|
||||
rect: { x: 0, y: 0, w: 100, h: 30 },
|
||||
localRect: { x: 0, y: 0, w: 100, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
});
|
||||
const mainNodes = [
|
||||
node(1, null, "main", "body"),
|
||||
node(10, 1, "main", "iframe", { title: "First" }),
|
||||
node(20, 1, "main", "iframe", { title: "Second" }),
|
||||
];
|
||||
const firstNodes = [
|
||||
node(100, null, "first", "body"),
|
||||
node(101, 100, "first", "button", { "aria-label": "First action" }),
|
||||
];
|
||||
const secondNodes = [
|
||||
node(200, null, "second", "body"),
|
||||
node(201, 200, "second", "h2"),
|
||||
node(210, 200, "second", "iframe", { title: "Nested" }),
|
||||
];
|
||||
const nestedNodes = [
|
||||
node(300, null, "nested", "body"),
|
||||
node(301, 300, "nested", "input", { placeholder: "Nested field" }),
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
viewport: { width: 1200, height: 800 },
|
||||
nodes: mainNodes,
|
||||
iframeNodes: new Map([
|
||||
[10, firstNodes],
|
||||
[20, secondNodes],
|
||||
[210, nestedNodes],
|
||||
]),
|
||||
frameNodes: new Map([
|
||||
["main", mainNodes],
|
||||
["first", firstNodes],
|
||||
["second", secondNodes],
|
||||
["nested", nestedNodes],
|
||||
]),
|
||||
frameOwnerBackendNodeIds: new Map([
|
||||
["first", 10],
|
||||
["second", 20],
|
||||
["nested", 210],
|
||||
]),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
const axNode = (
|
||||
nodeId: string,
|
||||
backendDOMNodeId: number,
|
||||
role: string,
|
||||
parentId?: string,
|
||||
name?: string,
|
||||
): CdpAxNode => ({
|
||||
nodeId,
|
||||
backendDOMNodeId,
|
||||
role: { type: "role", value: role },
|
||||
...(parentId ? { parentId } : {}),
|
||||
...(name ? { name: { type: "computedString", value: name } } : {}),
|
||||
});
|
||||
const result = buildFrameVomScene(
|
||||
[
|
||||
{
|
||||
frameId: "main",
|
||||
contextScopeId: "main",
|
||||
target: { tabId: 7 },
|
||||
domNodes: mainNodes,
|
||||
axNodes: [
|
||||
axNode("root", 1, "RootWebArea"),
|
||||
axNode("frame-1", 10, "Iframe", "root", "First"),
|
||||
axNode("frame-2", 20, "Iframe", "root", "Second"),
|
||||
],
|
||||
},
|
||||
{
|
||||
frameId: "first",
|
||||
contextScopeId: "first",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 7, sessionId: "session-first" },
|
||||
domNodes: firstNodes,
|
||||
axNodes: [
|
||||
axNode("root", 100, "RootWebArea"),
|
||||
axNode("button", 101, "button", "root", "First action"),
|
||||
],
|
||||
},
|
||||
{
|
||||
frameId: "second",
|
||||
contextScopeId: "second",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 7, sessionId: "session-second" },
|
||||
domNodes: secondNodes,
|
||||
axNodes: [
|
||||
axNode("root", 200, "RootWebArea"),
|
||||
axNode("heading", 201, "heading", "root", "Second heading"),
|
||||
axNode("nested-frame", 210, "Iframe", "root", "Nested"),
|
||||
],
|
||||
},
|
||||
{
|
||||
frameId: "nested",
|
||||
contextScopeId: "nested",
|
||||
parentFrameId: "second",
|
||||
ownerBackendNodeId: 210,
|
||||
target: { tabId: 7, sessionId: "session-nested" },
|
||||
domNodes: nestedNodes,
|
||||
axNodes: [
|
||||
axNode("root", 300, "RootWebArea"),
|
||||
axNode("input", 301, "textbox", "root", "Nested field"),
|
||||
],
|
||||
},
|
||||
],
|
||||
captured,
|
||||
);
|
||||
|
||||
const rendered = renderVom(result);
|
||||
expect(rendered.text).toContain('Iframe "First"');
|
||||
expect(rendered.text).toContain('@e1 button "First action"');
|
||||
expect(rendered.text).toContain('heading "Second heading"');
|
||||
expect(rendered.text).toContain('Iframe "Nested"');
|
||||
expect(rendered.text).toContain('textbox "Nested field"');
|
||||
expect(rendered.refs.find((ref) => ref.backendNodeId === 101)?.frameId).toBe("first");
|
||||
expect(rendered.refs.find((ref) => ref.backendNodeId === 301)?.frameId).toBe("nested");
|
||||
|
||||
const firstAction = result.nodes.find((item) => item.backendNodeId === 101);
|
||||
const nestedInput = result.nodes.find((item) => item.backendNodeId === 301);
|
||||
expect(firstAction).toMatchObject({ frameId: "first", contextScopeId: "first" });
|
||||
expect(nestedInput).toMatchObject({ frameId: "nested", contextScopeId: "nested" });
|
||||
expect(nestedInput?.parentId).toBe(result.nodes.find((item) => item.backendNodeId === 210)?.id);
|
||||
});
|
||||
|
||||
it("does not place a child document at the page root when its frame boundary is unresolved", () => {
|
||||
const childNode: CapturedNode = {
|
||||
backendNodeId: 101,
|
||||
parentBackendNodeId: null,
|
||||
frameId: "child",
|
||||
tag: "button",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 100, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
};
|
||||
const captured: CapturedViewModel = {
|
||||
viewport: { width: 800, height: 600 },
|
||||
nodes: [],
|
||||
iframeNodes: new Map(),
|
||||
frameNodes: new Map([["child", [childNode]]]),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
|
||||
const result = buildFrameVomScene(
|
||||
[
|
||||
{
|
||||
frameId: "main",
|
||||
contextScopeId: "main",
|
||||
target: { tabId: 7 },
|
||||
domNodes: [],
|
||||
axNodes: [],
|
||||
},
|
||||
{
|
||||
frameId: "child",
|
||||
contextScopeId: "child",
|
||||
parentFrameId: "main",
|
||||
target: { tabId: 7 },
|
||||
domNodes: [childNode],
|
||||
axNodes: [
|
||||
{
|
||||
nodeId: "button",
|
||||
frameId: "child",
|
||||
backendDOMNodeId: 101,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Child action" },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
captured,
|
||||
);
|
||||
|
||||
expect(result.nodes).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps AX-only frame content attached to its iframe boundary", () => {
|
||||
const mainNodes: CapturedNode[] = [
|
||||
{
|
||||
backendNodeId: 1,
|
||||
parentBackendNodeId: null,
|
||||
frameId: "main",
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 800, h: 600 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: 1,
|
||||
frameId: "main",
|
||||
tag: "iframe",
|
||||
attrs: { title: "Remote" },
|
||||
rect: { x: 100, y: 100, w: 400, h: 300 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
viewport: { width: 800, height: 600 },
|
||||
nodes: mainNodes,
|
||||
iframeNodes: new Map(),
|
||||
frameNodes: new Map([
|
||||
["main", mainNodes],
|
||||
["child", []],
|
||||
]),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
|
||||
const result = buildFrameVomScene(
|
||||
[
|
||||
{
|
||||
frameId: "main",
|
||||
contextScopeId: "main",
|
||||
target: { tabId: 7 },
|
||||
domNodes: mainNodes,
|
||||
axNodes: [
|
||||
{
|
||||
nodeId: "main-root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
},
|
||||
{
|
||||
nodeId: "frame-owner",
|
||||
parentId: "main-root",
|
||||
backendDOMNodeId: 10,
|
||||
role: { type: "role", value: "Iframe" },
|
||||
name: { type: "computedString", value: "Remote" },
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
frameId: "child",
|
||||
contextScopeId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 7, sessionId: "child-session" },
|
||||
domNodes: [],
|
||||
axNodes: [
|
||||
{
|
||||
nodeId: "child-root",
|
||||
backendDOMNodeId: 100,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
},
|
||||
{
|
||||
nodeId: "child-button",
|
||||
parentId: "child-root",
|
||||
backendDOMNodeId: 101,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "AX fallback action" },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
captured,
|
||||
);
|
||||
|
||||
const childButton = result.nodes.find((node) => node.backendNodeId === 101);
|
||||
expect(childButton).toMatchObject({ frameId: "child", contextScopeId: "child" });
|
||||
expect(childButton?.parentId).toBe(result.nodes.find((node) => node.backendNodeId === 10)?.id);
|
||||
expect(renderVom(result).text).toContain('@e1 button "AX fallback action"');
|
||||
});
|
||||
|
||||
it("joins AX semantics with captured geometry by backendDOMNodeId", () => {
|
||||
const axNodes: CdpAxNode[] = [
|
||||
{
|
||||
@@ -816,6 +1107,7 @@ describe("buildVomScene", () => {
|
||||
parentId: "2",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "x", value: "输入密码" },
|
||||
value: { value: "iframe-secret" },
|
||||
backendDOMNodeId: 202,
|
||||
properties: [{ name: "inputType", value: { value: "password" } }],
|
||||
},
|
||||
@@ -870,6 +1162,10 @@ describe("buildVomScene", () => {
|
||||
|
||||
expect(passwordNodes).toHaveLength(1);
|
||||
expect(passwordNodes[0]).toEqual(expect.objectContaining({ id: 202, sensitive: true }));
|
||||
expect(passwordNodes[0].value).toBeUndefined();
|
||||
const rendered = renderVom(buildVomScene(axNodes, captured)).text;
|
||||
expect(rendered).toContain('textbox "输入密码" [filled] ="•••"');
|
||||
expect(rendered).not.toContain("iframe-secret");
|
||||
});
|
||||
|
||||
it("keeps unnamed iframe controls but skips unnamed iframe links", () => {
|
||||
@@ -1034,7 +1330,7 @@ describe("buildVomScene", () => {
|
||||
expect(rendered).toContain(' @e2 textbox "验证码"');
|
||||
});
|
||||
|
||||
it("does not mark non-password inputs sensitive from password-like labels", () => {
|
||||
it("marks current-password autocomplete as sensitive even for text inputs", () => {
|
||||
const axNodes: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "1",
|
||||
@@ -1069,7 +1365,7 @@ describe("buildVomScene", () => {
|
||||
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
id: 10,
|
||||
sensitive: false,
|
||||
sensitive: true,
|
||||
}),
|
||||
);
|
||||
});
|
||||
@@ -1364,7 +1660,9 @@ describe("buildVomScene", () => {
|
||||
);
|
||||
const rendered = renderVom(scene);
|
||||
expect(rendered.text).toContain('@e1 button "close"');
|
||||
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
|
||||
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
|
||||
{ ref: "e1", backendNodeId: 20 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not promote a clickable container that wraps a real interactive control", () => {
|
||||
@@ -1508,7 +1806,9 @@ describe("buildVomScene", () => {
|
||||
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic");
|
||||
const rendered = renderVom(scene);
|
||||
expect(rendered.text).toContain('@e1 button "收藏"');
|
||||
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
|
||||
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
|
||||
{ ref: "e1", backendNodeId: 20 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("builds active scope blocks from active aria-controls relationships", () => {
|
||||
@@ -2609,10 +2909,360 @@ describe("handleSnapshot", () => {
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function secretsSnapshotReply(secrets: {
|
||||
email: string;
|
||||
password: string;
|
||||
otp: string;
|
||||
card: string;
|
||||
}) {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"input",
|
||||
"position",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"type",
|
||||
"text",
|
||||
"password",
|
||||
"autocomplete",
|
||||
"one-time-code",
|
||||
"cc-number",
|
||||
"value",
|
||||
secrets.email,
|
||||
secrets.password,
|
||||
secrets.otp,
|
||||
secrets.card,
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const style = [i("static"), i("auto"), i("auto")];
|
||||
return {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 1, 1, 1],
|
||||
nodeName: [i("html"), i("body"), i("input"), i("input"), i("input"), i("input")],
|
||||
backendNodeId: [10, 11, 12, 13, 14, 15],
|
||||
attributes: [
|
||||
[],
|
||||
[],
|
||||
[i("type"), i("text"), i("value"), i(secrets.email)],
|
||||
[i("type"), i("password"), i("value"), i(secrets.password)],
|
||||
[
|
||||
i("type"),
|
||||
i("text"),
|
||||
i("autocomplete"),
|
||||
i("one-time-code"),
|
||||
i("value"),
|
||||
i(secrets.otp),
|
||||
],
|
||||
[
|
||||
i("type"),
|
||||
i("text"),
|
||||
i("autocomplete"),
|
||||
i("cc-number"),
|
||||
i("value"),
|
||||
i(secrets.card),
|
||||
],
|
||||
],
|
||||
inputValue: {
|
||||
index: [2, 3, 4, 5],
|
||||
value: [i(secrets.email), i(secrets.password), i(secrets.otp), i(secrets.card)],
|
||||
},
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 3, 4, 5],
|
||||
styles: [style, style, style, style, style],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[10, 10, 200, 32],
|
||||
[10, 50, 200, 32],
|
||||
[10, 90, 200, 32],
|
||||
[10, 130, 200, 32],
|
||||
],
|
||||
paintOrders: [0, 1, 2, 3, 4],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
const VP_METRICS = {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
|
||||
function makeFrameAwareDeps() {
|
||||
const strings = ["html", "body", "iframe", "button", "title", "Remote", "static", "auto"];
|
||||
const i = (value: string) => strings.indexOf(value);
|
||||
const styles = [i("static"), i("auto"), i("auto")];
|
||||
const snapshot = {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
frameId: "main",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("iframe")],
|
||||
backendNodeId: [10, 11, 12],
|
||||
attributes: [[], [], [i("title"), i("Remote")]],
|
||||
contentDocumentIndex: { index: [2], value: [1] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1, 2],
|
||||
styles: [styles, styles, styles],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[0, 0, 1000, 800],
|
||||
[100, 100, 400, 300],
|
||||
],
|
||||
paintOrders: [0, 0, 1],
|
||||
},
|
||||
},
|
||||
{
|
||||
frameId: "child",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("button")],
|
||||
backendNodeId: [20, 21, 22],
|
||||
attributes: [[], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1, 2],
|
||||
styles: [styles, styles, styles],
|
||||
bounds: [
|
||||
[0, 0, 400, 300],
|
||||
[0, 0, 400, 300],
|
||||
[20, 30, 120, 40],
|
||||
],
|
||||
paintOrders: [0, 0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const mainAx: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "main-root",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 11,
|
||||
childIds: ["frame-owner"],
|
||||
},
|
||||
{
|
||||
nodeId: "frame-owner",
|
||||
parentId: "main-root",
|
||||
role: { type: "role", value: "Iframe" },
|
||||
name: { type: "computedString", value: "Remote" },
|
||||
backendDOMNodeId: 12,
|
||||
},
|
||||
];
|
||||
const childAx: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "child-root",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 21,
|
||||
childIds: ["child-button"],
|
||||
},
|
||||
{
|
||||
nodeId: "child-button",
|
||||
parentId: "child-root",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Frame action" },
|
||||
backendDOMNodeId: 22,
|
||||
},
|
||||
];
|
||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "Page.getLayoutMetrics") return VP_METRICS;
|
||||
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: mainAx };
|
||||
throw new Error(`unexpected root CDP method ${method}`);
|
||||
});
|
||||
const sendToTarget = vi.fn(async (_target, method: string) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: childAx };
|
||||
throw new Error(`unexpected child CDP method ${method}`);
|
||||
});
|
||||
const cdp = {
|
||||
send: send as unknown as CdpRunner["send"],
|
||||
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 12,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
trackSessionTab: vi.fn(),
|
||||
} satisfies CdpRunner;
|
||||
return {
|
||||
cdp,
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
it("exposes the frame-aware observation result for recording", async () => {
|
||||
const ax: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "root",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 11,
|
||||
childIds: ["password"],
|
||||
},
|
||||
{
|
||||
nodeId: "password",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Password" },
|
||||
value: { value: "hunter2" },
|
||||
backendDOMNodeId: 13,
|
||||
},
|
||||
];
|
||||
const deps = makeOverlayDeps(ax, loginSnapshotReply(), VP_METRICS);
|
||||
|
||||
const result = await captureVomObservation(deps.cdp, 4, "https://example.com", {
|
||||
redactValues: true,
|
||||
});
|
||||
|
||||
expect(result.text).not.toContain("hunter2");
|
||||
expect(result.text).toContain("•••");
|
||||
expect(result.refs[0]).toMatchObject({
|
||||
backendNodeId: 13,
|
||||
role: "textbox",
|
||||
name: "Password",
|
||||
line: expect.any(Number),
|
||||
});
|
||||
expect(result.rootFrameId).toBe("root");
|
||||
expect(result).not.toHaveProperty("frameDocuments");
|
||||
expect(result.frames).toEqual([{ frameId: "root", target: { tabId: 4 } }]);
|
||||
expect(result.matchNodes.find((node) => node.backendNodeId === 13)).toMatchObject({
|
||||
frameId: "root",
|
||||
backendNodeId: 13,
|
||||
tag: "input",
|
||||
rect: { x: 400, y: 300, w: 200, h: 40 },
|
||||
localRect: { x: 400, y: 300, w: 200, h: 40 },
|
||||
});
|
||||
});
|
||||
|
||||
it("does not leak form secrets through the record-safe observation payload", async () => {
|
||||
const secrets = {
|
||||
email: "user@example.com",
|
||||
password: "hunter2",
|
||||
otp: "847291",
|
||||
card: "4111111111111111",
|
||||
};
|
||||
const ax: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "root",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 11,
|
||||
childIds: ["email", "password", "otp", "card"],
|
||||
},
|
||||
{
|
||||
nodeId: "email",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Email" },
|
||||
value: { value: secrets.email },
|
||||
backendDOMNodeId: 12,
|
||||
},
|
||||
{
|
||||
nodeId: "password",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Password" },
|
||||
value: { value: secrets.password },
|
||||
backendDOMNodeId: 13,
|
||||
},
|
||||
{
|
||||
nodeId: "otp",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Code" },
|
||||
value: { value: secrets.otp },
|
||||
backendDOMNodeId: 14,
|
||||
},
|
||||
{
|
||||
nodeId: "card",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Card" },
|
||||
value: { value: secrets.card },
|
||||
backendDOMNodeId: 15,
|
||||
},
|
||||
];
|
||||
const deps = makeOverlayDeps(ax, secretsSnapshotReply(secrets), VP_METRICS);
|
||||
|
||||
const result = await captureVomObservation(deps.cdp, 4, "https://example.com", {
|
||||
redactValues: true,
|
||||
});
|
||||
const dumped = JSON.stringify(result);
|
||||
|
||||
expect(result).not.toHaveProperty("frameDocuments");
|
||||
expect(dumped).not.toContain(secrets.email);
|
||||
expect(dumped).not.toContain(secrets.password);
|
||||
expect(dumped).not.toContain(secrets.otp);
|
||||
expect(dumped).not.toContain(secrets.card);
|
||||
expect(dumped).not.toContain("formValue");
|
||||
expect(dumped).not.toContain("formDefaultValue");
|
||||
expect(dumped).not.toContain("axNodes");
|
||||
expect(dumped).not.toContain("domNodes");
|
||||
expect(dumped).not.toContain("attrs");
|
||||
expect(result.text).toContain("•••");
|
||||
});
|
||||
|
||||
it("keeps frames and rendered refs on the same frame identity", async () => {
|
||||
const deps = makeFrameAwareDeps();
|
||||
|
||||
const result = await captureVomObservation(deps.cdp, 4, "https://example.com");
|
||||
|
||||
expect(result.rootFrameId).toBe("main");
|
||||
expect(result.frames).toHaveLength(2);
|
||||
expect(result.frames.find((frame) => frame.frameId === "child")).toEqual({
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 12,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
});
|
||||
expect(result.matchNodes.find((node) => node.backendNodeId === 22)).toMatchObject({
|
||||
frameId: "child",
|
||||
tag: "button",
|
||||
localRect: { x: 20, y: 30, w: 120, h: 40 },
|
||||
});
|
||||
expect(result.refs.find((ref) => ref.backendNodeId === 22)).toMatchObject({
|
||||
frameId: "child",
|
||||
name: "Frame action",
|
||||
});
|
||||
});
|
||||
|
||||
it("stores iframe refs with their owning CDP session", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
const deps = makeFrameAwareDeps();
|
||||
|
||||
const result = await handleSnapshot(sm, { session_id: "aa11" }, deps);
|
||||
|
||||
if ("code" in result) throw new Error(`unexpected error: ${JSON.stringify(result)}`);
|
||||
expect(result.text).toContain('@e1 button "Frame action"');
|
||||
const frameRef = [...ctx.refStore.entries()].find(([, entry]) => entry.backendNodeId === 22);
|
||||
expect(frameRef?.[1]).toMatchObject({
|
||||
tabId: 4,
|
||||
frameId: "child",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
});
|
||||
|
||||
it("renders a blocking login overlay as the focused top layer", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
@@ -3123,6 +3773,32 @@ describe("handleGetHtml", () => {
|
||||
expect(deps.send).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("reads a frame ref through its child CDP session", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e7", 4242, {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
const deps = makeDeps({});
|
||||
const sendToTarget = vi.fn(async (target, method, params) => {
|
||||
expect(target).toEqual({ tabId: 4, sessionId: "child-session" });
|
||||
expect(method).toBe("DOM.getOuterHTML");
|
||||
expect(params).toEqual({ backendNodeId: 4242 });
|
||||
return { outerHTML: "<button>frame</button>" };
|
||||
});
|
||||
(deps.cdp as CdpRunner).sendToTarget = sendToTarget as unknown as NonNullable<
|
||||
CdpRunner["sendToTarget"]
|
||||
>;
|
||||
|
||||
const res = await handleGetHtml(sm, { session_id: "aa11", ref: "@e7" }, deps);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.html).toBe("<button>frame</button>");
|
||||
expect(deps.send).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns not_found when a ref belongs to another tab", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
|
||||
@@ -66,6 +66,52 @@ function makeApis(
|
||||
}
|
||||
|
||||
describe("handleSessionStop with auto-return", () => {
|
||||
it("leaves the session untouched when cancellation arrived before teardown", async () => {
|
||||
const aw = fakeAgentWindow([100]);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
await sm.start("aa11");
|
||||
const cdp = { detachSession: vi.fn(async () => {}) };
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ cdp, signal: controller.signal },
|
||||
);
|
||||
|
||||
expect(res).toMatchObject({ code: "cancelled" });
|
||||
expect(sm.has("aa11")).toBe(true);
|
||||
expect(cdp.detachSession).not.toHaveBeenCalled();
|
||||
expect(aw.remove).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not close the Agent Window when cancellation lands during CDP detach", async () => {
|
||||
const aw = fakeAgentWindow([100]);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
await sm.start("aa11");
|
||||
const controller = new AbortController();
|
||||
let finishDetach: () => void = () => {};
|
||||
const detach = new Promise<void>((resolve) => {
|
||||
finishDetach = resolve;
|
||||
});
|
||||
const cdp = { detachSession: vi.fn(() => detach) };
|
||||
|
||||
const stopping = handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ cdp, signal: controller.signal },
|
||||
);
|
||||
await vi.waitFor(() => expect(cdp.detachSession).toHaveBeenCalledWith("aa11"));
|
||||
controller.abort();
|
||||
finishDetach();
|
||||
|
||||
const res = await stopping;
|
||||
expect(res).toMatchObject({ code: "cancelled" });
|
||||
expect(sm.has("aa11")).toBe(true);
|
||||
expect(aw.remove).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns every borrowed tab and closes the Agent Window in the right order", async () => {
|
||||
const aw = fakeAgentWindow([100]);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
|
||||
@@ -46,6 +46,25 @@ describe("lookupSnapshotRef", () => {
|
||||
});
|
||||
|
||||
describe("resolveSnapshotRef", () => {
|
||||
it("preserves the frame and child session needed to route iframe refs", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
|
||||
const expected = {
|
||||
backendNodeId: 1234,
|
||||
refKey: "e3",
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
};
|
||||
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
||||
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
||||
});
|
||||
|
||||
it("returns not_found for unknown ref", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
|
||||
@@ -250,9 +250,10 @@ const DEFAULT_NOTIFICATION_COPY: BorrowNotificationCopy = {
|
||||
// URL injection-eligibility test
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const CHROME_WEB_STORE_RES = [
|
||||
const EXTENSION_STORE_RES = [
|
||||
/^https:\/\/chrome\.google\.com\/webstore/i,
|
||||
/^https:\/\/chromewebstore\.google\.com/i,
|
||||
/^https:\/\/microsoftedge\.microsoft\.com\/addons/i,
|
||||
];
|
||||
|
||||
/**
|
||||
@@ -264,14 +265,15 @@ const CHROME_WEB_STORE_RES = [
|
||||
* - `ftp://` no longer hosts content scripts reliably in modern Chrome.
|
||||
* - `about:` / `chrome:` / `chrome-extension:` / `edge:` / `devtools:` /
|
||||
* `view-source:` / `data:` / `blob:` are blocked by the platform.
|
||||
* - The Chrome Web Store (both legacy and new domains) is also blocked.
|
||||
* - Extension storefronts are blocked by their own browser: the Chrome Web
|
||||
* Store (legacy and new domains) under Chrome, and Edge Add-ons under Edge.
|
||||
*
|
||||
* Mirrors the implicit scheme list documented in
|
||||
* https://developer.chrome.com/docs/extensions/develop/concepts/match-patterns.
|
||||
*/
|
||||
export function isInjectableContentScriptUrl(url: string | undefined): boolean {
|
||||
if (!url) return false;
|
||||
for (const re of CHROME_WEB_STORE_RES) {
|
||||
for (const re of EXTENSION_STORE_RES) {
|
||||
if (re.test(url)) return false;
|
||||
}
|
||||
return /^https?:\/\//i.test(url);
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
export {
|
||||
type CaptureVomObservationOptions,
|
||||
captureVomObservation,
|
||||
} from "./observation";
|
||||
export {
|
||||
type CaptureVomFrame,
|
||||
type CaptureVomMatchNode,
|
||||
type CaptureVomObservationResult,
|
||||
} from "./vom/record-safe-observation";
|
||||
@@ -290,6 +290,7 @@ export class ToolDispatcher {
|
||||
await this.releaseHoverLatch((req.params as SessionStopParams).session_id);
|
||||
return handleSessionStop(this.sessions, req.params as SessionStopParams, {
|
||||
cdp: this.cdp,
|
||||
signal,
|
||||
});
|
||||
}
|
||||
case "tool.tab_list":
|
||||
|
||||
@@ -1,9 +1,15 @@
|
||||
// Shared CDP helpers for scrolling nodes into view and computing
|
||||
// viewport-space bounding rectangles. Used by interaction tools (click
|
||||
// centre) and observation tools (element screenshot clip).
|
||||
// Resolve and scroll nodes inside one CDP target. Cross-frame projection
|
||||
// belongs to frame-geometry.ts.
|
||||
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { rpcError } from "./errors";
|
||||
import {
|
||||
parseCdpQuad,
|
||||
polygonBounds,
|
||||
type Region,
|
||||
rectPolygon,
|
||||
type ViewportRect,
|
||||
} from "./geometry";
|
||||
import { type CdpRunner, isRpcError } from "./shared";
|
||||
|
||||
const ELEMENT_NOT_VISIBLE_MESSAGE =
|
||||
@@ -13,45 +19,6 @@ function elementNotVisibleError(): RpcError {
|
||||
return rpcError("permission_denied", "element_not_visible", ELEMENT_NOT_VISIBLE_MESSAGE);
|
||||
}
|
||||
|
||||
/** Viewport-space axis-aligned bounding box for CDP `Page.captureScreenshot` clip. */
|
||||
export interface ViewportRect {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the axis-aligned bounding box of an 8-double CDP content
|
||||
* quad / box-model polygon. Exported for unit tests.
|
||||
*/
|
||||
export function quadBoundingRect(quad: number[]): ViewportRect | null {
|
||||
if (quad.length !== 8) return null;
|
||||
const xs = [quad[0], quad[2], quad[4], quad[6]];
|
||||
const ys = [quad[1], quad[3], quad[5], quad[7]];
|
||||
const minX = Math.min(...xs);
|
||||
const maxX = Math.max(...xs);
|
||||
const minY = Math.min(...ys);
|
||||
const maxY = Math.max(...ys);
|
||||
const width = maxX - minX;
|
||||
const height = maxY - minY;
|
||||
if (width <= 0 || height <= 0) return null;
|
||||
return { x: minX, y: minY, width, height };
|
||||
}
|
||||
|
||||
function rectToQuad(rect: ViewportRect): number[] {
|
||||
return [
|
||||
rect.x,
|
||||
rect.y,
|
||||
rect.x + rect.width,
|
||||
rect.y,
|
||||
rect.x + rect.width,
|
||||
rect.y + rect.height,
|
||||
rect.x,
|
||||
rect.y + rect.height,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve `backendNodeId` → CDP `objectId` so we can invoke
|
||||
* `Runtime.callFunctionOn` against the live JS object.
|
||||
@@ -112,48 +79,24 @@ export async function scrollNodeIntoView(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Viewport-space bounding box for `backendNodeId`. Prefers
|
||||
* `DOM.getContentQuads`, then `DOM.getBoxModel`, then the union of
|
||||
* visible descendant client rects for zero-size overflow containers.
|
||||
*/
|
||||
export async function nodeBoundingRect(
|
||||
function validCdpRegion(quads: number[][] | undefined): Region {
|
||||
return (quads ?? []).flatMap((raw) => {
|
||||
const polygon = parseCdpQuad(raw);
|
||||
return polygon && polygonBounds(polygon) ? [polygon] : [];
|
||||
});
|
||||
}
|
||||
|
||||
async function resolveVisibleContentRegion(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
backendNodeId: number,
|
||||
): Promise<ViewportRect | RpcError> {
|
||||
const polygon = await visibleContentPolygon(cdp, tabId, backendNodeId);
|
||||
if (isRpcError(polygon)) return polygon;
|
||||
// visibleContentPolygon only returns quads with positive area.
|
||||
return quadBoundingRect(polygon)!;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the centroid of a CDP content quad. Quads are reported as
|
||||
* 8 doubles in clockwise order: (x1, y1, x2, y2, x3, y3, x4, y4).
|
||||
*/
|
||||
export function quadCentre(quad: number[]): { x: number; y: number } {
|
||||
const x = (quad[0] + quad[4]) / 2;
|
||||
const y = (quad[1] + quad[5]) / 2;
|
||||
return { x, y };
|
||||
}
|
||||
|
||||
/** Centre of a `DOM.getBoxModel` `content` polygon (also 8 doubles). */
|
||||
export function boxCentre(box: number[]): { x: number; y: number } {
|
||||
return quadCentre(box);
|
||||
}
|
||||
|
||||
async function visibleContentPolygon(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
backendNodeId: number,
|
||||
): Promise<number[] | RpcError> {
|
||||
): Promise<Region | RpcError> {
|
||||
try {
|
||||
const quads = await cdp.send<{ quads?: number[][] }>(tabId, "DOM.getContentQuads", {
|
||||
backendNodeId,
|
||||
});
|
||||
const quad = quads.quads?.find((q) => q.length === 8 && quadBoundingRect(q) !== null);
|
||||
if (quad) return quad;
|
||||
const visible = validCdpRegion(quads.quads);
|
||||
if (visible.length > 0) return visible;
|
||||
} catch (err) {
|
||||
console.debug("[bsk element-geometry] getContentQuads failed", err);
|
||||
}
|
||||
@@ -162,15 +105,34 @@ async function visibleContentPolygon(
|
||||
backendNodeId,
|
||||
});
|
||||
const content = box.model?.content;
|
||||
if (content && content.length === 8 && quadBoundingRect(content)) return content;
|
||||
const fallback = validCdpRegion(content ? [content] : undefined);
|
||||
if (fallback.length > 0) return fallback;
|
||||
} catch (err) {
|
||||
console.debug("[bsk element-geometry] getBoxModel failed", err);
|
||||
}
|
||||
const descendantRect = await descendantBoundingRect(cdp, tabId, backendNodeId);
|
||||
if (descendantRect) return rectToQuad(descendantRect);
|
||||
if (descendantRect) {
|
||||
return [
|
||||
rectPolygon({
|
||||
x: descendantRect.x,
|
||||
y: descendantRect.y,
|
||||
w: descendantRect.width,
|
||||
h: descendantRect.height,
|
||||
}),
|
||||
];
|
||||
}
|
||||
return elementNotVisibleError();
|
||||
}
|
||||
|
||||
/** Resolve every visible content region in the node's target-local viewport. */
|
||||
export async function nodeContentRegion(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
backendNodeId: number,
|
||||
): Promise<Region | RpcError> {
|
||||
return resolveVisibleContentRegion(cdp, tabId, backendNodeId);
|
||||
}
|
||||
|
||||
async function descendantBoundingRect(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
@@ -241,20 +203,3 @@ function parseViewportRect(value: unknown): ViewportRect | null {
|
||||
if (![x, y, width, height].every(Number.isFinite) || width <= 0 || height <= 0) return null;
|
||||
return { x, y, width, height };
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the click point for `backendNodeId`. Prefers
|
||||
* `DOM.getContentQuads` (richer for rotated / transformed elements);
|
||||
* falls back to `DOM.getBoxModel`, then visible descendant bounds.
|
||||
* Returns `permission_denied` with "element not visible" when every
|
||||
* geometry path comes back empty / fails.
|
||||
*/
|
||||
export async function nodeCentre(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
backendNodeId: number,
|
||||
): Promise<{ x: number; y: number } | RpcError> {
|
||||
const polygon = await visibleContentPolygon(cdp, tabId, backendNodeId);
|
||||
if (isRpcError(polygon)) return polygon;
|
||||
return quadCentre(polygon);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry";
|
||||
import {
|
||||
clipPolygon,
|
||||
type GeometryProjection,
|
||||
type Point,
|
||||
type Polygon,
|
||||
type ProjectiveEdge,
|
||||
parseCdpQuad,
|
||||
polygonArea,
|
||||
polygonCentroid,
|
||||
projectRegionToViewport,
|
||||
type Quad,
|
||||
type Region,
|
||||
rectPolygon,
|
||||
regionBounds,
|
||||
type Size,
|
||||
type ViewportRect,
|
||||
} from "./geometry";
|
||||
import { type CdpRunner, cdpRunnerForTarget, isRpcError, sendToCdpTarget } from "./shared";
|
||||
|
||||
export interface NodeAddress {
|
||||
target: CdpTarget;
|
||||
backendNodeId: number;
|
||||
frameId?: string;
|
||||
}
|
||||
|
||||
export interface ResolvedNodeGeometry {
|
||||
topVisibleRegions: Region;
|
||||
topBounds: ViewportRect;
|
||||
actionPoint: Point;
|
||||
}
|
||||
|
||||
function geometryError(message: string): RpcError {
|
||||
return { code: "cdp_failed", message };
|
||||
}
|
||||
|
||||
function frameMap(graph: CdpFrameGraph): Map<string, CdpFrame> {
|
||||
return new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
||||
}
|
||||
|
||||
function targetRootFrame(byId: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
|
||||
const seen = new Set<string>();
|
||||
let current = frame;
|
||||
while (current.parentFrameId) {
|
||||
if (seen.has(current.frameId)) return null;
|
||||
seen.add(current.frameId);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent || parent.target.sessionId !== current.target.sessionId) break;
|
||||
current = parent;
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
function frameAncestry(graph: CdpFrameGraph, frameId: string): CdpFrame[] | null {
|
||||
const byId = frameMap(graph);
|
||||
const path: CdpFrame[] = [];
|
||||
const seen = new Set<string>();
|
||||
let current = byId.get(frameId);
|
||||
if (!current) return null;
|
||||
while (current.parentFrameId) {
|
||||
if (seen.has(current.frameId)) return null;
|
||||
seen.add(current.frameId);
|
||||
path.push(current);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent) return null;
|
||||
current = parent;
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise<Size | null> {
|
||||
const metrics = await sendToCdpTarget<{
|
||||
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
}>(cdp, target, "Page.getLayoutMetrics", {});
|
||||
const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
const width = viewport.clientWidth ?? 0;
|
||||
const height = viewport.clientHeight ?? 0;
|
||||
return width > 0 && height > 0 ? { width, height } : null;
|
||||
}
|
||||
|
||||
async function ownerContentQuad(
|
||||
cdp: CdpRunner,
|
||||
parent: CdpFrame,
|
||||
ownerBackendNodeId: number,
|
||||
): Promise<Quad | null> {
|
||||
const result = await sendToCdpTarget<{ model?: { content?: number[] } }>(
|
||||
cdp,
|
||||
parent.target,
|
||||
"DOM.getBoxModel",
|
||||
{ backendNodeId: ownerBackendNodeId },
|
||||
);
|
||||
return parseCdpQuad(result.model?.content);
|
||||
}
|
||||
|
||||
async function sameTargetFrameClips(
|
||||
cdp: CdpRunner,
|
||||
byId: Map<string, CdpFrame>,
|
||||
frame: CdpFrame,
|
||||
targetRoot: CdpFrame,
|
||||
): Promise<Polygon[] | null> {
|
||||
const clips: Polygon[] = [];
|
||||
let current = frame;
|
||||
const seen = new Set<string>();
|
||||
while (current.frameId !== targetRoot.frameId) {
|
||||
if (seen.has(current.frameId) || !current.parentFrameId) return null;
|
||||
seen.add(current.frameId);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent || current.ownerBackendNodeId === undefined) return null;
|
||||
const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId);
|
||||
if (!clip) return null;
|
||||
clips.push(clip);
|
||||
current = parent;
|
||||
}
|
||||
return clips;
|
||||
}
|
||||
|
||||
export async function resolveFrameProjection(
|
||||
cdp: CdpRunner,
|
||||
graph: CdpFrameGraph,
|
||||
frameId: string,
|
||||
): Promise<GeometryProjection | null> {
|
||||
const byId = frameMap(graph);
|
||||
const frame = byId.get(frameId);
|
||||
if (!frame) return null;
|
||||
let targetRoot = targetRootFrame(byId, frame);
|
||||
if (!targetRoot) return null;
|
||||
const sourceViewport = await targetViewport(cdp, targetRoot.target);
|
||||
if (!sourceViewport) return null;
|
||||
const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot);
|
||||
if (!sourceClips) return null;
|
||||
|
||||
const edges: ProjectiveEdge[] = [];
|
||||
while (targetRoot.parentFrameId) {
|
||||
const parent = byId.get(targetRoot.parentFrameId);
|
||||
if (!parent || targetRoot.ownerBackendNodeId === undefined) return null;
|
||||
const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId);
|
||||
if (!destinationQuad) return null;
|
||||
const source =
|
||||
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
||||
if (!source) return null;
|
||||
const parentTargetRoot = targetRootFrame(byId, parent);
|
||||
if (!parentTargetRoot) return null;
|
||||
const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot);
|
||||
if (!destinationClips) return null;
|
||||
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
|
||||
targetRoot = parentTargetRoot;
|
||||
}
|
||||
|
||||
const topViewport =
|
||||
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
||||
return topViewport ? { sourceClips, edges, topViewport } : null;
|
||||
}
|
||||
|
||||
async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
|
||||
if (!cdp.getFrameGraph) return null;
|
||||
try {
|
||||
return await cdp.getFrameGraph(tabId);
|
||||
} catch (error) {
|
||||
console.debug("[bsk frame-geometry] frame graph resolution failed", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function scrollFrameOwners(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
graph: CdpFrameGraph,
|
||||
frameId: string,
|
||||
): Promise<RpcError | null> {
|
||||
const byId = frameMap(graph);
|
||||
const ancestry = frameAncestry(graph, frameId);
|
||||
if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`);
|
||||
for (const child of [...ancestry].reverse()) {
|
||||
const parent = child.parentFrameId ? byId.get(child.parentFrameId) : undefined;
|
||||
if (!parent || child.ownerBackendNodeId === undefined) {
|
||||
return geometryError(`could not resolve frame owner for ${child.frameId}`);
|
||||
}
|
||||
const error = await scrollNodeIntoView(
|
||||
cdpRunnerForTarget(cdp, parent.target),
|
||||
tabId,
|
||||
child.ownerBackendNodeId,
|
||||
);
|
||||
if (error) return error;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function scrollElementWithFrameGraph(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
target: CdpTarget,
|
||||
backendNodeId: number,
|
||||
frameId: string | undefined,
|
||||
graph: CdpFrameGraph | null,
|
||||
): Promise<RpcError | null> {
|
||||
if (frameId) {
|
||||
if (!graph) return geometryError(`could not resolve frame graph for ${frameId}`);
|
||||
const error = await scrollFrameOwners(cdp, tabId, graph, frameId);
|
||||
if (error) return error;
|
||||
} else if (target.sessionId) {
|
||||
return geometryError("an OOPIF node address requires frameId");
|
||||
}
|
||||
return scrollNodeIntoView(cdpRunnerForTarget(cdp, target), tabId, backendNodeId);
|
||||
}
|
||||
|
||||
export async function scrollElementAndFramesIntoView(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
target: CdpTarget,
|
||||
backendNodeId: number,
|
||||
frameId?: string,
|
||||
): Promise<RpcError | null> {
|
||||
const graph = frameId ? await loadFrameGraph(cdp, tabId) : null;
|
||||
return scrollElementWithFrameGraph(cdp, tabId, target, backendNodeId, frameId, graph);
|
||||
}
|
||||
|
||||
function largestRegion(regions: Region): Polygon | null {
|
||||
let largest: { polygon: Polygon; area: number } | null = null;
|
||||
for (const polygon of regions) {
|
||||
const area = polygonArea(polygon);
|
||||
if (area <= 0) continue;
|
||||
if (!largest || area > largest.area) largest = { polygon, area };
|
||||
}
|
||||
return largest?.polygon ?? null;
|
||||
}
|
||||
|
||||
export async function resolveNodeGeometry(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
address: NodeAddress,
|
||||
options: { scrollIntoView?: boolean } = {},
|
||||
): Promise<ResolvedNodeGeometry | RpcError> {
|
||||
try {
|
||||
if (address.target.sessionId && !address.frameId) {
|
||||
return geometryError("an OOPIF node address requires frameId");
|
||||
}
|
||||
const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null;
|
||||
if (address.frameId && !graph) {
|
||||
return geometryError(`could not resolve frame graph for ${address.frameId}`);
|
||||
}
|
||||
|
||||
if (options.scrollIntoView) {
|
||||
const scrollError = await scrollElementWithFrameGraph(
|
||||
cdp,
|
||||
tabId,
|
||||
address.target,
|
||||
address.backendNodeId,
|
||||
address.frameId,
|
||||
graph,
|
||||
);
|
||||
if (scrollError) return scrollError;
|
||||
}
|
||||
|
||||
const localRegion = await nodeContentRegion(
|
||||
cdpRunnerForTarget(cdp, address.target),
|
||||
tabId,
|
||||
address.backendNodeId,
|
||||
);
|
||||
if (isRpcError(localRegion)) return localRegion;
|
||||
|
||||
let topVisibleRegions: Region;
|
||||
if (address.frameId && graph) {
|
||||
const projection = await resolveFrameProjection(cdp, graph, address.frameId);
|
||||
if (!projection)
|
||||
return geometryError(`could not resolve frame geometry for ${address.frameId}`);
|
||||
topVisibleRegions = projectRegionToViewport(localRegion, projection);
|
||||
} else {
|
||||
const viewport = await targetViewport(cdp, address.target);
|
||||
if (!viewport) return geometryError("could not resolve top viewport geometry");
|
||||
topVisibleRegions = localRegion
|
||||
.map((polygon) =>
|
||||
clipPolygon(polygon, rectPolygon({ x: 0, y: 0, w: viewport.width, h: viewport.height })),
|
||||
)
|
||||
.filter((polygon) => polygon.length >= 3);
|
||||
}
|
||||
|
||||
const topBounds = regionBounds(topVisibleRegions);
|
||||
const actionRegion = largestRegion(topVisibleRegions);
|
||||
const actionPoint = actionRegion ? polygonCentroid(actionRegion) : null;
|
||||
if (!topBounds || !actionPoint) {
|
||||
return { code: "permission_denied", message: "element not visible" };
|
||||
}
|
||||
return { topVisibleRegions, topBounds, actionPoint };
|
||||
} catch (error) {
|
||||
return geometryError(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,266 @@
|
||||
export interface Point {
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
export type Polygon = Point[];
|
||||
export type Region = Polygon[];
|
||||
export type Quad = [Point, Point, Point, Point];
|
||||
|
||||
export interface Size {
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface ViewportRect {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface ProjectiveEdge {
|
||||
sourceViewport: Size;
|
||||
destinationQuad: Quad;
|
||||
destinationClips?: Polygon[];
|
||||
}
|
||||
|
||||
export interface GeometryProjection {
|
||||
sourceClips: Polygon[];
|
||||
edges: ProjectiveEdge[];
|
||||
topViewport: Size;
|
||||
}
|
||||
|
||||
export function parseCdpQuad(raw: number[] | undefined): Quad | null {
|
||||
if (!raw || raw.length !== 8 || !raw.every(Number.isFinite)) return null;
|
||||
return [
|
||||
{ x: raw[0], y: raw[1] },
|
||||
{ x: raw[2], y: raw[3] },
|
||||
{ x: raw[4], y: raw[5] },
|
||||
{ x: raw[6], y: raw[7] },
|
||||
];
|
||||
}
|
||||
|
||||
export function rectPolygon(rect: { x: number; y: number; w: number; h: number }): Quad {
|
||||
return [
|
||||
{ x: rect.x, y: rect.y },
|
||||
{ x: rect.x + rect.w, y: rect.y },
|
||||
{ x: rect.x + rect.w, y: rect.y + rect.h },
|
||||
{ x: rect.x, y: rect.y + rect.h },
|
||||
];
|
||||
}
|
||||
|
||||
export function viewportPolygon(viewport: Size): Quad {
|
||||
return rectPolygon({ x: 0, y: 0, w: viewport.width, h: viewport.height });
|
||||
}
|
||||
|
||||
/** Project a normalized unit-square point into an arbitrary content quad. */
|
||||
export function projectUnitPoint(point: Point, quad: Quad): Point {
|
||||
const [p0, p1, p2, p3] = quad;
|
||||
const dx1 = p1.x - p2.x;
|
||||
const dx2 = p3.x - p2.x;
|
||||
const dx3 = p0.x - p1.x + p2.x - p3.x;
|
||||
const dy1 = p1.y - p2.y;
|
||||
const dy2 = p3.y - p2.y;
|
||||
const dy3 = p0.y - p1.y + p2.y - p3.y;
|
||||
|
||||
let a: number;
|
||||
let b: number;
|
||||
let c: number;
|
||||
let d: number;
|
||||
let e: number;
|
||||
let f: number;
|
||||
let g = 0;
|
||||
let h = 0;
|
||||
|
||||
if (Math.abs(dx3) < 1e-9 && Math.abs(dy3) < 1e-9) {
|
||||
a = p1.x - p0.x;
|
||||
b = p3.x - p0.x;
|
||||
c = p0.x;
|
||||
d = p1.y - p0.y;
|
||||
e = p3.y - p0.y;
|
||||
f = p0.y;
|
||||
} else {
|
||||
const denominator = dx1 * dy2 - dx2 * dy1;
|
||||
if (Math.abs(denominator) < 1e-9) {
|
||||
return {
|
||||
x: p0.x + (p1.x - p0.x) * point.x + (p3.x - p0.x) * point.y,
|
||||
y: p0.y + (p1.y - p0.y) * point.x + (p3.y - p0.y) * point.y,
|
||||
};
|
||||
}
|
||||
g = (dx3 * dy2 - dx2 * dy3) / denominator;
|
||||
h = (dx1 * dy3 - dx3 * dy1) / denominator;
|
||||
a = p1.x - p0.x + g * p1.x;
|
||||
b = p3.x - p0.x + h * p3.x;
|
||||
c = p0.x;
|
||||
d = p1.y - p0.y + g * p1.y;
|
||||
e = p3.y - p0.y + h * p3.y;
|
||||
f = p0.y;
|
||||
}
|
||||
|
||||
const scale = g * point.x + h * point.y + 1;
|
||||
return {
|
||||
x: (a * point.x + b * point.y + c) / scale,
|
||||
y: (d * point.x + e * point.y + f) / scale,
|
||||
};
|
||||
}
|
||||
|
||||
export function projectPolygon(polygon: Polygon, edge: ProjectiveEdge): Polygon {
|
||||
const { width, height } = edge.sourceViewport;
|
||||
if (width <= 0 || height <= 0) return [];
|
||||
return polygon.map((point) =>
|
||||
projectUnitPoint({ x: point.x / width, y: point.y / height }, edge.destinationQuad),
|
||||
);
|
||||
}
|
||||
|
||||
function polygonSignedArea(points: Polygon): number {
|
||||
let area = 0;
|
||||
for (let index = 0; index < points.length; index += 1) {
|
||||
const current = points[index];
|
||||
const next = points[(index + 1) % points.length];
|
||||
area += current.x * next.y - next.x * current.y;
|
||||
}
|
||||
return area / 2;
|
||||
}
|
||||
|
||||
function edgeIntersection(start: Point, end: Point, clipStart: Point, clipEnd: Point): Point {
|
||||
const subjectX = end.x - start.x;
|
||||
const subjectY = end.y - start.y;
|
||||
const clipX = clipEnd.x - clipStart.x;
|
||||
const clipY = clipEnd.y - clipStart.y;
|
||||
const denominator = subjectX * clipY - subjectY * clipX;
|
||||
if (Math.abs(denominator) < 1e-9) return end;
|
||||
const offsetX = clipStart.x - start.x;
|
||||
const offsetY = clipStart.y - start.y;
|
||||
const scale = (offsetX * clipY - offsetY * clipX) / denominator;
|
||||
return { x: start.x + scale * subjectX, y: start.y + scale * subjectY };
|
||||
}
|
||||
|
||||
/** Clip a polygon to a convex polygon, preserving every resulting vertex. */
|
||||
export function clipPolygon(subject: Polygon, clip: Polygon): Polygon {
|
||||
if (subject.length < 3 || clip.length < 3) return [];
|
||||
const orientation = polygonSignedArea(clip) >= 0 ? 1 : -1;
|
||||
let output = [...subject];
|
||||
for (let edge = 0; edge < clip.length && output.length > 0; edge += 1) {
|
||||
const clipStart = clip[edge];
|
||||
const clipEnd = clip[(edge + 1) % clip.length];
|
||||
const input = output;
|
||||
output = [];
|
||||
const inside = (point: Point) =>
|
||||
orientation *
|
||||
((clipEnd.x - clipStart.x) * (point.y - clipStart.y) -
|
||||
(clipEnd.y - clipStart.y) * (point.x - clipStart.x)) >=
|
||||
-1e-9;
|
||||
let start = input[input.length - 1];
|
||||
for (const end of input) {
|
||||
const startInside = inside(start);
|
||||
const endInside = inside(end);
|
||||
if (endInside) {
|
||||
if (!startInside) output.push(edgeIntersection(start, end, clipStart, clipEnd));
|
||||
output.push(end);
|
||||
} else if (startInside) {
|
||||
output.push(edgeIntersection(start, end, clipStart, clipEnd));
|
||||
}
|
||||
start = end;
|
||||
}
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
export function polygonBounds(points: Polygon): ViewportRect | null {
|
||||
if (points.length < 3) return null;
|
||||
const xs = points.map((point) => point.x);
|
||||
const ys = points.map((point) => point.y);
|
||||
const x = Math.min(...xs);
|
||||
const y = Math.min(...ys);
|
||||
const right = Math.max(...xs);
|
||||
const bottom = Math.max(...ys);
|
||||
if (right <= x || bottom <= y) return null;
|
||||
return { x, y, width: right - x, height: bottom - y };
|
||||
}
|
||||
|
||||
export function regionBounds(region: Region): ViewportRect | null {
|
||||
return polygonBounds(region.flat());
|
||||
}
|
||||
|
||||
export function polygonArea(points: Polygon): number {
|
||||
return Math.abs(polygonSignedArea(points));
|
||||
}
|
||||
|
||||
export function polygonCentroid(points: Polygon): Point | null {
|
||||
if (points.length < 3) return null;
|
||||
let twiceArea = 0;
|
||||
let x = 0;
|
||||
let y = 0;
|
||||
for (let index = 0; index < points.length; index += 1) {
|
||||
const current = points[index];
|
||||
const next = points[(index + 1) % points.length];
|
||||
const cross = current.x * next.y - next.x * current.y;
|
||||
twiceArea += cross;
|
||||
x += (current.x + next.x) * cross;
|
||||
y += (current.y + next.y) * cross;
|
||||
}
|
||||
if (Math.abs(twiceArea) < 1e-9) return null;
|
||||
return { x: x / (3 * twiceArea), y: y / (3 * twiceArea) };
|
||||
}
|
||||
|
||||
export function projectAndClipRegion(
|
||||
region: Region,
|
||||
edges: ProjectiveEdge[],
|
||||
topViewport: Size,
|
||||
): Region {
|
||||
let projected = region;
|
||||
for (const edge of edges) {
|
||||
const sourceClip = viewportPolygon(edge.sourceViewport);
|
||||
projected = projected
|
||||
.map((polygon) => clipPolygon(polygon, sourceClip))
|
||||
.filter((polygon) => polygon.length >= 3)
|
||||
.map((polygon) =>
|
||||
[edge.destinationQuad, ...(edge.destinationClips ?? [])].reduce(
|
||||
(current, clip) => clipPolygon(current, clip),
|
||||
projectPolygon(polygon, edge),
|
||||
),
|
||||
)
|
||||
.filter((polygon) => polygon.length >= 3);
|
||||
}
|
||||
const topClip = viewportPolygon(topViewport);
|
||||
return projected
|
||||
.map((polygon) => clipPolygon(polygon, topClip))
|
||||
.filter((polygon) => polygon.length >= 3);
|
||||
}
|
||||
|
||||
export function projectRegionToViewport(region: Region, projection: GeometryProjection): Region {
|
||||
const clipped = region
|
||||
.map((polygon) =>
|
||||
projection.sourceClips.reduce((current, clip) => clipPolygon(current, clip), polygon),
|
||||
)
|
||||
.filter((polygon) => polygon.length >= 3);
|
||||
return projectAndClipRegion(clipped, projection.edges, projection.topViewport);
|
||||
}
|
||||
|
||||
export function projectRectToViewport(
|
||||
rect: { x: number; y: number; w: number; h: number } | null,
|
||||
projection: GeometryProjection,
|
||||
): ViewportRect | null {
|
||||
if (!rect) return null;
|
||||
return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection));
|
||||
}
|
||||
|
||||
export function childFrameProjection(
|
||||
parent: GeometryProjection,
|
||||
ownerRectInParent: { x: number; y: number; w: number; h: number },
|
||||
): GeometryProjection {
|
||||
const destinationQuad = rectPolygon(ownerRectInParent);
|
||||
return {
|
||||
sourceClips: [],
|
||||
edges: [
|
||||
{
|
||||
sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h },
|
||||
destinationQuad,
|
||||
destinationClips: parent.sourceClips,
|
||||
},
|
||||
...parent.edges,
|
||||
],
|
||||
topViewport: parent.topViewport,
|
||||
};
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
HELP_REQUEST,
|
||||
type HelpCancelMessage,
|
||||
type HelpFinishMessage,
|
||||
type HelpHighlightRect,
|
||||
type HelpQueryResponse,
|
||||
type HelpRequestMessage,
|
||||
isHelpAckMessage,
|
||||
@@ -25,6 +26,7 @@ import type {
|
||||
ResolvedTarget,
|
||||
RpcError,
|
||||
} from "@/transport/types";
|
||||
import { resolveNodeGeometry } from "./frame-geometry";
|
||||
import {
|
||||
type CdpRunner,
|
||||
type ChromeTabsApi,
|
||||
@@ -37,7 +39,6 @@ import {
|
||||
import { lookupSnapshotRef } from "./snapshot-ref";
|
||||
|
||||
const DEFAULT_HELP_TIMEOUT_MS = 300_000;
|
||||
const HELP_ATTR = "data-bsk-help";
|
||||
const HELP_SEND_RETRIES = 3;
|
||||
const HELP_SEND_RETRY_DELAY_MS = 350;
|
||||
const HELP_REARM_DEBOUNCE_MS = 150;
|
||||
@@ -74,6 +75,7 @@ interface ActiveHelpRequest {
|
||||
title?: string;
|
||||
targets: HelpTarget[];
|
||||
selectors: string[];
|
||||
rects: HelpHighlightRect[];
|
||||
timeoutMs: number;
|
||||
notificationId: string;
|
||||
resolvedTargets?: ResolvedTarget[];
|
||||
@@ -134,6 +136,7 @@ function helpRequestMessage(help: ActiveHelpRequest, tabId: number): HelpRequest
|
||||
...(help.title ? { title: help.title } : {}),
|
||||
displayMode: isPrimaryTab ? "full" : "compact",
|
||||
selectors: isPrimaryTab ? help.selectors : [],
|
||||
rects: isPrimaryTab ? help.rects : [],
|
||||
timeoutMs: help.timeoutMs,
|
||||
};
|
||||
}
|
||||
@@ -165,33 +168,6 @@ async function sendHelpRequestWithAck(
|
||||
throw lastError ?? new Error("failed to show help overlay");
|
||||
}
|
||||
|
||||
async function tagRefTarget(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
backendNodeId: number,
|
||||
index: number,
|
||||
): Promise<string | null> {
|
||||
let objectId: string | undefined;
|
||||
try {
|
||||
const resolved = await cdp.send<{ object?: { objectId?: string } }>(tabId, "DOM.resolveNode", {
|
||||
backendNodeId,
|
||||
});
|
||||
objectId = resolved.object?.objectId;
|
||||
if (!objectId) return null;
|
||||
await cdp.send(tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function(){ this.setAttribute("${HELP_ATTR}", "${index}"); }`,
|
||||
});
|
||||
return `[${HELP_ATTR}="${index}"]`;
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
if (objectId) {
|
||||
await cdp.send(tabId, "Runtime.releaseObject", { objectId }).catch(() => {});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function selectorExists(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
@@ -209,36 +185,22 @@ async function selectorExists(
|
||||
}
|
||||
}
|
||||
|
||||
async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise<void> {
|
||||
if (!cdp) return;
|
||||
try {
|
||||
const doc = await cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
||||
depth: 0,
|
||||
});
|
||||
const rootId = doc.root?.nodeId;
|
||||
if (rootId === undefined) return;
|
||||
const { nodeIds } = await cdp.send<{ nodeIds: number[] }>(tabId, "DOM.querySelectorAll", {
|
||||
nodeId: rootId,
|
||||
selector: `[${HELP_ATTR}]`,
|
||||
});
|
||||
for (const nodeId of nodeIds ?? []) {
|
||||
await cdp.send(tabId, "DOM.removeAttribute", { nodeId, name: HELP_ATTR }).catch(() => {});
|
||||
}
|
||||
} catch {
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveHelpTargets(
|
||||
ctx: ReturnType<SessionManager["get"]>,
|
||||
tabId: number,
|
||||
targets: HelpTarget[],
|
||||
deps: RequestHelpDeps,
|
||||
): Promise<{ selectors: string[]; resolvedTargets?: ResolvedTarget[] }> {
|
||||
options: { scrollIntoView: boolean },
|
||||
): Promise<{
|
||||
selectors: string[];
|
||||
rects: HelpHighlightRect[];
|
||||
resolvedTargets?: ResolvedTarget[];
|
||||
}> {
|
||||
const selectors: string[] = [];
|
||||
const rects: HelpHighlightRect[] = [];
|
||||
const resolved: ResolvedTarget[] = [];
|
||||
let rootNodeId: number | undefined;
|
||||
if (deps.cdp && targets.length > 0) {
|
||||
if (deps.cdp && targets.some((target) => Boolean(target.selector))) {
|
||||
try {
|
||||
const doc = await deps.cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
||||
depth: 0,
|
||||
@@ -262,28 +224,55 @@ async function resolveHelpTargets(
|
||||
resolved.push({ matched, selector: tgt.selector });
|
||||
} else if (tgt.ref) {
|
||||
const looked = ctx ? lookupSnapshotRef(ctx, tgt.ref, tabId) : null;
|
||||
const backendNodeId = looked?.backendNodeId ?? null;
|
||||
let sel: string | null = null;
|
||||
if (backendNodeId !== null && deps.cdp) {
|
||||
sel = await tagRefTarget(deps.cdp, tabId, backendNodeId, i);
|
||||
let rect: HelpHighlightRect | null = null;
|
||||
if (looked && deps.cdp) {
|
||||
const target = {
|
||||
tabId,
|
||||
...(looked.cdpSessionId ? { sessionId: looked.cdpSessionId } : {}),
|
||||
};
|
||||
const geometry = await resolveNodeGeometry(
|
||||
deps.cdp,
|
||||
tabId,
|
||||
{
|
||||
target,
|
||||
backendNodeId: looked.backendNodeId,
|
||||
...(looked.frameId ? { frameId: looked.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: options.scrollIntoView },
|
||||
);
|
||||
if (!isRpcError(geometry)) {
|
||||
rect = {
|
||||
top: geometry.topBounds.y,
|
||||
left: geometry.topBounds.x,
|
||||
width: geometry.topBounds.width,
|
||||
height: geometry.topBounds.height,
|
||||
};
|
||||
}
|
||||
}
|
||||
if (sel) selectors.push(sel);
|
||||
resolved.push({ matched: sel !== null, ref: tgt.ref });
|
||||
if (rect) rects.push(rect);
|
||||
resolved.push({
|
||||
matched: rect !== null,
|
||||
ref: tgt.ref,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return { selectors, resolvedTargets: resolved.length > 0 ? resolved : undefined };
|
||||
return { selectors, rects, resolvedTargets: resolved.length > 0 ? resolved : undefined };
|
||||
}
|
||||
|
||||
async function refreshHelpTargets(help: ActiveHelpRequest): Promise<void> {
|
||||
await clearRefTags(help.deps.cdp, help.primaryTabId);
|
||||
const { selectors, resolvedTargets } = await resolveHelpTargets(
|
||||
async function refreshHelpTargets(
|
||||
help: ActiveHelpRequest,
|
||||
options: { scrollIntoView: boolean } = { scrollIntoView: true },
|
||||
): Promise<void> {
|
||||
const { selectors, rects, resolvedTargets } = await resolveHelpTargets(
|
||||
help.ctx,
|
||||
help.primaryTabId,
|
||||
help.targets,
|
||||
help.deps,
|
||||
options,
|
||||
);
|
||||
help.selectors = selectors;
|
||||
help.rects = rects;
|
||||
help.resolvedTargets = resolvedTargets;
|
||||
}
|
||||
|
||||
@@ -294,12 +283,7 @@ async function cleanupHelp(help: ActiveHelpRequest): Promise<void> {
|
||||
const tabsToCancel = new Set([help.primaryTabId, ...help.overlayTabIds]);
|
||||
await Promise.all(
|
||||
[...tabsToCancel].map((tabId) =>
|
||||
Promise.all([
|
||||
clearRefTags(help.deps.cdp, tabId),
|
||||
help.deps
|
||||
.sendToTab(tabId, { type: HELP_CANCEL, requestId: help.requestId })
|
||||
.catch(() => {}),
|
||||
]),
|
||||
help.deps.sendToTab(tabId, { type: HELP_CANCEL, requestId: help.requestId }).catch(() => {}),
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -476,7 +460,9 @@ function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void {
|
||||
sendResponse({ active: false });
|
||||
return;
|
||||
}
|
||||
if (tabId === help.primaryTabId) await refreshHelpTargets(help);
|
||||
if (tabId === help.primaryTabId) {
|
||||
await refreshHelpTargets(help, { scrollIntoView: false });
|
||||
}
|
||||
help.overlayTabIds.add(tabId);
|
||||
sendResponse({
|
||||
active: true,
|
||||
@@ -486,6 +472,7 @@ function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void {
|
||||
...(help.title ? { title: help.title } : {}),
|
||||
displayMode: tabId === help.primaryTabId ? "full" : "compact",
|
||||
selectors: tabId === help.primaryTabId ? help.selectors : [],
|
||||
rects: tabId === help.primaryTabId ? help.rects : [],
|
||||
timeoutMs: help.timeoutMs,
|
||||
},
|
||||
});
|
||||
@@ -686,7 +673,13 @@ export async function handleRequestHelp(
|
||||
if (deps.autoAttachLifecycle !== false) ensureHelpLifecycleListeners(deps);
|
||||
const tabId = target.tabId;
|
||||
const initialTargets = params.targets ?? [];
|
||||
const { selectors, resolvedTargets } = await resolveHelpTargets(ctx, tabId, initialTargets, deps);
|
||||
const { selectors, rects, resolvedTargets } = await resolveHelpTargets(
|
||||
ctx,
|
||||
tabId,
|
||||
initialTargets,
|
||||
deps,
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
|
||||
await deps.windows.update(target.windowId, { focused: true }).catch(() => {});
|
||||
await deps.activateTab(tabId).catch(() => {});
|
||||
@@ -729,6 +722,7 @@ export async function handleRequestHelp(
|
||||
...(params.title ? { title: params.title } : {}),
|
||||
targets: initialTargets,
|
||||
selectors,
|
||||
rects,
|
||||
timeoutMs,
|
||||
notificationId,
|
||||
resolvedTargets,
|
||||
|
||||
@@ -3,14 +3,15 @@
|
||||
//
|
||||
// All interaction tools:
|
||||
// 1. Resolve target tab (sandbox: must be inside Agent Window).
|
||||
// 2. Resolve target element by `ref` (RefStore.resolve with tabId
|
||||
// binding) or `selector` (DOM.querySelector + describeNode).
|
||||
// 3. Scroll the node into view, then dispatch the appropriate
|
||||
// `Input.*` CDP events.
|
||||
// 2. Resolve target element by `ref` (compound frame/session identity)
|
||||
// or `selector` (DOM.querySelector + describeNode).
|
||||
// 3. Scroll the node and its frame owners into view, project its live
|
||||
// content quad into the top viewport, then dispatch `Input.*` events.
|
||||
// 4. Honour `AbortSignal` so canceled calls don't issue follow-up CDP
|
||||
// commands.
|
||||
|
||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||
import type {
|
||||
ClickParams,
|
||||
@@ -28,17 +29,13 @@ import type {
|
||||
SelectResult,
|
||||
} from "@/transport/types";
|
||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||
import {
|
||||
backendNodeToObject,
|
||||
boxCentre,
|
||||
nodeCentre,
|
||||
quadCentre,
|
||||
scrollNodeIntoView,
|
||||
} from "./element-geometry";
|
||||
import { backendNodeToObject } from "./element-geometry";
|
||||
import { rpcError } from "./errors";
|
||||
import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry";
|
||||
import {
|
||||
type CdpRunner,
|
||||
type ChromeTabsApi,
|
||||
cdpRunnerForTarget,
|
||||
chromeTabsApi,
|
||||
enforceAgentWindow,
|
||||
isRpcError,
|
||||
@@ -137,7 +134,16 @@ async function resolveBackendNode(
|
||||
target: { tabId: number },
|
||||
params: { ref?: string; selector?: string },
|
||||
toolName: string,
|
||||
): Promise<{ backendNodeId: number; usedRef?: string; usedSelector?: string } | RpcError> {
|
||||
): Promise<
|
||||
| {
|
||||
backendNodeId: number;
|
||||
cdpTarget: CdpTarget;
|
||||
frameId?: string;
|
||||
usedRef?: string;
|
||||
usedSelector?: string;
|
||||
}
|
||||
| RpcError
|
||||
> {
|
||||
const hasRef = typeof params.ref === "string" && params.ref.length > 0;
|
||||
const hasSelector = typeof params.selector === "string" && params.selector.length > 0;
|
||||
if (hasRef && hasSelector) {
|
||||
@@ -155,7 +161,15 @@ async function resolveBackendNode(
|
||||
if (hasRef) {
|
||||
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId);
|
||||
if (isRpcError(resolved)) return resolved;
|
||||
return { backendNodeId: resolved.backendNodeId, usedRef: resolved.refKey };
|
||||
return {
|
||||
backendNodeId: resolved.backendNodeId,
|
||||
cdpTarget: {
|
||||
tabId: target.tabId,
|
||||
...(resolved.cdpSessionId ? { sessionId: resolved.cdpSessionId } : {}),
|
||||
},
|
||||
...(resolved.frameId ? { frameId: resolved.frameId } : {}),
|
||||
usedRef: resolved.refKey,
|
||||
};
|
||||
}
|
||||
// selector path
|
||||
try {
|
||||
@@ -192,7 +206,7 @@ async function resolveBackendNode(
|
||||
message: "DOM.describeNode returned no backendNodeId",
|
||||
};
|
||||
}
|
||||
return { backendNodeId, usedSelector: params.selector };
|
||||
return { backendNodeId, cdpTarget: { tabId: target.tabId }, usedSelector: params.selector };
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -228,19 +242,19 @@ export async function handleClick(
|
||||
return { code: "cancelled", message: "click aborted" };
|
||||
}
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (scrollErr) return scrollErr;
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
|
||||
const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(centre)) return centre;
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const geometry = await resolveNodeGeometry(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
{
|
||||
target: node.cdpTarget,
|
||||
backendNodeId: node.backendNodeId,
|
||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
const centre = geometry.actionPoint;
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "click aborted" };
|
||||
@@ -353,19 +367,19 @@ export async function handleHover(
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "hover");
|
||||
if (isRpcError(node)) return node;
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (scrollErr) return scrollErr;
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
|
||||
const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(centre)) return centre;
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const geometry = await resolveNodeGeometry(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
{
|
||||
target: node.cdpTarget,
|
||||
backendNodeId: node.backendNodeId,
|
||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
const centre = geometry.actionPoint;
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "hover aborted" };
|
||||
@@ -585,10 +599,11 @@ export async function handleFill(
|
||||
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill");
|
||||
if (isRpcError(node)) return node;
|
||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const described = await deps.cdp.send<{ node?: DescribedNode }>(
|
||||
const described = await nodeCdp.send<{ node?: DescribedNode }>(
|
||||
target.tabId,
|
||||
"DOM.describeNode",
|
||||
{
|
||||
@@ -602,12 +617,18 @@ export async function handleFill(
|
||||
`element ${described.node?.nodeName ?? "?"} not fillable (need input/textarea/contenteditable)`,
|
||||
);
|
||||
}
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
const scrollErr = await scrollElementAndFramesIntoView(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
node.cdpTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
);
|
||||
if (scrollErr) return scrollErr;
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -619,7 +640,7 @@ export async function handleFill(
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
|
||||
const objectIdOrErr = await backendNodeToObject(deps.cdp, target.tabId, node.backendNodeId);
|
||||
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
||||
const objectId = objectIdOrErr;
|
||||
const clearBefore = params.clear_before ?? true;
|
||||
@@ -628,7 +649,7 @@ export async function handleFill(
|
||||
if (clearBefore) {
|
||||
// Clear input/textarea value or wipe contenteditable innerText,
|
||||
// then fire `input` so frameworks observe the empty state.
|
||||
await deps.cdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
if (this.isContentEditable) { this.textContent = ''; }
|
||||
@@ -655,7 +676,7 @@ export async function handleFill(
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
// Fire `input` + `change` so React / Vue controlled inputs commit.
|
||||
await deps.cdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
@@ -853,14 +874,21 @@ export async function handlePress(
|
||||
if (params.ref || params.selector) {
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press");
|
||||
if (isRpcError(node)) return node;
|
||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
const scrollErr = await scrollElementAndFramesIntoView(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
node.cdpTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
);
|
||||
if (scrollErr) return scrollErr;
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "press aborted" };
|
||||
}
|
||||
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -965,10 +993,11 @@ export async function handleSelect(
|
||||
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "select");
|
||||
if (isRpcError(node)) return node;
|
||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const described = await deps.cdp.send<{ node?: DescribedNode }>(
|
||||
const described = await nodeCdp.send<{ node?: DescribedNode }>(
|
||||
target.tabId,
|
||||
"DOM.describeNode",
|
||||
{ backendNodeId: node.backendNodeId },
|
||||
@@ -991,12 +1020,18 @@ export async function handleSelect(
|
||||
"single-select <select> requires exactly one value",
|
||||
);
|
||||
}
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
const scrollErr = await scrollElementAndFramesIntoView(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
node.cdpTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
);
|
||||
if (scrollErr) return scrollErr;
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "select aborted" };
|
||||
}
|
||||
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -1008,12 +1043,12 @@ export async function handleSelect(
|
||||
return { code: "cancelled", message: "select aborted" };
|
||||
}
|
||||
|
||||
const objectIdOrErr = await backendNodeToObject(deps.cdp, target.tabId, node.backendNodeId);
|
||||
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
||||
const objectId = objectIdOrErr;
|
||||
|
||||
try {
|
||||
const evaluated = await deps.cdp.send<{
|
||||
const evaluated = await nodeCdp.send<{
|
||||
result?: { value?: SelectMutationResult | null };
|
||||
}>(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
@@ -1072,9 +1107,6 @@ export async function handleSelect(
|
||||
|
||||
export const __testing__ = {
|
||||
DEFAULT_TIMEOUT_MS,
|
||||
quadCentre,
|
||||
boxCentre,
|
||||
resolveBackendNode,
|
||||
nodeCentre,
|
||||
isFillable,
|
||||
};
|
||||
|
||||
@@ -10,9 +10,11 @@ import {
|
||||
isVomReferenceNode,
|
||||
renderVom,
|
||||
type VomNode,
|
||||
type VomOptions,
|
||||
type VomScene,
|
||||
} from "@browser-skill/vom";
|
||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import {
|
||||
type CaptureSuppressSendToTab,
|
||||
withOverlaysHiddenForCapture,
|
||||
@@ -30,8 +32,8 @@ import type {
|
||||
SnapshotResult,
|
||||
} from "@/transport/types";
|
||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||
import { nodeBoundingRect, scrollNodeIntoView } from "./element-geometry";
|
||||
import { rpcError } from "./errors";
|
||||
import { resolveNodeGeometry } from "./frame-geometry";
|
||||
import {
|
||||
type ChromeTabsApi,
|
||||
enforceToolTargetScope,
|
||||
@@ -40,6 +42,7 @@ import {
|
||||
type ResolvedTargetTab,
|
||||
resolveCdpAccessibleTargetTab,
|
||||
type CdpRunner as SharedCdpRunner,
|
||||
sendToCdpTarget,
|
||||
normaliseRef as sharedNormaliseRef,
|
||||
type ToolEffect,
|
||||
} from "./shared";
|
||||
@@ -50,6 +53,11 @@ import {
|
||||
captureViewModel,
|
||||
collectOverlayExcludedBackendIds,
|
||||
} from "./vom/capture";
|
||||
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
|
||||
import {
|
||||
type CaptureVomObservationResult,
|
||||
projectRecordSafeObservation,
|
||||
} from "./vom/record-safe-observation";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared helpers (legacy aliases — observation.ts kept exporting these
|
||||
@@ -162,26 +170,30 @@ function throwIfAborted(signal: AbortSignal | undefined, tool: string): void {
|
||||
async function captureElementScreenshot(
|
||||
cdp: SharedCdpRunner,
|
||||
tabId: number,
|
||||
target: CdpTarget,
|
||||
backendNodeId: number,
|
||||
frameId?: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ image_base64: string; width: number; height: number } | RpcError> {
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const scrollErr = await scrollNodeIntoView(cdp, tabId, backendNodeId);
|
||||
if (scrollErr) return scrollErr;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
|
||||
const rectOrErr = await nodeBoundingRect(cdp, tabId, backendNodeId);
|
||||
if (isRpcError(rectOrErr)) return rectOrErr;
|
||||
const geometry = await resolveNodeGeometry(
|
||||
cdp,
|
||||
tabId,
|
||||
{ target, backendNodeId, ...(frameId ? { frameId } : {}) },
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const rect = geometry.topBounds;
|
||||
|
||||
try {
|
||||
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
|
||||
format: "png",
|
||||
clip: {
|
||||
x: rectOrErr.x,
|
||||
y: rectOrErr.y,
|
||||
width: rectOrErr.width,
|
||||
height: rectOrErr.height,
|
||||
x: rect.x,
|
||||
y: rect.y,
|
||||
width: rect.width,
|
||||
height: rect.height,
|
||||
scale: 1,
|
||||
},
|
||||
});
|
||||
@@ -191,8 +203,8 @@ async function captureElementScreenshot(
|
||||
return { code: "cdp_failed", message: "Page.captureScreenshot returned no data" };
|
||||
}
|
||||
const dims = parsePngDimensions(image_base64) ?? {
|
||||
width: Math.round(rectOrErr.width),
|
||||
height: Math.round(rectOrErr.height),
|
||||
width: Math.round(rect.width),
|
||||
height: Math.round(rect.height),
|
||||
};
|
||||
return { image_base64, width: dims.width, height: dims.height };
|
||||
} catch (err) {
|
||||
@@ -296,9 +308,21 @@ export async function handleScreenshot(
|
||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const cdp = deps.cdp;
|
||||
const nodeTarget = {
|
||||
tabId: target.tabId,
|
||||
...(node.cdpSessionId ? { sessionId: node.cdpSessionId } : {}),
|
||||
};
|
||||
const captured = await withOverlaysHiddenForCapture(
|
||||
target.tabId,
|
||||
() => captureElementScreenshot(cdp, target.tabId, node.backendNodeId, signal),
|
||||
() =>
|
||||
captureElementScreenshot(
|
||||
cdp,
|
||||
target.tabId,
|
||||
nodeTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
signal,
|
||||
),
|
||||
deps.sendToTab,
|
||||
);
|
||||
if (isRpcError(captured)) return captured;
|
||||
@@ -352,6 +376,7 @@ export type CdpRunner = SharedCdpRunner;
|
||||
/** Subset of CDP `AXNode` we care about — see `Accessibility.AXNode`. */
|
||||
export interface CdpAxNode {
|
||||
nodeId: string;
|
||||
frameId?: string;
|
||||
parentId?: string;
|
||||
backendDOMNodeId?: number;
|
||||
ignored?: boolean;
|
||||
@@ -391,7 +416,15 @@ function isModalSignal(
|
||||
|
||||
function isSensitive(capturedNode: CapturedNode | undefined): boolean {
|
||||
const attrs = capturedNode?.attrs ?? {};
|
||||
return (attrs.type ?? "").toLowerCase() === "password";
|
||||
const type = (attrs.type ?? "").toLowerCase();
|
||||
if (type === "password") return true;
|
||||
const autocomplete = (attrs.autocomplete ?? "").toLowerCase();
|
||||
return (
|
||||
autocomplete.startsWith("cc-") ||
|
||||
autocomplete === "one-time-code" ||
|
||||
autocomplete === "current-password" ||
|
||||
autocomplete === "new-password"
|
||||
);
|
||||
}
|
||||
|
||||
interface AxNodeSignals {
|
||||
@@ -441,7 +474,13 @@ const AX_TEXT_STOP_ROLES = new Set([
|
||||
"columnheader",
|
||||
"rowheader",
|
||||
]);
|
||||
const SENSITIVE_AX_INPUT_TYPES = new Set(["password", "credit-card"]);
|
||||
const SENSITIVE_AX_INPUT_TYPES = new Set([
|
||||
"password",
|
||||
"credit-card",
|
||||
"one-time-code",
|
||||
"current-password",
|
||||
"new-password",
|
||||
]);
|
||||
|
||||
function axPropertyString(axNode: CdpAxNode, name: string): string | undefined {
|
||||
const prop = axNode.properties?.find((item) => item.name === name);
|
||||
@@ -584,7 +623,7 @@ function iframeNameFor(node: CapturedNode): string | undefined {
|
||||
function isRenderableIframeControl(node: CapturedNode): boolean {
|
||||
const tag = normalizeTag(node.tag);
|
||||
if (!IFRAME_RENDERABLE_TAGS.has(tag)) return false;
|
||||
if (!node.rect) return false;
|
||||
if (!node.localRect && !node.rect) return false;
|
||||
if (node.pointerEvents === "none") return false;
|
||||
if ((node.attrs.type ?? "").toLowerCase() === "hidden") return false;
|
||||
|
||||
@@ -756,7 +795,12 @@ function applyCapturedSignals(
|
||||
signals: VomNodeDomSignals,
|
||||
): VomNode {
|
||||
if (!capturedNode) return node;
|
||||
const attrs = capturedNode.attrs;
|
||||
const attrs =
|
||||
node.sensitive && Object.prototype.hasOwnProperty.call(capturedNode.attrs, "value")
|
||||
? Object.fromEntries(
|
||||
Object.entries(capturedNode.attrs).filter(([name]) => name.toLowerCase() !== "value"),
|
||||
)
|
||||
: capturedNode.attrs;
|
||||
const text = cleanAttr(capturedNode.textContent);
|
||||
const nearbyText = nearbyTextFor(capturedNode, signals.childrenByParentId);
|
||||
const placeholder = cleanAttr(capturedNode.formPlaceholder) ?? cleanAttr(attrs.placeholder);
|
||||
@@ -823,8 +867,7 @@ function vomNodeFromCaptured(
|
||||
signals: VomNodeDomSignals,
|
||||
): VomNode {
|
||||
const sensitive = isSensitive(capturedNode);
|
||||
const value =
|
||||
capturedNode.formValue ?? (sensitive ? (capturedNode.attrs.value ?? "") : undefined);
|
||||
const value = sensitive ? undefined : capturedNode.formValue;
|
||||
const tag = normalizeTag(capturedNode.tag);
|
||||
const node = applyCapturedSignals(
|
||||
{
|
||||
@@ -860,8 +903,17 @@ function axVomNode(
|
||||
): VomNode {
|
||||
const role = axString(axNode.role);
|
||||
const capturedValue = capturedNode?.formValue;
|
||||
const value = capturedValue ?? axValue(axNode.value);
|
||||
const sourceValue = capturedValue ?? axValue(axNode.value);
|
||||
const signalsForNode = axSignals.get(axNode.nodeId);
|
||||
const sensitive = isSensitive(capturedNode) || signalsForNode?.sensitive === true;
|
||||
const value = sensitive ? undefined : sourceValue;
|
||||
const inputState =
|
||||
inputStateFor(capturedNode, sourceValue) ??
|
||||
(sensitive && ["textbox", "searchbox"].includes(role?.toLowerCase() ?? "")
|
||||
? sourceValue === undefined || sourceValue === ""
|
||||
? "empty"
|
||||
: "filled"
|
||||
: undefined);
|
||||
let name = FORM_CONTROL_TAGS.has(normalizeTag(capturedNode?.tag))
|
||||
? formControlName(capturedNode, axString(axNode.name), axSiblingLabel(axNode, axById), signals)
|
||||
: (axString(axNode.name) ?? signalsForNode?.aggregatedText);
|
||||
@@ -877,8 +929,8 @@ function axVomNode(
|
||||
position: capturedNode?.position || "static",
|
||||
pointerEvents: capturedNode?.pointerEvents || "auto",
|
||||
modal: isModalSignal(axNode, capturedNode),
|
||||
sensitive: isSensitive(capturedNode) || signalsForNode?.sensitive === true,
|
||||
inputState: inputStateFor(capturedNode, value),
|
||||
sensitive,
|
||||
inputState,
|
||||
};
|
||||
if (role) node.role = role;
|
||||
if (name) node.name = name;
|
||||
@@ -1141,6 +1193,8 @@ export interface BuildVomSceneOptions {
|
||||
pageUrl?: string;
|
||||
}
|
||||
|
||||
export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>;
|
||||
|
||||
function externalHrefHost(
|
||||
href: string | undefined,
|
||||
pageUrl: string | undefined,
|
||||
@@ -1234,6 +1288,119 @@ export function buildVomScene(
|
||||
};
|
||||
}
|
||||
|
||||
export function buildFrameVomScene(
|
||||
documents: VomFrameDocument[],
|
||||
captured: CapturedViewModel,
|
||||
options: BuildVomSceneOptions = {},
|
||||
): VomScene {
|
||||
const documentByFrameId = new Map(documents.map((document) => [document.frameId, document]));
|
||||
const rootFrameId =
|
||||
captured.rootFrameId ?? documents.find((document) => !document.parentFrameId)?.frameId;
|
||||
const orderedFrameIds = [
|
||||
...(rootFrameId ? [rootFrameId] : []),
|
||||
...documents.map((document) => document.frameId),
|
||||
].filter((frameId, index, all) => all.indexOf(frameId) === index);
|
||||
|
||||
const localScenes = new Map<string, VomScene>();
|
||||
for (const frameId of orderedFrameIds) {
|
||||
const document = documentByFrameId.get(frameId);
|
||||
if (!document) continue;
|
||||
localScenes.set(
|
||||
frameId,
|
||||
buildVomScene(
|
||||
document.axNodes,
|
||||
{
|
||||
viewport: captured.viewport,
|
||||
nodes: document.domNodes,
|
||||
iframeNodes: new Map(),
|
||||
surfaceProbes: frameId === rootFrameId ? captured.surfaceProbes : [],
|
||||
excludedBackendNodeIds:
|
||||
frameId === rootFrameId ? captured.excludedBackendNodeIds : new Set(),
|
||||
},
|
||||
{ pageUrl: document.url ?? options.pageUrl },
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const sceneIdByFrameAndBackend = new Map<string, number>();
|
||||
let nextSceneId = 1;
|
||||
for (const [frameId, scene] of localScenes) {
|
||||
for (const node of scene.nodes) {
|
||||
sceneIdByFrameAndBackend.set(`${frameId}:${node.id}`, nextSceneId);
|
||||
nextSceneId += 1;
|
||||
}
|
||||
}
|
||||
|
||||
const nodes: VomNode[] = [];
|
||||
const surfaces: CondSurface[] = [];
|
||||
const activeScopeBlocks: ActiveScopeBlock[] = [];
|
||||
|
||||
for (const [frameId, scene] of localScenes) {
|
||||
const document = documentByFrameId.get(frameId);
|
||||
const ownerBackendNodeId = document?.ownerBackendNodeId;
|
||||
const parentFrameId = document?.parentFrameId;
|
||||
const ownerSceneId =
|
||||
ownerBackendNodeId !== undefined && parentFrameId
|
||||
? sceneIdByFrameAndBackend.get(`${parentFrameId}:${ownerBackendNodeId}`)
|
||||
: undefined;
|
||||
if (frameId !== rootFrameId && ownerSceneId === undefined) continue;
|
||||
|
||||
const suppressedRoots = new Set(
|
||||
frameId === rootFrameId
|
||||
? []
|
||||
: scene.nodes
|
||||
.filter(
|
||||
(node) =>
|
||||
node.parentId === null &&
|
||||
["rootwebarea", "webarea"].includes(node.role?.toLowerCase() ?? ""),
|
||||
)
|
||||
.map((node) => node.id),
|
||||
);
|
||||
|
||||
for (const node of scene.nodes) {
|
||||
if (suppressedRoots.has(node.id)) continue;
|
||||
const sceneNodeId = sceneIdByFrameAndBackend.get(`${frameId}:${node.id}`) as number;
|
||||
let parentId =
|
||||
node.parentId !== null
|
||||
? sceneIdByFrameAndBackend.get(`${frameId}:${node.parentId}`)
|
||||
: ownerSceneId;
|
||||
if (node.parentId !== null && suppressedRoots.has(node.parentId)) parentId = ownerSceneId;
|
||||
const remapId = (id: number) => sceneIdByFrameAndBackend.get(`${frameId}:${id}`);
|
||||
nodes.push({
|
||||
...node,
|
||||
id: sceneNodeId,
|
||||
backendNodeId: node.id,
|
||||
frameId,
|
||||
contextScopeId: document?.contextScopeId ?? frameId,
|
||||
parentId: parentId ?? null,
|
||||
...(node.domParentId !== undefined
|
||||
? { domParentId: node.domParentId === null ? null : (remapId(node.domParentId) ?? null) }
|
||||
: {}),
|
||||
...(node.domAncestorIds
|
||||
? { domAncestorIds: node.domAncestorIds.flatMap((id) => remapId(id) ?? []) }
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
for (const surface of scene.surfaces ?? []) {
|
||||
const triggerId = sceneIdByFrameAndBackend.get(`${frameId}:${surface.triggerId}`);
|
||||
if (triggerId !== undefined) surfaces.push({ ...surface, triggerId });
|
||||
}
|
||||
for (const block of scene.activeScopeBlocks ?? []) {
|
||||
const triggerId = sceneIdByFrameAndBackend.get(`${frameId}:${block.triggerId}`);
|
||||
if (triggerId !== undefined) activeScopeBlocks.push({ ...block, triggerId });
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
viewport: captured.viewport,
|
||||
nodes,
|
||||
...(rootFrameId ? { rootFrameId } : {}),
|
||||
...(surfaces.length > 0 ? { surfaces } : {}),
|
||||
...(activeScopeBlocks.length > 0 ? { activeScopeBlocks } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export interface SnapshotDeps {
|
||||
cdp: CdpRunner;
|
||||
tabsApi: {
|
||||
@@ -1322,9 +1489,15 @@ export async function handleGetHtml(
|
||||
if (params.ref) {
|
||||
const resolved = resolveSnapshotRef(ctx, params.ref, target.tabId);
|
||||
if (isRpcError(resolved)) return resolved;
|
||||
const resp = await deps.cdp.send<{ outerHTML?: string }>(target.tabId, "DOM.getOuterHTML", {
|
||||
backendNodeId: resolved.backendNodeId,
|
||||
});
|
||||
const resp = await sendToCdpTarget<{ outerHTML?: string }>(
|
||||
deps.cdp,
|
||||
{
|
||||
tabId: target.tabId,
|
||||
...(resolved.cdpSessionId ? { sessionId: resolved.cdpSessionId } : {}),
|
||||
},
|
||||
"DOM.getOuterHTML",
|
||||
{ backendNodeId: resolved.backendNodeId },
|
||||
);
|
||||
throwIfAborted(signal, "get_html");
|
||||
html = resp.outerHTML ?? "";
|
||||
} else {
|
||||
@@ -1383,14 +1556,13 @@ async function fallbackCapturedViewModel(
|
||||
try {
|
||||
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
throwIfAborted(signal, "observation");
|
||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
const source = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
viewport = {
|
||||
width: vpSrc.clientWidth ?? 0,
|
||||
height: vpSrc.clientHeight ?? 0,
|
||||
width: source.clientWidth ?? 0,
|
||||
height: source.clientHeight ?? 0,
|
||||
};
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
// viewport stays zero-sized
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
}
|
||||
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId, signal);
|
||||
throwIfAborted(signal, "observation");
|
||||
@@ -1400,22 +1572,58 @@ async function fallbackCapturedViewModel(
|
||||
async function captureForVom(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
conditionalSurfaceProbe: boolean,
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>,
|
||||
signal?: AbortSignal,
|
||||
options: CaptureVomObservationOptions,
|
||||
): Promise<CapturedViewModel> {
|
||||
try {
|
||||
return await captureViewModel(cdp, tabId, {
|
||||
conditionalSurfaceProbe,
|
||||
hoverProbeBypassOverlay,
|
||||
signal,
|
||||
conditionalSurfaceProbe: options.conditionalSurfaceProbe ?? false,
|
||||
hoverProbeBypassOverlay: options.hoverProbeBypassOverlay,
|
||||
signal: options.signal,
|
||||
});
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
return fallbackCapturedViewModel(cdp, tabId, signal);
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
return fallbackCapturedViewModel(cdp, tabId, options.signal);
|
||||
}
|
||||
}
|
||||
|
||||
export interface CaptureVomObservationOptions extends VomOptions {
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export async function captureVomObservation(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
url: string | undefined,
|
||||
options: CaptureVomObservationOptions = {},
|
||||
): Promise<CaptureVomObservationResult> {
|
||||
throwIfAborted(options.signal, "observation");
|
||||
await cdp.ensureAttachedToUrl?.(tabId, url);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const captured = await captureForVom(cdp, tabId, options);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const documents = await captureFrameData<CdpAxNode>(cdp, tabId, captured, options.signal);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const scene =
|
||||
captured.rootFrameId && captured.frameNodes?.size
|
||||
? buildFrameVomScene(documents, captured, { pageUrl: url })
|
||||
: buildVomScene(documents[0]?.axNodes ?? [], captured, { pageUrl: url });
|
||||
const rendered = renderVom(scene, {
|
||||
maxDepth: options.maxDepth,
|
||||
maxTokens: options.maxTokens,
|
||||
redactValues: options.redactValues,
|
||||
activeRegionPolicy: options.activeRegionPolicy,
|
||||
});
|
||||
throwIfAborted(options.signal, "observation");
|
||||
return projectRecordSafeObservation({
|
||||
rootFrameId: captured.rootFrameId ?? documents[0]?.frameId ?? "root",
|
||||
frameDocuments: documents,
|
||||
rendered,
|
||||
surfaceProbes: captured.surfaceProbes,
|
||||
});
|
||||
}
|
||||
|
||||
async function handleVomObservation(
|
||||
manager: SessionManager,
|
||||
params: SnapshotParams | ObserveParams,
|
||||
@@ -1445,48 +1653,43 @@ async function handleVomObservation(
|
||||
try {
|
||||
throwIfAborted(signal, toolName);
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||
throwIfAborted(signal, toolName);
|
||||
await deps.cdp.send<unknown>(target.tabId, "Accessibility.enable", {});
|
||||
throwIfAborted(signal, toolName);
|
||||
const result = await deps.cdp.send<{ nodes: CdpAxNode[] }>(
|
||||
target.tabId,
|
||||
"Accessibility.getFullAXTree",
|
||||
{},
|
||||
);
|
||||
throwIfAborted(signal, toolName);
|
||||
const axNodes = result.nodes ?? [];
|
||||
const effectiveConditionalSurfaceProbe =
|
||||
deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe;
|
||||
const captured = await captureForVom(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
effectiveConditionalSurfaceProbe,
|
||||
deps.hoverProbeBypassOverlay,
|
||||
signal,
|
||||
);
|
||||
throwIfAborted(signal, toolName);
|
||||
const scene = buildVomScene(axNodes, captured, { pageUrl: target.url });
|
||||
const rendered = renderVom(scene, {
|
||||
const observation = await captureVomObservation(deps.cdp, target.tabId, target.url, {
|
||||
maxDepth: params.max_depth,
|
||||
maxTokens: params.max_tokens,
|
||||
activeRegionPolicy: true,
|
||||
conditionalSurfaceProbe: effectiveConditionalSurfaceProbe,
|
||||
hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay,
|
||||
signal,
|
||||
});
|
||||
throwIfAborted(signal, toolName);
|
||||
const targetByFrameId = new Map(
|
||||
observation.frames.map((frame) => [frame.frameId, frame.target]),
|
||||
);
|
||||
ctx.refStore.replace(
|
||||
rendered.refs.map(
|
||||
(r) => [r.ref, { backendNodeId: r.backendNodeId, tabId: target.tabId }] as const,
|
||||
),
|
||||
observation.refs.map((ref) => {
|
||||
const refTarget = ref.frameId ? targetByFrameId.get(ref.frameId) : undefined;
|
||||
return [
|
||||
ref.ref,
|
||||
{
|
||||
backendNodeId: ref.backendNodeId,
|
||||
tabId: target.tabId,
|
||||
...(ref.frameId ? { frameId: ref.frameId } : {}),
|
||||
...(refTarget?.sessionId ? { cdpSessionId: refTarget.sessionId } : {}),
|
||||
},
|
||||
] as const;
|
||||
}),
|
||||
);
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
text: rendered.text,
|
||||
ref_count: rendered.refs.length,
|
||||
text: observation.text,
|
||||
ref_count: observation.refs.length,
|
||||
tab_id: target.tabId,
|
||||
truncated: rendered.truncated,
|
||||
truncated: observation.truncated,
|
||||
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
|
||||
? {
|
||||
debug: {
|
||||
surface_probes: (captured.surfaceProbes ?? []).map((probe) => ({
|
||||
surface_probes: (observation.surfaceProbes ?? []).map((probe) => ({
|
||||
trigger_backend_node_id: probe.triggerBackendNodeId,
|
||||
...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}),
|
||||
trigger_action: probe.triggerAction,
|
||||
|
||||
@@ -76,6 +76,7 @@ export interface SessionStopResult {
|
||||
}
|
||||
|
||||
export interface SessionStopDeps {
|
||||
signal?: AbortSignal;
|
||||
cdp?: {
|
||||
detachSession(sessionId: string): Promise<void>;
|
||||
};
|
||||
@@ -177,6 +178,9 @@ export async function handleSessionStop(
|
||||
message: `session ${params.session_id} unknown`,
|
||||
};
|
||||
}
|
||||
if (deps.signal?.aborted) {
|
||||
return { code: "cancelled", message: "session_stop aborted before teardown" };
|
||||
}
|
||||
|
||||
// Step 1: auto-return borrowed tabs. Iterate over a snapshot of the
|
||||
// ids so deletions during iteration do not break the Map iterator.
|
||||
@@ -187,6 +191,7 @@ export async function handleSessionStop(
|
||||
try {
|
||||
const tabManagement = {
|
||||
...(deps.tabManagement ?? {}),
|
||||
signal: deps.signal,
|
||||
isAgentWindowId:
|
||||
deps.tabManagement?.isAgentWindowId ??
|
||||
((windowId: number) => manager.findByWindowId(windowId) !== null),
|
||||
@@ -214,6 +219,17 @@ export async function handleSessionStop(
|
||||
}
|
||||
}
|
||||
|
||||
if (deps.signal?.aborted) {
|
||||
const nonCancelFailures = returnFailures?.filter((failure) => failure.code !== "cancelled");
|
||||
if (nonCancelFailures && nonCancelFailures.length > 0) {
|
||||
return {
|
||||
...(returnedTabIds.length > 0 ? { returned_tab_ids: returnedTabIds } : {}),
|
||||
return_failures: nonCancelFailures,
|
||||
};
|
||||
}
|
||||
return { code: "cancelled", message: "session_stop aborted during tab return" };
|
||||
}
|
||||
|
||||
const result: SessionStopResult = {};
|
||||
if (returnedTabIds.length > 0) result.returned_tab_ids = returnedTabIds;
|
||||
if (returnFailures && returnFailures.length > 0) {
|
||||
@@ -233,6 +249,10 @@ export async function handleSessionStop(
|
||||
// Step 3: detach CDP sessions this session opened (no-op if none).
|
||||
await deps.cdp?.detachSession(params.session_id);
|
||||
|
||||
if (deps.signal?.aborted) {
|
||||
return { code: "cancelled", message: "session_stop aborted before window close" };
|
||||
}
|
||||
|
||||
// Step 4: close the Agent Window and drop the context.
|
||||
await manager.stop(params.session_id);
|
||||
|
||||
|
||||
@@ -6,7 +6,8 @@
|
||||
// exactly the same sandbox + visibility rules as the M6 observation
|
||||
// handlers (review parity).
|
||||
|
||||
import type { DialogCursor } from "@/browser-driver/chromium-cdp";
|
||||
import type { CdpDebuggee, DialogCursor } from "@/browser-driver/chromium-cdp";
|
||||
import type { CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||
import { normaliseRef } from "@/session-manager/ref-store";
|
||||
import type { ConsoleResult, JavaScriptDialogInfo, RpcError } from "@/transport/types";
|
||||
@@ -51,9 +52,11 @@ export type { DialogCursor };
|
||||
*/
|
||||
export interface CdpRunner {
|
||||
send<T = unknown>(tabId: number, method: string, params?: object): Promise<T>;
|
||||
sendToTarget?<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T>;
|
||||
getFrameGraph?(tabId: number): Promise<CdpFrameGraph>;
|
||||
ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>;
|
||||
trackSessionTab?(sessionId: string, tabId: number): void;
|
||||
onEvent?(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): {
|
||||
onEvent?(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
|
||||
dispose(): void;
|
||||
};
|
||||
dialogCursor?(tabId: number): DialogCursor;
|
||||
@@ -68,6 +71,25 @@ export interface CdpRunner {
|
||||
): ConsoleResult;
|
||||
}
|
||||
|
||||
export function sendToCdpTarget<T = unknown>(
|
||||
cdp: CdpRunner,
|
||||
target: CdpTarget,
|
||||
method: string,
|
||||
params?: object,
|
||||
): Promise<T> {
|
||||
if (target.sessionId && cdp.sendToTarget) {
|
||||
return cdp.sendToTarget<T>(target, method, params);
|
||||
}
|
||||
return cdp.send<T>(target.tabId, method, params);
|
||||
}
|
||||
|
||||
export function cdpRunnerForTarget(cdp: CdpRunner, target: CdpTarget): CdpRunner {
|
||||
return {
|
||||
send: <T = unknown>(_tabId: number, method: string, params?: object) =>
|
||||
sendToCdpTarget<T>(cdp, target, method, params),
|
||||
};
|
||||
}
|
||||
|
||||
export interface BufferedReadBounds {
|
||||
since: number | undefined;
|
||||
limit: number;
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Snapshot ref resolution — normalise `@eN` / `eN`, look up
|
||||
// `backendNodeId` via the session RefStore, and build stable
|
||||
// compound CDP node identity via the session RefStore, and build stable
|
||||
// `ref_not_found` errors for hard-failure tool paths.
|
||||
|
||||
import type { SessionContext } from "@/session-manager/manager";
|
||||
@@ -10,6 +10,13 @@ import { rpcError } from "./errors";
|
||||
export interface SnapshotRefLookup {
|
||||
backendNodeId: number;
|
||||
refKey: string;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
}
|
||||
|
||||
function refEntryForTab(ctx: SessionContext, refKey: string, tabId: number) {
|
||||
const entry = ctx.refStore.resolveEntry(refKey);
|
||||
return entry?.tabId === tabId ? entry : null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -23,9 +30,14 @@ export function lookupSnapshotRef(
|
||||
tabId: number,
|
||||
): SnapshotRefLookup | null {
|
||||
const refKey = normaliseRef(ref);
|
||||
const backendNodeId = ctx.refStore.resolve(refKey, { tabId });
|
||||
if (backendNodeId === null) return null;
|
||||
return { backendNodeId, refKey };
|
||||
const entry = refEntryForTab(ctx, refKey, tabId);
|
||||
if (!entry) return null;
|
||||
return {
|
||||
backendNodeId: entry.backendNodeId,
|
||||
refKey,
|
||||
...(entry.frameId ? { frameId: entry.frameId } : {}),
|
||||
...(entry.cdpSessionId ? { cdpSessionId: entry.cdpSessionId } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -37,7 +49,7 @@ export function resolveSnapshotRef(
|
||||
ctx: SessionContext,
|
||||
ref: string,
|
||||
tabId: number,
|
||||
): { backendNodeId: number; refKey: string } | RpcError {
|
||||
): SnapshotRefLookup | RpcError {
|
||||
const looked = lookupSnapshotRef(ctx, ref, tabId);
|
||||
if (looked === null) {
|
||||
return rpcError(
|
||||
|
||||
@@ -18,6 +18,8 @@ function fakeSnapshotReply() {
|
||||
"auto",
|
||||
"type",
|
||||
"password",
|
||||
"value",
|
||||
"hunter2",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
return {
|
||||
@@ -29,7 +31,8 @@ function fakeSnapshotReply() {
|
||||
nodeType: [1, 1, 1, 1],
|
||||
nodeName: [i("html"), i("body"), i("div"), i("input")],
|
||||
backendNodeId: [10, 11, 12, 13],
|
||||
attributes: [[], [], [], [i("type"), i("password")]],
|
||||
attributes: [[], [], [], [i("type"), i("password"), i("value"), i("hunter2")]],
|
||||
inputValue: { index: [3], value: [i("hunter2")] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 3],
|
||||
@@ -217,7 +220,8 @@ function makeCdp(snapshot: unknown) {
|
||||
{
|
||||
state: "filled",
|
||||
sensitive: true,
|
||||
defaultValue: "",
|
||||
defaultValue: "hunter2",
|
||||
value: "hunter2",
|
||||
placeholder: "secret",
|
||||
},
|
||||
],
|
||||
@@ -262,10 +266,11 @@ describe("captureViewModel", () => {
|
||||
|
||||
expect(input).toMatchObject({
|
||||
formState: "filled",
|
||||
formDefaultValue: "",
|
||||
formPlaceholder: "secret",
|
||||
attrs: { type: "password" },
|
||||
});
|
||||
expect(input?.formValue).toBeUndefined();
|
||||
expect(input?.formDefaultValue).toBeUndefined();
|
||||
expect(cdp.send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
@@ -692,9 +697,7 @@ describe("captureViewModel", () => {
|
||||
};
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
const div = nodes.find((n) => n.backendNodeId === 12);
|
||||
// local rect preserves viewport-relative scroll math.
|
||||
expect(div?.localRect?.y).toBe(-200);
|
||||
// exported rect is clipped to the top-level viewport before VOM consumes it.
|
||||
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
|
||||
});
|
||||
|
||||
@@ -839,6 +842,7 @@ describe("captureViewModel", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "main-frame",
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
@@ -866,6 +870,7 @@ describe("captureViewModel", () => {
|
||||
},
|
||||
// documents[1]: the iframe's sub-document with a login input
|
||||
{
|
||||
frameId: "child-frame",
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
@@ -895,7 +900,14 @@ describe("captureViewModel", () => {
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes, iframeNodes } = await captureViewModel(cdp, 4);
|
||||
const {
|
||||
nodes,
|
||||
iframeNodes,
|
||||
frameNodes,
|
||||
frameOwnerBackendNodeIds,
|
||||
frameParentIds,
|
||||
rootFrameId,
|
||||
} = await captureViewModel(cdp, 4);
|
||||
// Main frame has 4 nodes; iframe is included
|
||||
expect(nodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
|
||||
// iframeNodes keyed by the <iframe> element's backendNodeId=13
|
||||
@@ -908,9 +920,13 @@ describe("captureViewModel", () => {
|
||||
expect(input?.ownerFrameBackendNodeId).toBe(13);
|
||||
expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
|
||||
expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
|
||||
expect(rootFrameId).toBe("main-frame");
|
||||
expect(frameNodes?.get("child-frame")).toBe(subNodes);
|
||||
expect(frameOwnerBackendNodeIds?.get("child-frame")).toBe(13);
|
||||
expect(frameParentIds?.get("child-frame")).toBe("main-frame");
|
||||
});
|
||||
|
||||
it("clips iframe sub-document rects to the iframe viewport in top coordinates", async () => {
|
||||
it("captures iframe documents without requiring owner-frame geometry", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
@@ -938,16 +954,10 @@ describe("captureViewModel", () => {
|
||||
contentDocumentIndex: { index: [2], value: [1] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[950, 760, 200, 100],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
nodeIndex: [1],
|
||||
styles: [[i("static"), i("auto")]],
|
||||
bounds: [[0, 0, 1000, 800]],
|
||||
paintOrders: [0],
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -985,7 +995,7 @@ describe("captureViewModel", () => {
|
||||
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101);
|
||||
|
||||
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
|
||||
expect(input?.rect).toEqual({ x: 970, y: 780, w: 30, h: 20 });
|
||||
expect(input?.rect).toBeNull();
|
||||
});
|
||||
|
||||
it("recursively normalizes nested iframe sub-documents", async () => {
|
||||
@@ -1080,8 +1090,10 @@ describe("captureViewModel", () => {
|
||||
const nestedIframe = iframeNodes.get(13)?.find((n) => n.backendNodeId === 23);
|
||||
const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31);
|
||||
|
||||
expect(nestedIframe?.localRect).toEqual({ x: 5, y: 6, w: 100, h: 80 });
|
||||
expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
|
||||
expect(input?.ownerFrameBackendNodeId).toBe(23);
|
||||
expect(input?.localRect).toEqual({ x: 1, y: 2, w: 40, h: 20 });
|
||||
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { CdpRunner } from "../../shared";
|
||||
import type { CapturedNode, CapturedViewModel } from "../capture";
|
||||
import { captureFrameData } from "../frame-capture";
|
||||
|
||||
function ownerNode(backendNodeId: number, x: number): CapturedNode {
|
||||
return {
|
||||
backendNodeId,
|
||||
parentBackendNodeId: 1,
|
||||
frameId: "main",
|
||||
ownerFrameBackendNodeId: null,
|
||||
tag: "iframe",
|
||||
attrs: {},
|
||||
rect: { x, y: 100, w: 300, h: 200 },
|
||||
localRect: { x, y: 100, w: 300, h: 200 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
};
|
||||
}
|
||||
|
||||
function childSnapshot(frameId: string, backendNodeId: number) {
|
||||
const strings = [frameId, "body", "button", "static", "auto", "pointer"];
|
||||
return {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
frameId,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [1, 2],
|
||||
backendNodeId: [backendNodeId - 1, backendNodeId],
|
||||
attributes: [[], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[3, 4, 4],
|
||||
[3, 4, 5],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 300, 200],
|
||||
[10, 20, 100, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe("captureFrameData", () => {
|
||||
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
|
||||
const leftOwner = ownerNode(10, 50);
|
||||
const rightOwner = ownerNode(20, 500);
|
||||
const captured: CapturedViewModel = {
|
||||
nodes: [leftOwner, rightOwner],
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
frameNodes: new Map([["main", [leftOwner, rightOwner]]]),
|
||||
frameOwnerBackendNodeIds: new Map(),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
const sendToTarget = vi.fn(async (target, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return target.sessionId === "left-session"
|
||||
? childSnapshot("left", 101)
|
||||
: childSnapshot("right", 201);
|
||||
}
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
throw new Error(`unexpected ${method}`);
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method, params) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
if (method === "DOM.getBoxModel") {
|
||||
const backendNodeId = (params as { backendNodeId?: number })?.backendNodeId;
|
||||
const x = backendNodeId === 10 ? 50 : 500;
|
||||
return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
|
||||
}
|
||||
throw new Error(`unexpected root ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "left",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "left-session" },
|
||||
},
|
||||
{
|
||||
frameId: "right",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 4, sessionId: "right-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const trees = await captureFrameData(cdp, 4, captured);
|
||||
|
||||
expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]);
|
||||
expect(
|
||||
captured.frameNodes?.get("left")?.find((node) => node.backendNodeId === 101)?.rect,
|
||||
).toEqual({ x: 60, y: 120, w: 100, h: 40 });
|
||||
expect(
|
||||
captured.frameNodes?.get("right")?.find((node) => node.backendNodeId === 201)?.rect,
|
||||
).toEqual({ x: 510, y: 120, w: 100, h: 40 });
|
||||
expect(captured.frameOwnerBackendNodeIds).toEqual(
|
||||
new Map([
|
||||
["left", 10],
|
||||
["right", 20],
|
||||
]),
|
||||
);
|
||||
expect(captured.frameParentIds).toEqual(
|
||||
new Map([
|
||||
["left", "main"],
|
||||
["right", "main"],
|
||||
]),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,174 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||
import type { CapturedNode, CapturedViewModel } from "../capture";
|
||||
import { buildFrameDocuments, type FrameAxBatch, type FrameOwnedAxNode } from "../frame-document";
|
||||
|
||||
function domNode(frameId: string, backendNodeId: number): CapturedNode {
|
||||
return {
|
||||
backendNodeId,
|
||||
parentBackendNodeId: null,
|
||||
frameId,
|
||||
tag: "div",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 20, h: 20 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
};
|
||||
}
|
||||
|
||||
function captured(frameNodes: Map<string, CapturedNode[]>): CapturedViewModel {
|
||||
return {
|
||||
nodes: frameNodes.get("main") ?? [],
|
||||
viewport: { width: 800, height: 600 },
|
||||
iframeNodes: new Map(),
|
||||
frameNodes,
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
}
|
||||
|
||||
describe("buildFrameDocuments", () => {
|
||||
it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", () => {
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{ frameId: "child", parentFrameId: "main", target: { tabId: 4 } },
|
||||
],
|
||||
};
|
||||
const batches: FrameAxBatch<FrameOwnedAxNode>[] = [
|
||||
{
|
||||
frame: graph.frames[0],
|
||||
nodes: [
|
||||
{ nodeId: "root", frameId: "main", backendDOMNodeId: 1, childIds: ["h"] },
|
||||
{
|
||||
nodeId: "h",
|
||||
frameId: "child",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 100,
|
||||
childIds: ["virtual"],
|
||||
},
|
||||
{ nodeId: "virtual", parentId: "h" },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const documents = buildFrameDocuments(
|
||||
graph,
|
||||
batches,
|
||||
captured(
|
||||
new Map([
|
||||
["main", [domNode("main", 1)]],
|
||||
["child", [domNode("child", 100)]],
|
||||
]),
|
||||
),
|
||||
);
|
||||
|
||||
expect(documents.find((document) => document.frameId === "main")?.axNodes).toHaveLength(1);
|
||||
const childNodes = documents.find((document) => document.frameId === "child")?.axNodes ?? [];
|
||||
expect(childNodes.map((node) => node.nodeId)).toEqual(["h", "virtual"]);
|
||||
expect(childNodes.find((node) => node.nodeId === "h")?.parentId).toBeUndefined();
|
||||
expect(childNodes.find((node) => node.nodeId === "virtual")?.frameId).toBe("child");
|
||||
});
|
||||
|
||||
it("deduplicates overlapping same-target AX results using the strongest ownership", () => {
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{ frameId: "child", parentFrameId: "main", target: { tabId: 4 } },
|
||||
],
|
||||
};
|
||||
const childNode = { nodeId: "shared", backendDOMNodeId: 100 };
|
||||
const batches: FrameAxBatch<FrameOwnedAxNode>[] = [
|
||||
{
|
||||
frame: graph.frames[0],
|
||||
nodes: [{ ...childNode, frameId: "child" }],
|
||||
},
|
||||
{
|
||||
frame: graph.frames[1],
|
||||
nodes: [childNode],
|
||||
},
|
||||
];
|
||||
|
||||
const documents = buildFrameDocuments(
|
||||
graph,
|
||||
batches,
|
||||
captured(
|
||||
new Map([
|
||||
["main", [domNode("main", 1)]],
|
||||
["child", [domNode("child", 100)]],
|
||||
]),
|
||||
),
|
||||
);
|
||||
|
||||
expect(documents.find((document) => document.frameId === "main")?.axNodes).toEqual([]);
|
||||
expect(documents.find((document) => document.frameId === "child")?.axNodes).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("uses target-scoped backend ownership when OOPIF backend ids collide", () => {
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "left",
|
||||
parentFrameId: "main",
|
||||
target: { tabId: 4, sessionId: "left-session" },
|
||||
},
|
||||
{
|
||||
frameId: "right",
|
||||
parentFrameId: "main",
|
||||
target: { tabId: 4, sessionId: "right-session" },
|
||||
},
|
||||
],
|
||||
};
|
||||
const batches: FrameAxBatch<FrameOwnedAxNode>[] = graph.frames.slice(1).map((frame) => ({
|
||||
frame,
|
||||
nodes: [{ nodeId: `${frame.frameId}-node`, backendDOMNodeId: 7 }],
|
||||
}));
|
||||
|
||||
const documents = buildFrameDocuments(
|
||||
graph,
|
||||
batches,
|
||||
captured(
|
||||
new Map([
|
||||
["main", []],
|
||||
["left", [domNode("left", 7)]],
|
||||
["right", [domNode("right", 7)]],
|
||||
]),
|
||||
),
|
||||
);
|
||||
|
||||
expect(documents.find((document) => document.frameId === "left")?.axNodes[0]?.nodeId).toBe(
|
||||
"left-node",
|
||||
);
|
||||
expect(documents.find((document) => document.frameId === "right")?.axNodes[0]?.nodeId).toBe(
|
||||
"right-node",
|
||||
);
|
||||
});
|
||||
|
||||
it("uses captured frame parent identity instead of guessing from backend ids", () => {
|
||||
const frameNodes = new Map([
|
||||
["main", [domNode("main", 1)]],
|
||||
["left", [domNode("left", 7)]],
|
||||
["right", [domNode("right", 7)]],
|
||||
["child", [domNode("child", 9)]],
|
||||
]);
|
||||
const capture = captured(frameNodes);
|
||||
capture.frameOwnerBackendNodeIds = new Map([["child", 7]]);
|
||||
capture.frameParentIds = new Map([["child", "left"]]);
|
||||
const frames = ["main", "left", "right", "child"].map((frameId) => ({
|
||||
frame: { frameId, target: { tabId: 4 } },
|
||||
nodes: [],
|
||||
}));
|
||||
|
||||
const documents = buildFrameDocuments(null, frames, capture);
|
||||
|
||||
expect(documents.find((document) => document.frameId === "child")).toMatchObject({
|
||||
parentFrameId: "left",
|
||||
ownerBackendNodeId: 7,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -7,6 +7,7 @@
|
||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
||||
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
const REQUESTED_STYLES = ["position", "pointer-events", "cursor"] as const;
|
||||
@@ -17,13 +18,14 @@ const STYLE_COL = Object.fromEntries(
|
||||
export interface CapturedNode {
|
||||
backendNodeId: number;
|
||||
parentBackendNodeId: number | null;
|
||||
frameId?: string;
|
||||
/** Owning iframe backend node id; `null` for the top-level document. */
|
||||
ownerFrameBackendNodeId?: number | null;
|
||||
tag: string;
|
||||
attrs: Record<string, string>;
|
||||
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
|
||||
rect: Rect | null;
|
||||
/** Frame-local viewport-relative CSS px before top-level translation/clipping. */
|
||||
/** Frame-local viewport-relative CSS px before top-level projection. */
|
||||
localRect?: Rect | null;
|
||||
paintOrder: number;
|
||||
position: string;
|
||||
@@ -57,6 +59,11 @@ export interface CapturedViewModel {
|
||||
nodes: CapturedNode[];
|
||||
viewport: Viewport;
|
||||
iframeNodes: CapturedIframeNodes;
|
||||
frameNodes?: Map<string, CapturedNode[]>;
|
||||
frameOwnerBackendNodeIds?: Map<string, number>;
|
||||
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
|
||||
frameParentIds?: Map<string, string>;
|
||||
rootFrameId?: string;
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
@@ -92,7 +99,12 @@ interface SparseArray {
|
||||
value: number[];
|
||||
}
|
||||
|
||||
interface RareBooleanData {
|
||||
index: number[];
|
||||
}
|
||||
|
||||
interface SnapshotDocument {
|
||||
frameId?: string | number;
|
||||
scrollOffsetX?: number;
|
||||
scrollOffsetY?: number;
|
||||
nodes?: {
|
||||
@@ -107,6 +119,10 @@ interface SnapshotDocument {
|
||||
nodeValue?: number[];
|
||||
/** Maps node array index → index into `documents[]` for frame content. */
|
||||
contentDocumentIndex?: SparseArray;
|
||||
inputValue?: SparseArray;
|
||||
textValue?: SparseArray;
|
||||
inputChecked?: RareBooleanData;
|
||||
optionSelected?: RareBooleanData;
|
||||
};
|
||||
layout?: {
|
||||
nodeIndex?: number[];
|
||||
@@ -116,6 +132,12 @@ interface SnapshotDocument {
|
||||
};
|
||||
}
|
||||
|
||||
function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined {
|
||||
if (typeof document.frameId === "string") return document.frameId || undefined;
|
||||
if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
interface SnapshotReply {
|
||||
strings?: string[];
|
||||
documents?: SnapshotDocument[];
|
||||
@@ -135,6 +157,25 @@ function isFormControlTag(tag: string): boolean {
|
||||
return tag === "input" || tag === "textarea" || tag === "select";
|
||||
}
|
||||
|
||||
function isSensitiveFormControl(node: Pick<CapturedNode, "tag" | "attrs">): boolean {
|
||||
return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password";
|
||||
}
|
||||
|
||||
function snapshotFormState(
|
||||
value: string | undefined,
|
||||
defaultValue: string,
|
||||
hasDefaultValue: boolean,
|
||||
sensitive: boolean,
|
||||
): CapturedNode["formState"] {
|
||||
if (sensitive) {
|
||||
if (value === undefined && !hasDefaultValue) return undefined;
|
||||
return (value ?? defaultValue) === "" ? "empty" : "filled";
|
||||
}
|
||||
if (value === undefined) return undefined;
|
||||
if (value === "") return "empty";
|
||||
return value === defaultValue ? "default" : "filled";
|
||||
}
|
||||
|
||||
const MAX_FORM_ENRICH_CONTROLS = 250;
|
||||
|
||||
interface CapturedFormState {
|
||||
@@ -166,9 +207,8 @@ function formStateBatchExpression(maxControls: number): string {
|
||||
return {
|
||||
state,
|
||||
sensitive,
|
||||
defaultValue,
|
||||
placeholder,
|
||||
...(sensitive ? {} : { value: rawValue }),
|
||||
...(sensitive ? {} : { value: rawValue, defaultValue }),
|
||||
};
|
||||
};
|
||||
const collect = (doc) => {
|
||||
@@ -210,10 +250,14 @@ function applyFormStates(nodes: CapturedNode[], states: CapturedFormState[]): vo
|
||||
index += 1;
|
||||
if (!state) continue;
|
||||
node.formState = state.state;
|
||||
node.formDefaultValue = state.defaultValue ?? "";
|
||||
node.formPlaceholder = state.placeholder ?? "";
|
||||
if (!state.sensitive && state.value !== undefined) {
|
||||
node.formValue = state.value;
|
||||
if (state.sensitive || isSensitiveFormControl(node)) {
|
||||
delete node.formValue;
|
||||
delete node.formDefaultValue;
|
||||
delete node.attrs.value;
|
||||
} else {
|
||||
node.formDefaultValue = state.defaultValue ?? "";
|
||||
if (state.value !== undefined) node.formValue = state.value;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -272,9 +316,9 @@ interface ParseDocumentResult {
|
||||
}
|
||||
|
||||
interface FrameContext {
|
||||
frameId?: string;
|
||||
ownerFrameBackendNodeId: number | null;
|
||||
originInTop: { x: number; y: number };
|
||||
clipRectInTop: Rect;
|
||||
projection: GeometryProjection | null;
|
||||
scrollX: number;
|
||||
scrollY: number;
|
||||
}
|
||||
@@ -293,38 +337,6 @@ function devicePixelRatio(metrics: LayoutMetricsReply): number {
|
||||
return dpr >= 1 ? dpr : 1;
|
||||
}
|
||||
|
||||
function viewportRect(viewport: Viewport): Rect {
|
||||
return { x: 0, y: 0, w: Math.max(0, viewport.width), h: Math.max(0, viewport.height) };
|
||||
}
|
||||
|
||||
function intersectRects(a: Rect, b: Rect): Rect | null {
|
||||
const x1 = Math.max(a.x, b.x);
|
||||
const y1 = Math.max(a.y, b.y);
|
||||
const x2 = Math.min(a.x + a.w, b.x + b.w);
|
||||
const y2 = Math.min(a.y + a.h, b.y + b.h);
|
||||
if (x2 <= x1 || y2 <= y1) return null;
|
||||
return { x: x1, y: y1, w: x2 - x1, h: y2 - y1 };
|
||||
}
|
||||
|
||||
function rectInTop(localRect: Rect, context: FrameContext): Rect | null {
|
||||
const translated = {
|
||||
x: context.originInTop.x + localRect.x,
|
||||
y: context.originInTop.y + localRect.y,
|
||||
w: localRect.w,
|
||||
h: localRect.h,
|
||||
};
|
||||
return intersectRects(translated, context.clipRectInTop);
|
||||
}
|
||||
|
||||
function unclipRectInTop(localRect: Rect, context: FrameContext): Rect {
|
||||
return {
|
||||
x: context.originInTop.x + localRect.x,
|
||||
y: context.originInTop.y + localRect.y,
|
||||
w: localRect.w,
|
||||
h: localRect.h,
|
||||
};
|
||||
}
|
||||
|
||||
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
|
||||
const out = new Map<number, number>();
|
||||
if (!sparse?.index || !sparse.value) return out;
|
||||
@@ -805,6 +817,10 @@ function parseDocumentNodes(
|
||||
const posCol = STYLE_COL.position;
|
||||
const peCol = STYLE_COL["pointer-events"];
|
||||
const cursorCol = STYLE_COL.cursor;
|
||||
const inputValues = sparseIndexMap(dn.inputValue);
|
||||
const textValues = sparseIndexMap(dn.textValue);
|
||||
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
|
||||
const selectedOptions = new Set(dn.optionSelected?.index ?? []);
|
||||
|
||||
// Collect visible text from #text child nodes so element CapturedNodes
|
||||
// carry a textContent value usable as a button/link label fallback.
|
||||
@@ -855,7 +871,10 @@ function parseDocumentNodes(
|
||||
w: b[2] / dpr,
|
||||
h: b[3] / dpr,
|
||||
};
|
||||
rect = rectInTop(localRect, context);
|
||||
const bounds = context.projection
|
||||
? projectRectToViewport(localRect, context.projection)
|
||||
: null;
|
||||
rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||
}
|
||||
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
||||
const styleRow = dl?.styles?.[li] ?? [];
|
||||
@@ -870,9 +889,24 @@ function parseDocumentNodes(
|
||||
|
||||
const textContent = nodeTextContent.get(n);
|
||||
|
||||
const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n);
|
||||
const rawFormValue =
|
||||
rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined;
|
||||
const sensitive = isSensitiveFormControl({ tag, attrs });
|
||||
const formDefaultValue = attrs.value ?? "";
|
||||
const formValue = sensitive ? undefined : rawFormValue;
|
||||
const formState = snapshotFormState(
|
||||
rawFormValue,
|
||||
formDefaultValue,
|
||||
Object.prototype.hasOwnProperty.call(attrs, "value"),
|
||||
sensitive,
|
||||
);
|
||||
if (sensitive) delete attrs.value;
|
||||
|
||||
nodes.push({
|
||||
backendNodeId,
|
||||
parentBackendNodeId,
|
||||
...(context.frameId ? { frameId: context.frameId } : {}),
|
||||
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
|
||||
tag,
|
||||
attrs,
|
||||
@@ -883,6 +917,16 @@ function parseDocumentNodes(
|
||||
pointerEvents,
|
||||
cursor,
|
||||
textContent,
|
||||
...(formValue !== undefined ? { formValue } : {}),
|
||||
...(tag === "input" || tag === "textarea"
|
||||
? {
|
||||
formPlaceholder: attrs.placeholder ?? "",
|
||||
...(!sensitive ? { formDefaultValue } : {}),
|
||||
...(formState ? { formState } : {}),
|
||||
}
|
||||
: {}),
|
||||
...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}),
|
||||
...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}),
|
||||
});
|
||||
}
|
||||
return { nodes, excludedBackendNodeIds };
|
||||
@@ -890,6 +934,8 @@ function parseDocumentNodes(
|
||||
|
||||
interface ParseFrameDocumentsResult {
|
||||
iframeNodes: CapturedIframeNodes;
|
||||
frameOwnerBackendNodeIds: Map<string, number>;
|
||||
frameParentIds: Map<string, string>;
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
@@ -903,10 +949,14 @@ function parseChildFrameDocuments(
|
||||
visited = new Set<number>(),
|
||||
): ParseFrameDocumentsResult {
|
||||
const iframeNodes: CapturedIframeNodes = new Map();
|
||||
const frameOwnerBackendNodeIds = new Map<string, number>();
|
||||
const frameParentIds = new Map<string, string>();
|
||||
const excludedBackendNodeIds = new Set<number>();
|
||||
const parentDoc = documents[parentDocIndex];
|
||||
const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex);
|
||||
if (cdi.size === 0) return { iframeNodes, excludedBackendNodeIds };
|
||||
if (cdi.size === 0) {
|
||||
return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
|
||||
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
|
||||
@@ -918,19 +968,15 @@ function parseChildFrameDocuments(
|
||||
const iframeBackendId = parentBackendIds[nodeArrayIdx];
|
||||
if (iframeBackendId === undefined) continue;
|
||||
const iframeNode = parentNodeByBackendId.get(iframeBackendId);
|
||||
if (!iframeNode?.localRect) continue;
|
||||
|
||||
const iframeRectInTop = unclipRectInTop(iframeNode.localRect, parentContext);
|
||||
const childClip = intersectRects(iframeRectInTop, parentContext.clipRectInTop);
|
||||
if (!childClip) {
|
||||
iframeNodes.set(iframeBackendId, []);
|
||||
continue;
|
||||
}
|
||||
const projection =
|
||||
parentContext.projection && iframeNode?.localRect
|
||||
? childFrameProjection(parentContext.projection, iframeNode.localRect)
|
||||
: null;
|
||||
|
||||
const childContext: FrameContext = {
|
||||
frameId: snapshotFrameId(childDoc, strings),
|
||||
ownerFrameBackendNodeId: iframeBackendId,
|
||||
originInTop: { x: iframeRectInTop.x, y: iframeRectInTop.y },
|
||||
clipRectInTop: childClip,
|
||||
projection,
|
||||
scrollX: childDoc.scrollOffsetX ?? 0,
|
||||
scrollY: childDoc.scrollOffsetY ?? 0,
|
||||
};
|
||||
@@ -938,6 +984,10 @@ function parseChildFrameDocuments(
|
||||
nextVisited.add(childDocIndex);
|
||||
const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext);
|
||||
iframeNodes.set(iframeBackendId, parsed.nodes);
|
||||
if (childContext.frameId) {
|
||||
frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId);
|
||||
if (parentContext.frameId) frameParentIds.set(childContext.frameId, parentContext.frameId);
|
||||
}
|
||||
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||
|
||||
const nested = parseChildFrameDocuments(
|
||||
@@ -952,10 +1002,16 @@ function parseChildFrameDocuments(
|
||||
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
|
||||
iframeNodes.set(nestedFrameId, nestedNodes);
|
||||
}
|
||||
for (const [frameId, ownerBackendNodeId] of nested.frameOwnerBackendNodeIds) {
|
||||
frameOwnerBackendNodeIds.set(frameId, ownerBackendNodeId);
|
||||
}
|
||||
for (const [frameId, parentFrameId] of nested.frameParentIds) {
|
||||
frameParentIds.set(frameId, parentFrameId);
|
||||
}
|
||||
for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||
}
|
||||
|
||||
return { iframeNodes, excludedBackendNodeIds };
|
||||
return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
export async function captureViewModel(
|
||||
@@ -998,9 +1054,13 @@ export async function captureViewModel(
|
||||
}
|
||||
|
||||
const topContext: FrameContext = {
|
||||
frameId: snapshotFrameId(doc0, strings),
|
||||
ownerFrameBackendNodeId: null,
|
||||
originInTop: { x: 0, y: 0 },
|
||||
clipRectInTop: viewportRect(viewport),
|
||||
projection: {
|
||||
sourceClips: [],
|
||||
edges: [],
|
||||
topViewport: viewport,
|
||||
},
|
||||
scrollX,
|
||||
scrollY,
|
||||
};
|
||||
@@ -1022,5 +1082,27 @@ export async function captureViewModel(
|
||||
: [];
|
||||
throwIfAborted(options.signal);
|
||||
|
||||
return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds };
|
||||
const frameNodes = new Map<string, CapturedNode[]>();
|
||||
if (topContext.frameId) frameNodes.set(topContext.frameId, nodes);
|
||||
for (const iframeFrameNodes of iframeNodes.values()) {
|
||||
const frameId = iframeFrameNodes.find((node) => node.frameId)?.frameId;
|
||||
if (frameId) frameNodes.set(frameId, iframeFrameNodes);
|
||||
}
|
||||
for (const [frameId, ownerBackendNodeId] of frameParsed.frameOwnerBackendNodeIds) {
|
||||
if (!frameNodes.has(frameId)) {
|
||||
frameNodes.set(frameId, iframeNodes.get(ownerBackendNodeId) ?? []);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
nodes,
|
||||
viewport,
|
||||
iframeNodes,
|
||||
frameNodes,
|
||||
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
|
||||
frameParentIds: frameParsed.frameParentIds,
|
||||
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
|
||||
surfaceProbes,
|
||||
excludedBackendNodeIds,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||
import { resolveFrameProjection } from "../frame-geometry";
|
||||
import { type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
|
||||
import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
|
||||
import {
|
||||
buildFrameDocuments,
|
||||
type FrameAxBatch,
|
||||
type FrameDocument,
|
||||
type FrameOwnedAxNode,
|
||||
} from "./frame-document";
|
||||
|
||||
export type FrameAxNode = FrameOwnedAxNode;
|
||||
export type CapturedFrameDocument<T extends FrameAxNode> = FrameDocument<T>;
|
||||
|
||||
function throwIfAborted(signal?: AbortSignal): void {
|
||||
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
|
||||
}
|
||||
|
||||
function isAbortError(err: unknown): boolean {
|
||||
return err instanceof Error && err.name === "AbortError";
|
||||
}
|
||||
|
||||
async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
|
||||
if (!cdp.getFrameGraph) return null;
|
||||
try {
|
||||
return await cdp.getFrameGraph(tabId);
|
||||
} catch (err) {
|
||||
console.debug("[bsk observation] frame graph capture failed", err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function transformFrameNodes(
|
||||
nodes: CapturedNode[],
|
||||
projection: GeometryProjection,
|
||||
): CapturedNode[] {
|
||||
return nodes.map((node) => ({
|
||||
...node,
|
||||
rect: (() => {
|
||||
const bounds = projectRectToViewport(node.rect, projection);
|
||||
return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||
})(),
|
||||
}));
|
||||
}
|
||||
|
||||
function frameOwnerNode(captured: CapturedViewModel, frame: CdpFrame): CapturedNode | undefined {
|
||||
if (frame.ownerBackendNodeId === undefined) return undefined;
|
||||
const parentNodes = frame.parentFrameId
|
||||
? captured.frameNodes?.get(frame.parentFrameId)
|
||||
: captured.nodes;
|
||||
return parentNodes?.find((node) => node.backendNodeId === frame.ownerBackendNodeId);
|
||||
}
|
||||
|
||||
async function captureMissingFrameDocuments(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
graph: CdpFrameGraph,
|
||||
captured: CapturedViewModel,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
if (!captured.frameNodes) captured.frameNodes = new Map();
|
||||
if (!captured.frameOwnerBackendNodeIds) captured.frameOwnerBackendNodeIds = new Map();
|
||||
if (!captured.frameParentIds) captured.frameParentIds = new Map();
|
||||
|
||||
for (const frame of graph.frames) {
|
||||
if (!frame.target.sessionId || captured.frameNodes.has(frame.frameId)) continue;
|
||||
const owner = frameOwnerNode(captured, frame);
|
||||
if (!owner?.rect || frame.ownerBackendNodeId === undefined) continue;
|
||||
try {
|
||||
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
|
||||
signal,
|
||||
});
|
||||
const projection = await resolveFrameProjection(cdp, graph, frame.frameId);
|
||||
if (!projection) continue;
|
||||
const rootFrameId = child.rootFrameId ?? frame.frameId;
|
||||
const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
|
||||
if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
|
||||
if (rootFrameId !== frame.frameId && !childFrames.has(frame.frameId)) {
|
||||
childFrames.set(
|
||||
frame.frameId,
|
||||
child.nodes.map((node) => ({ ...node, frameId: frame.frameId })),
|
||||
);
|
||||
}
|
||||
for (const [childFrameId, nodes] of childFrames) {
|
||||
captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
|
||||
}
|
||||
captured.iframeNodes.set(
|
||||
frame.ownerBackendNodeId,
|
||||
captured.frameNodes.get(frame.frameId) ?? [],
|
||||
);
|
||||
captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
|
||||
if (frame.parentFrameId) captured.frameParentIds.set(frame.frameId, frame.parentFrameId);
|
||||
for (const [childFrameId, ownerBackendNodeId] of child.frameOwnerBackendNodeIds ?? []) {
|
||||
captured.frameOwnerBackendNodeIds.set(childFrameId, ownerBackendNodeId);
|
||||
}
|
||||
for (const [childFrameId, parentFrameId] of child.frameParentIds ?? []) {
|
||||
captured.frameParentIds.set(childFrameId, parentFrameId);
|
||||
}
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
console.debug("[bsk observation] child frame DOM capture failed", {
|
||||
frameId: frame.frameId,
|
||||
err,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function captureAxTrees<T extends FrameAxNode>(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
discoveredFrames: CdpFrame[],
|
||||
captured: CapturedViewModel,
|
||||
signal?: AbortSignal,
|
||||
): Promise<FrameAxBatch<T>[]> {
|
||||
let frames = [...discoveredFrames];
|
||||
const capturedFrameIds = [...(captured.frameNodes?.keys() ?? [])];
|
||||
if (frames.length === 0) {
|
||||
const rootFrameId = captured.rootFrameId ?? capturedFrameIds[0] ?? "root";
|
||||
frames = [{ frameId: rootFrameId, target: { tabId } }];
|
||||
}
|
||||
for (const frameId of capturedFrameIds) {
|
||||
if (!frames.some((frame) => frame.frameId === frameId)) {
|
||||
frames.push({ frameId, target: { tabId } });
|
||||
}
|
||||
}
|
||||
|
||||
let firstFailure: unknown;
|
||||
const trees = await Promise.all(
|
||||
frames.map(async (frame): Promise<FrameAxBatch<T>> => {
|
||||
try {
|
||||
throwIfAborted(signal);
|
||||
await sendToCdpTarget(cdp, frame.target, "Accessibility.enable", {});
|
||||
const result = await sendToCdpTarget<{ nodes?: T[] }>(
|
||||
cdp,
|
||||
frame.target,
|
||||
"Accessibility.getFullAXTree",
|
||||
frame.frameId === "root" ? {} : { frameId: frame.frameId },
|
||||
);
|
||||
throwIfAborted(signal);
|
||||
return {
|
||||
frame,
|
||||
nodes: result.nodes ?? [],
|
||||
};
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
firstFailure ??= err;
|
||||
console.debug("[bsk observation] frame AX capture failed", {
|
||||
frameId: frame.frameId,
|
||||
err,
|
||||
});
|
||||
return { frame, nodes: [] };
|
||||
}
|
||||
}),
|
||||
);
|
||||
const capturedNodeCount = [...captured.iframeNodes.values()].reduce(
|
||||
(count, nodes) => count + nodes.length,
|
||||
captured.nodes.length,
|
||||
);
|
||||
if (firstFailure && capturedNodeCount === 0 && trees.every((tree) => tree.nodes.length === 0)) {
|
||||
throw firstFailure;
|
||||
}
|
||||
return trees;
|
||||
}
|
||||
|
||||
export async function captureFrameData<T extends FrameAxNode>(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
captured: CapturedViewModel,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CapturedFrameDocument<T>[]> {
|
||||
const graph = await discoverFrameGraph(cdp, tabId);
|
||||
const frames = graph?.frames ?? [];
|
||||
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal);
|
||||
throwIfAborted(signal);
|
||||
const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
|
||||
return buildFrameDocuments(graph, batches, captured);
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
import {
|
||||
type CdpFrame,
|
||||
type CdpFrameGraph,
|
||||
type CdpTarget,
|
||||
cdpTargetKey,
|
||||
} from "@/browser-driver/frame-graph";
|
||||
import type { CapturedNode, CapturedViewModel } from "./capture";
|
||||
|
||||
export interface FrameOwnedAxNode {
|
||||
nodeId: string;
|
||||
frameId?: string;
|
||||
parentId?: string;
|
||||
childIds?: string[];
|
||||
backendDOMNodeId?: number;
|
||||
}
|
||||
|
||||
export interface FrameAxBatch<T extends FrameOwnedAxNode> {
|
||||
frame: CdpFrame;
|
||||
nodes: T[];
|
||||
}
|
||||
|
||||
export interface FrameDocument<T extends FrameOwnedAxNode> extends CdpFrame {
|
||||
contextScopeId: string;
|
||||
axNodes: T[];
|
||||
domNodes: CapturedNode[];
|
||||
}
|
||||
|
||||
interface Ownership {
|
||||
frameId: string;
|
||||
strength: number;
|
||||
}
|
||||
|
||||
interface OwnedCandidate<T> {
|
||||
node: T;
|
||||
target: CdpTarget;
|
||||
ownership: Ownership;
|
||||
}
|
||||
|
||||
function targetNodeKey(target: CdpTarget, nodeId: string): string {
|
||||
return `${cdpTargetKey(target)}:${nodeId}`;
|
||||
}
|
||||
|
||||
function targetBackendKey(target: CdpTarget, backendNodeId: number): string {
|
||||
return `${cdpTargetKey(target)}:${backendNodeId}`;
|
||||
}
|
||||
|
||||
function frameList<T extends FrameOwnedAxNode>(
|
||||
graph: CdpFrameGraph | null,
|
||||
batches: FrameAxBatch<T>[],
|
||||
captured: CapturedViewModel,
|
||||
): CdpFrame[] {
|
||||
const frames = new Map<string, CdpFrame>();
|
||||
for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame);
|
||||
for (const batch of batches) {
|
||||
if (!frames.has(batch.frame.frameId)) frames.set(batch.frame.frameId, batch.frame);
|
||||
}
|
||||
for (const frameId of captured.frameNodes?.keys() ?? []) {
|
||||
if (!frames.has(frameId))
|
||||
frames.set(frameId, { frameId, target: { tabId: batches[0]?.frame.target.tabId ?? 0 } });
|
||||
}
|
||||
return [...frames.values()].map((frame) => {
|
||||
const ownerBackendNodeId =
|
||||
frame.ownerBackendNodeId ?? captured.frameOwnerBackendNodeIds?.get(frame.frameId);
|
||||
const parentFrameId = frame.parentFrameId ?? captured.frameParentIds?.get(frame.frameId);
|
||||
return {
|
||||
...frame,
|
||||
...(ownerBackendNodeId !== undefined ? { ownerBackendNodeId } : {}),
|
||||
...(parentFrameId ? { parentFrameId } : {}),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
graph: CdpFrameGraph | null,
|
||||
batches: FrameAxBatch<T>[],
|
||||
captured: CapturedViewModel,
|
||||
): FrameDocument<T>[] {
|
||||
const frames = frameList(graph, batches, captured);
|
||||
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
|
||||
const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId;
|
||||
const domNodesForFrame = (frameId: string): CapturedNode[] =>
|
||||
captured.frameNodes?.get(frameId) ?? (frameId === rootFrameId ? captured.nodes : []);
|
||||
const backendOwner = new Map<string, string>();
|
||||
for (const frame of frames) {
|
||||
for (const node of domNodesForFrame(frame.frameId)) {
|
||||
backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId);
|
||||
}
|
||||
}
|
||||
|
||||
const candidates = new Map<string, OwnedCandidate<T>>();
|
||||
for (const batch of batches) {
|
||||
const nodeById = new Map(batch.nodes.map((node) => [node.nodeId, node]));
|
||||
const ownershipByNodeId = new Map<string, Ownership>();
|
||||
const resolving = new Set<string>();
|
||||
const resolveOwnership = (node: T): Ownership => {
|
||||
const cached = ownershipByNodeId.get(node.nodeId);
|
||||
if (cached) return cached;
|
||||
if (node.frameId && frameById.has(node.frameId)) {
|
||||
const ownership = { frameId: node.frameId, strength: 4 };
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
}
|
||||
if (typeof node.backendDOMNodeId === "number") {
|
||||
const frameId = backendOwner.get(
|
||||
targetBackendKey(batch.frame.target, node.backendDOMNodeId),
|
||||
);
|
||||
if (frameId) {
|
||||
const ownership = { frameId, strength: 3 };
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
}
|
||||
}
|
||||
if (node.parentId && !resolving.has(node.nodeId)) {
|
||||
const parent = nodeById.get(node.parentId);
|
||||
if (parent) {
|
||||
resolving.add(node.nodeId);
|
||||
const inherited = resolveOwnership(parent);
|
||||
resolving.delete(node.nodeId);
|
||||
const ownership = {
|
||||
frameId: inherited.frameId,
|
||||
strength: Math.min(2, inherited.strength),
|
||||
};
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
}
|
||||
}
|
||||
const ownership = { frameId: batch.frame.frameId, strength: 1 };
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
};
|
||||
|
||||
for (const node of batch.nodes) {
|
||||
const ownership = resolveOwnership(node);
|
||||
const key = targetNodeKey(batch.frame.target, node.nodeId);
|
||||
const existing = candidates.get(key);
|
||||
if (!existing || ownership.strength > existing.ownership.strength) {
|
||||
candidates.set(key, { node, target: batch.frame.target, ownership });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const ownershipByTargetNode = new Map(
|
||||
[...candidates].map(([key, candidate]) => [key, candidate.ownership.frameId]),
|
||||
);
|
||||
const axNodesByFrame = new Map<string, T[]>();
|
||||
for (const candidate of candidates.values()) {
|
||||
const { node, target, ownership } = candidate;
|
||||
const parentFrameId = node.parentId
|
||||
? ownershipByTargetNode.get(targetNodeKey(target, node.parentId))
|
||||
: undefined;
|
||||
const childIds = node.childIds?.filter(
|
||||
(childId) => ownershipByTargetNode.get(targetNodeKey(target, childId)) === ownership.frameId,
|
||||
);
|
||||
const ownedNode = {
|
||||
...node,
|
||||
frameId: ownership.frameId,
|
||||
...(parentFrameId === ownership.frameId
|
||||
? { parentId: node.parentId }
|
||||
: { parentId: undefined }),
|
||||
...(childIds ? { childIds } : {}),
|
||||
};
|
||||
const nodes = axNodesByFrame.get(ownership.frameId) ?? [];
|
||||
nodes.push(ownedNode);
|
||||
axNodesByFrame.set(ownership.frameId, nodes);
|
||||
}
|
||||
|
||||
return frames.map((frame) => ({
|
||||
...frame,
|
||||
contextScopeId: frame.frameId,
|
||||
axNodes: axNodesByFrame.get(frame.frameId) ?? [],
|
||||
domNodes: domNodesForFrame(frame.frameId),
|
||||
}));
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { CapturedSurfaceProbe } from "./capture";
|
||||
import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture";
|
||||
|
||||
/** Frame identity needed to resolve an `@eN` that lives in an iframe. */
|
||||
export interface CaptureVomFrame {
|
||||
frameId: string;
|
||||
target: CdpTarget;
|
||||
parentFrameId?: string;
|
||||
ownerBackendNodeId?: number;
|
||||
}
|
||||
|
||||
/** Allowlisted geometry used to match a recorded action to a rendered ref. */
|
||||
export interface CaptureVomMatchNode {
|
||||
frameId: string;
|
||||
backendNodeId: number;
|
||||
tag: string;
|
||||
/** Top-level viewport-relative geometry. */
|
||||
rect: Rect | null;
|
||||
/** Frame-local viewport-relative geometry, when available. */
|
||||
localRect?: Rect | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Raw AX/DOM trees are never returned. When `redactValues` is enabled by the
|
||||
* caller, form values in `text` are masked as well.
|
||||
*/
|
||||
export interface CaptureVomObservationResult {
|
||||
text: string;
|
||||
refs: RenderedRef[];
|
||||
truncated: boolean;
|
||||
rootFrameId: string;
|
||||
frames: CaptureVomFrame[];
|
||||
matchNodes: CaptureVomMatchNode[];
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
}
|
||||
|
||||
function projectFrames(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomFrame[] {
|
||||
return documents.map((document) => ({
|
||||
frameId: document.frameId,
|
||||
target: document.target,
|
||||
...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}),
|
||||
...(document.ownerBackendNodeId !== undefined
|
||||
? { ownerBackendNodeId: document.ownerBackendNodeId }
|
||||
: {}),
|
||||
}));
|
||||
}
|
||||
|
||||
function projectMatchNodes(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomMatchNode[] {
|
||||
return documents.flatMap((document) =>
|
||||
document.domNodes.map((node) => ({
|
||||
frameId: document.frameId,
|
||||
backendNodeId: node.backendNodeId,
|
||||
tag: node.tag,
|
||||
rect: node.rect,
|
||||
...(node.localRect !== undefined ? { localRect: node.localRect } : {}),
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
export function projectRecordSafeObservation(input: {
|
||||
rootFrameId: string;
|
||||
frameDocuments: CapturedFrameDocument<FrameAxNode>[];
|
||||
rendered: VomResult;
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
}): CaptureVomObservationResult {
|
||||
return {
|
||||
text: input.rendered.text,
|
||||
refs: input.rendered.refs,
|
||||
truncated: input.rendered.truncated,
|
||||
rootFrameId: input.rootFrameId,
|
||||
frames: projectFrames(input.frameDocuments),
|
||||
matchNodes: projectMatchNodes(input.frameDocuments),
|
||||
surfaceProbes: input.surfaceProbes,
|
||||
};
|
||||
}
|
||||
@@ -25,10 +25,8 @@ export default defineConfig({
|
||||
name: "BrowserSkill",
|
||||
description:
|
||||
"Let AI agents use your logged-in browser in a separate Agent Window—without interrupting your work. Powered by the bsk CLI.",
|
||||
// Chrome 116 is the first version where WebSocket activity resets the
|
||||
// service-worker idle timer, which the 20s heartbeat relies on to
|
||||
// keep the daemon link alive.
|
||||
minimum_chrome_version: "116",
|
||||
// Flat debugger sessions are required to address out-of-process iframes.
|
||||
minimum_chrome_version: "125",
|
||||
permissions: [
|
||||
"alarms",
|
||||
"debugger",
|
||||
|
||||
@@ -39,6 +39,11 @@
|
||||
"**",
|
||||
"!**/node_modules/**",
|
||||
"!**/dist/**",
|
||||
"!packages/dsh-plugin-browserskill/lib/**",
|
||||
"!packages/dsh-plugin-browserskill/src/skill-content.generated.ts",
|
||||
"!packages/dsh-plugin-browserskill/src/client/bsk-ui.nomodule.css",
|
||||
"!packages/dsh-plugin-browserskill/.bsk-ui.raw.css",
|
||||
"!packages/dsh-plugin-browserskill/.bsk-ui.input.css",
|
||||
"!**/target/**",
|
||||
"!**/.wxt/**",
|
||||
"!**/.output/**",
|
||||
|
||||
@@ -18,6 +18,12 @@ use crate::daemon::state::PROTOCOL_VERSION;
|
||||
const EXTENSION_STORE_URL: &str =
|
||||
"https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi";
|
||||
|
||||
/// Edge Add-ons listing for the browser-skill extension.
|
||||
const EXTENSION_STORE_URL_EDGE: &str = "https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg";
|
||||
|
||||
/// Store listings highlighted in repair hints, in the order they appear.
|
||||
const EXTENSION_STORE_URLS: [&str; 2] = [EXTENSION_STORE_URL, EXTENSION_STORE_URL_EDGE];
|
||||
|
||||
/// Status of a single doctor check. `Ok` / `Fail` are the legacy two
|
||||
/// states; `NotApplicable` (review M2) is reported as "N/A" in human
|
||||
/// output and as `"status": "na"` in `--json` output, so a check that
|
||||
@@ -420,7 +426,10 @@ fn check_extension_connected(status: Option<&StatusResult>) -> CheckResult {
|
||||
CheckResult::fail(
|
||||
name,
|
||||
"0 browsers connected",
|
||||
format!("install the extension from {EXTENSION_STORE_URL} and load it in Chromium"),
|
||||
format!(
|
||||
"install the extension from {EXTENSION_STORE_URL} (Chrome) \
|
||||
or {EXTENSION_STORE_URL_EDGE} (Edge) and load it in the browser"
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -428,17 +437,14 @@ fn check_extension_connected(status: Option<&StatusResult>) -> CheckResult {
|
||||
/// Highlight known URLs in repair hints for terminal output. Plain
|
||||
/// text is preserved in `--json` and in stored [`CheckResult::hint`].
|
||||
fn style_hint(hint: &str) -> String {
|
||||
if !hint.contains(EXTENSION_STORE_URL) {
|
||||
return hint.to_string();
|
||||
let mut styled = hint.to_string();
|
||||
for url in EXTENSION_STORE_URLS {
|
||||
if !styled.contains(url) {
|
||||
continue;
|
||||
}
|
||||
styled = styled.replace(url, &style(url).cyan().bold().underlined().to_string());
|
||||
}
|
||||
hint.replace(
|
||||
EXTENSION_STORE_URL,
|
||||
&style(EXTENSION_STORE_URL)
|
||||
.cyan()
|
||||
.bold()
|
||||
.underlined()
|
||||
.to_string(),
|
||||
)
|
||||
styled
|
||||
}
|
||||
|
||||
fn render_human(checks: &[CheckResult]) {
|
||||
@@ -504,6 +510,26 @@ mod m2_tests {
|
||||
hint.contains(EXTENSION_STORE_URL),
|
||||
"hint should include Chrome Web Store URL: {hint}"
|
||||
);
|
||||
assert!(
|
||||
hint.contains(EXTENSION_STORE_URL_EDGE),
|
||||
"hint should include Edge Add-ons URL: {hint}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn style_hint_preserves_every_store_url() {
|
||||
let hint = check_extension_connected(Some(&fake_status(Vec::new(), Vec::new())))
|
||||
.hint
|
||||
.expect("extension disconnected should include a hint");
|
||||
let styled = style_hint(&hint);
|
||||
// Whether or not the terminal accepts colors, styling must never drop or
|
||||
// mangle a URL the user has to click.
|
||||
for url in EXTENSION_STORE_URLS {
|
||||
assert!(
|
||||
styled.contains(url),
|
||||
"styled hint should still contain {url}: {styled}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -206,15 +206,15 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
|
||||
Ok(v) => ResponseBody::Ok(v),
|
||||
Err(e) => ResponseBody::Err(e),
|
||||
},
|
||||
Method::SessionStart => match handle_session_start(&state, params).await {
|
||||
Method::SessionStart => match handle_session_start(&state, rpc_id, params).await {
|
||||
Ok(v) => ResponseBody::Ok(v),
|
||||
Err(e) => ResponseBody::Err(e),
|
||||
},
|
||||
Method::SessionStop => match handle_session_stop(&state, params).await {
|
||||
Method::SessionStop => match handle_session_stop(&state, rpc_id, params).await {
|
||||
Ok(v) => ResponseBody::Ok(v),
|
||||
Err(e) => ResponseBody::Err(e),
|
||||
},
|
||||
Method::SessionStopAll => match handle_session_stop_all(&state).await {
|
||||
Method::SessionStopAll => match handle_session_stop_all_rpc(&state, rpc_id).await {
|
||||
Ok(v) => ResponseBody::Ok(v),
|
||||
Err(e) => ResponseBody::Err(e),
|
||||
},
|
||||
@@ -434,8 +434,9 @@ async fn handle_wait_ms(
|
||||
/// cancellation surfaces (M10.2 + review C2):
|
||||
///
|
||||
/// 1. [`AbortRegistry`] — answers daemon-local cancellable runners
|
||||
/// (currently `tool.wait_ms`). If a token is registered we trip it
|
||||
/// and stop.
|
||||
/// (`tool.wait_ms` and `session.*` lifecycle calls). If a token is
|
||||
/// registered we trip it and stop. Lifecycle handlers forward their
|
||||
/// own WS cancel after preserving request-before-cancel ordering.
|
||||
/// 2. [`super::inflight::ToolInflightRegistry`] — every IPC-tracked
|
||||
/// `tool.*` RPC, registered the moment the IPC handler accepts the
|
||||
/// request. Trips the entry's cancel token regardless of whether
|
||||
@@ -634,7 +635,20 @@ fn clamp_browser_wait(wait_ms: Option<u64>) -> Option<Duration> {
|
||||
))
|
||||
}
|
||||
|
||||
async fn handle_session_start(state: &Arc<DaemonState>, params: Value) -> Result<Value, RpcError> {
|
||||
async fn handle_session_start(
|
||||
state: &Arc<DaemonState>,
|
||||
rpc_id: RpcId,
|
||||
params: Value,
|
||||
) -> Result<Value, RpcError> {
|
||||
let abort_guard = state
|
||||
.abort_registry
|
||||
.register(rpc_id)
|
||||
.map_err(|err| RpcError {
|
||||
code: ErrorCode::ProtocolError,
|
||||
message: format!("session.start cancellation registration failed: {err:?}"),
|
||||
data: None,
|
||||
})?;
|
||||
let cancel = abort_guard.token().clone();
|
||||
let params: CliSessionStartParams = if params.is_null() {
|
||||
CliSessionStartParams {
|
||||
browser_instance_id: None,
|
||||
@@ -673,6 +687,7 @@ async fn handle_session_start(state: &Arc<DaemonState>, params: Value) -> Result
|
||||
},
|
||||
state.config.extension_connect_wait,
|
||||
DEFAULT_RPC_TIMEOUT,
|
||||
Some(cancel),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -696,6 +711,8 @@ fn map_start_error(err: StartSessionError) -> RpcError {
|
||||
StartSessionError::AmbiguousBrowserLabel { .. } => ErrorCode::InvalidParams,
|
||||
StartSessionError::IdExhausted => ErrorCode::ProtocolError,
|
||||
StartSessionError::Timeout => ErrorCode::Timeout,
|
||||
StartSessionError::Cancelled => ErrorCode::Cancelled,
|
||||
StartSessionError::CleanupFailed { .. } => ErrorCode::ProtocolError,
|
||||
StartSessionError::TransportClosed => ErrorCode::ProtocolError,
|
||||
StartSessionError::ExtensionError(inner) => inner.code,
|
||||
};
|
||||
@@ -711,6 +728,16 @@ fn map_start_error(err: StartSessionError) -> RpcError {
|
||||
"label": label,
|
||||
"instance_ids": instance_ids,
|
||||
})),
|
||||
StartSessionError::CleanupFailed {
|
||||
session_id,
|
||||
agent_window_id,
|
||||
..
|
||||
} => Some(serde_json::json!({
|
||||
"reason": "cleanup_failed",
|
||||
"session_id": session_id.0,
|
||||
"agent_window_id": agent_window_id,
|
||||
})),
|
||||
StartSessionError::ExtensionError(inner) => inner.data.clone(),
|
||||
_ => None,
|
||||
};
|
||||
RpcError {
|
||||
@@ -720,7 +747,20 @@ fn map_start_error(err: StartSessionError) -> RpcError {
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_session_stop(state: &Arc<DaemonState>, params: Value) -> Result<Value, RpcError> {
|
||||
async fn handle_session_stop(
|
||||
state: &Arc<DaemonState>,
|
||||
rpc_id: RpcId,
|
||||
params: Value,
|
||||
) -> Result<Value, RpcError> {
|
||||
let abort_guard = state
|
||||
.abort_registry
|
||||
.register(rpc_id)
|
||||
.map_err(|err| RpcError {
|
||||
code: ErrorCode::ProtocolError,
|
||||
message: format!("session.stop cancellation registration failed: {err:?}"),
|
||||
data: None,
|
||||
})?;
|
||||
let cancel = abort_guard.token().clone();
|
||||
let params: CliSessionStopParams = if params.is_null() {
|
||||
CliSessionStopParams {
|
||||
session_id: None,
|
||||
@@ -734,7 +774,7 @@ async fn handle_session_stop(state: &Arc<DaemonState>, params: Value) -> Result<
|
||||
})?
|
||||
};
|
||||
if params.all {
|
||||
return handle_session_stop_all(state).await;
|
||||
return handle_session_stop_all(state, Some(cancel)).await;
|
||||
}
|
||||
let session_id = match params.session_id {
|
||||
Some(s) => SessionId(s),
|
||||
@@ -753,6 +793,7 @@ async fn handle_session_stop(state: &Arc<DaemonState>, params: Value) -> Result<
|
||||
&state.session_interrupts,
|
||||
&session_id,
|
||||
DEFAULT_SESSION_STOP_TIMEOUT,
|
||||
Some(cancel),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -804,7 +845,25 @@ fn map_stop_error(err: StopSessionError) -> RpcError {
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_session_stop_all(state: &Arc<DaemonState>) -> Result<Value, RpcError> {
|
||||
async fn handle_session_stop_all_rpc(
|
||||
state: &Arc<DaemonState>,
|
||||
rpc_id: RpcId,
|
||||
) -> Result<Value, RpcError> {
|
||||
let abort_guard = state
|
||||
.abort_registry
|
||||
.register(rpc_id)
|
||||
.map_err(|err| RpcError {
|
||||
code: ErrorCode::ProtocolError,
|
||||
message: format!("session.stop_all cancellation registration failed: {err:?}"),
|
||||
data: None,
|
||||
})?;
|
||||
handle_session_stop_all(state, Some(abort_guard.token().clone())).await
|
||||
}
|
||||
|
||||
async fn handle_session_stop_all(
|
||||
state: &Arc<DaemonState>,
|
||||
cancel: Option<super::abort::AbortToken>,
|
||||
) -> Result<Value, RpcError> {
|
||||
let ids: Vec<SessionId> = state
|
||||
.sessions
|
||||
.snapshot()
|
||||
@@ -816,6 +875,16 @@ async fn handle_session_stop_all(state: &Arc<DaemonState>) -> Result<Value, RpcE
|
||||
let mut returned_tab_ids = Vec::new();
|
||||
let mut return_failures = Vec::new();
|
||||
for id in ids {
|
||||
if cancel
|
||||
.as_ref()
|
||||
.is_some_and(super::abort::AbortToken::is_cancelled)
|
||||
{
|
||||
return Err(RpcError {
|
||||
code: ErrorCode::Cancelled,
|
||||
message: "session.stop_all was cancelled".into(),
|
||||
data: None,
|
||||
});
|
||||
}
|
||||
match stop_session(
|
||||
&state.browsers,
|
||||
&state.sessions,
|
||||
@@ -823,6 +892,7 @@ async fn handle_session_stop_all(state: &Arc<DaemonState>) -> Result<Value, RpcE
|
||||
&state.session_interrupts,
|
||||
&id,
|
||||
DEFAULT_SESSION_STOP_TIMEOUT,
|
||||
cancel.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -848,6 +918,15 @@ async fn handle_session_stop_all(state: &Arc<DaemonState>) -> Result<Value, RpcE
|
||||
returned_tab_ids.extend(stop.returned_tab_ids);
|
||||
return_failures.extend(stop.return_failures);
|
||||
}
|
||||
Err(err)
|
||||
if matches!(
|
||||
&err,
|
||||
StopSessionError::ExtensionError(inner)
|
||||
if inner.code == ErrorCode::Cancelled
|
||||
) =>
|
||||
{
|
||||
return Err(map_stop_error(err));
|
||||
}
|
||||
Err(err) => {
|
||||
debug!(session = %id, ?err, "session.stop_all: failure (continuing)");
|
||||
let rpc = map_stop_error(err);
|
||||
|
||||
@@ -43,6 +43,7 @@ use serde_json::Value;
|
||||
use tokio::sync::{mpsc, oneshot};
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use super::abort::AbortToken;
|
||||
use super::browsers::BrowserRegistry;
|
||||
use super::inflight::{PromoteOutcome, ToolInflightEntry};
|
||||
use super::sessions::{SessionId, SessionRegistry};
|
||||
@@ -83,6 +84,14 @@ pub struct ToolJob {
|
||||
/// IPC request id (e.g. `session.stop`'s queued teardown call —
|
||||
/// those already carry their own bespoke retry / abort path).
|
||||
pub inflight: Option<Arc<ToolInflightEntry>>,
|
||||
/// Cancellation token for session-lifecycle work registered in the
|
||||
/// daemon-local abort registry. Unlike `inflight`, lifecycle callers
|
||||
/// forward their own WS cancel after the request has been queued.
|
||||
pub lifecycle_cancel: Option<AbortToken>,
|
||||
/// How long cancellation keeps the worker busy while the extension
|
||||
/// finishes compensation. Session teardown uses its full RPC budget
|
||||
/// so daemon state is reconciled even after the CLI exits.
|
||||
pub cancel_cleanup_timeout: Duration,
|
||||
/// Owning session — used by `ToolInflightRegistry::cancel_session`
|
||||
/// to drain queued jobs that have not yet been promoted to
|
||||
/// "forwarded". Worker callers also have access to it via the
|
||||
@@ -322,6 +331,8 @@ impl ToolQueueRegistry {
|
||||
timeout,
|
||||
false,
|
||||
inflight,
|
||||
None,
|
||||
CANCEL_CLEANUP_TIMEOUT,
|
||||
)
|
||||
.await;
|
||||
// A long-running request may outlive the idle threshold. Touching
|
||||
@@ -360,6 +371,8 @@ impl ToolQueueRegistry {
|
||||
timeout,
|
||||
respond: respond_tx,
|
||||
inflight,
|
||||
lifecycle_cancel: None,
|
||||
cancel_cleanup_timeout: CANCEL_CLEANUP_TIMEOUT,
|
||||
session_id: sid.clone(),
|
||||
};
|
||||
match forward_one(sid, &self.browsers, &self.sessions, &job).await {
|
||||
@@ -378,6 +391,7 @@ impl ToolQueueRegistry {
|
||||
method: Method,
|
||||
params: Value,
|
||||
timeout: Duration,
|
||||
cancel: Option<AbortToken>,
|
||||
) -> Result<Value, DispatchError> {
|
||||
let (sender, state) = {
|
||||
let mut guard = self.queues.lock().expect("tool queue registry poisoned");
|
||||
@@ -404,6 +418,8 @@ impl ToolQueueRegistry {
|
||||
timeout,
|
||||
true,
|
||||
None,
|
||||
cancel,
|
||||
timeout,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -419,6 +435,8 @@ async fn dispatch_with_sender(
|
||||
timeout: Duration,
|
||||
wait_for_capacity: bool,
|
||||
inflight: Option<Arc<ToolInflightEntry>>,
|
||||
lifecycle_cancel: Option<AbortToken>,
|
||||
cancel_cleanup_timeout: Duration,
|
||||
) -> Result<Value, DispatchError> {
|
||||
let (respond_tx, respond_rx) = oneshot::channel();
|
||||
let job = ToolJob {
|
||||
@@ -427,8 +445,11 @@ async fn dispatch_with_sender(
|
||||
timeout,
|
||||
respond: respond_tx,
|
||||
inflight,
|
||||
lifecycle_cancel,
|
||||
cancel_cleanup_timeout,
|
||||
session_id,
|
||||
};
|
||||
let lifecycle_cancellable = job.lifecycle_cancel.is_some();
|
||||
if wait_for_capacity {
|
||||
let waited = tokio::time::timeout(
|
||||
timeout.saturating_add(Duration::from_secs(1)),
|
||||
@@ -453,8 +474,15 @@ async fn dispatch_with_sender(
|
||||
mpsc::error::TrySendError::Closed(_) => DispatchError::QueueClosed,
|
||||
});
|
||||
}
|
||||
let waited =
|
||||
tokio::time::timeout(timeout.saturating_add(Duration::from_secs(1)), respond_rx).await;
|
||||
let mut response_timeout = timeout.saturating_add(Duration::from_secs(1));
|
||||
if lifecycle_cancellable {
|
||||
// A lifecycle cancel can arrive at the original RPC deadline and
|
||||
// intentionally keeps the worker alive while extension cleanup
|
||||
// settles. Do not let this outer waiter abandon that reconciliation
|
||||
// early, which could reopen a daemon session whose window is gone.
|
||||
response_timeout = response_timeout.saturating_add(cancel_cleanup_timeout);
|
||||
}
|
||||
let waited = tokio::time::timeout(response_timeout, respond_rx).await;
|
||||
match waited {
|
||||
Ok(Ok(Ok(v))) => Ok(v),
|
||||
Ok(Ok(Err(rpc))) => Err(DispatchError::Rpc(rpc)),
|
||||
@@ -492,11 +520,17 @@ async fn forward_one(
|
||||
// / browser resolution work, and before any WS frame leaves the
|
||||
// daemon (review C2). We keep the `tokio::select!` later so
|
||||
// cancels arriving mid-WS-hop also unblock the worker.
|
||||
if let Some(entry) = job.inflight.as_ref()
|
||||
&& entry.is_cancelled()
|
||||
if job
|
||||
.inflight
|
||||
.as_ref()
|
||||
.is_some_and(|entry| entry.is_cancelled())
|
||||
|| job
|
||||
.lifecycle_cancel
|
||||
.as_ref()
|
||||
.is_some_and(AbortToken::is_cancelled)
|
||||
{
|
||||
return Err(cancelled_error(
|
||||
Some(entry),
|
||||
job.inflight.as_deref(),
|
||||
"tool dispatch cancelled before forwarding",
|
||||
));
|
||||
}
|
||||
@@ -560,11 +594,20 @@ async fn forward_one(
|
||||
}
|
||||
}
|
||||
None => {
|
||||
// Daemon-internal callers (e.g. queued session.stop drain)
|
||||
// don't carry an inflight entry; they cannot be cancelled
|
||||
// through the IPC `cancel` surface, so a plain sink send
|
||||
// is sufficient and keeps the existing transport-error
|
||||
// behaviour.
|
||||
// Daemon-internal callers do not carry a tool-inflight entry.
|
||||
// Session lifecycle jobs instead use the abort-registry token
|
||||
// below; other internal callers keep the plain send path.
|
||||
if job
|
||||
.lifecycle_cancel
|
||||
.as_ref()
|
||||
.is_some_and(AbortToken::is_cancelled)
|
||||
{
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
return Err(cancelled_error(
|
||||
None,
|
||||
"session lifecycle cancelled before forwarding",
|
||||
));
|
||||
}
|
||||
if client.sink.send(request).is_err() {
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
return Err(RpcError {
|
||||
@@ -573,19 +616,45 @@ async fn forward_one(
|
||||
data: None,
|
||||
});
|
||||
}
|
||||
None
|
||||
job.lifecycle_cancel.clone()
|
||||
}
|
||||
};
|
||||
let forward_lifecycle_cancel = || {
|
||||
if job.lifecycle_cancel.is_none() {
|
||||
return;
|
||||
}
|
||||
let cancel = Frame::Request(RequestFrame {
|
||||
id: format!("cancel-{rpc_id}"),
|
||||
method: Method::Cancel,
|
||||
params: Some(serde_json::json!({ "rpc_id": rpc_id })),
|
||||
});
|
||||
if let Err(err) = client.sink.send(cancel) {
|
||||
warn!(%rpc_id, ?err, "failed to forward session lifecycle cancel");
|
||||
}
|
||||
};
|
||||
let on_abort: Option<&(dyn Fn() + Sync)> = job
|
||||
.lifecycle_cancel
|
||||
.as_ref()
|
||||
.map(|_| &forward_lifecycle_cancel as &(dyn Fn() + Sync));
|
||||
let waited = await_with_optional_cancel(
|
||||
job.timeout,
|
||||
CANCEL_CLEANUP_TIMEOUT,
|
||||
job.cancel_cleanup_timeout,
|
||||
waiter,
|
||||
cancel_token.as_ref(),
|
||||
on_abort,
|
||||
)
|
||||
.await;
|
||||
let response = match waited {
|
||||
WaitOutcome::Response(resp) => resp,
|
||||
WaitOutcome::CancelledAfterResponse(resp) => {
|
||||
if job.method == Method::ToolSessionStop
|
||||
&& let ResponseBody::Ok(value) = resp.body
|
||||
{
|
||||
// Teardown crossed its final irreversible boundary before
|
||||
// the cancel landed. Commit the real extension result so the
|
||||
// daemon does not retain a session whose window is gone.
|
||||
return Ok(value);
|
||||
}
|
||||
// Cancellation wins the external verdict, but only after the
|
||||
// extension's original RPC has settled. Preserve any non-cancel
|
||||
// error so compensation failures remain explicit instead of
|
||||
@@ -600,13 +669,33 @@ async fn forward_one(
|
||||
"tool dispatch cancelled after extension cleanup",
|
||||
));
|
||||
}
|
||||
WaitOutcome::TimedOutAfterResponse(resp) => match resp.body {
|
||||
ResponseBody::Ok(value) if job.method == Method::ToolSessionStop => {
|
||||
// The close crossed its irreversible boundary during the
|
||||
// timeout cleanup grace. Reconcile daemon state instead of
|
||||
// reopening a session whose window is already gone.
|
||||
return Ok(value);
|
||||
}
|
||||
ResponseBody::Err(err)
|
||||
if !matches!(err.code, ErrorCode::Cancelled | ErrorCode::UserAborted) =>
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
_ => {
|
||||
return Err(RpcError {
|
||||
code: ErrorCode::Timeout,
|
||||
message: format!("tool RPC timed out after {:?}", job.timeout),
|
||||
data: None,
|
||||
});
|
||||
}
|
||||
},
|
||||
WaitOutcome::CleanupTimeout => {
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
return Err(RpcError {
|
||||
code: ErrorCode::Timeout,
|
||||
message: format!(
|
||||
"cancelled tool did not finish cleanup within {:?}",
|
||||
CANCEL_CLEANUP_TIMEOUT
|
||||
job.cancel_cleanup_timeout
|
||||
),
|
||||
data: Some(serde_json::json!({ "reason": "cancel_cleanup_timeout" })),
|
||||
});
|
||||
@@ -638,6 +727,7 @@ async fn forward_one(
|
||||
enum WaitOutcome {
|
||||
Response(bsk_protocol::ResponseFrame),
|
||||
CancelledAfterResponse(bsk_protocol::ResponseFrame),
|
||||
TimedOutAfterResponse(bsk_protocol::ResponseFrame),
|
||||
CleanupTimeout,
|
||||
WaiterClosed,
|
||||
Timeout,
|
||||
@@ -648,6 +738,7 @@ async fn await_with_optional_cancel(
|
||||
cleanup_timeout: Duration,
|
||||
mut waiter: oneshot::Receiver<bsk_protocol::ResponseFrame>,
|
||||
cancel: Option<&super::abort::AbortToken>,
|
||||
on_abort: Option<&(dyn Fn() + Sync)>,
|
||||
) -> WaitOutcome {
|
||||
match cancel {
|
||||
Some(token) => {
|
||||
@@ -659,6 +750,9 @@ async fn await_with_optional_cancel(
|
||||
// after compensation or the cleanup deadline expires.
|
||||
biased;
|
||||
_ = token.cancelled() => {
|
||||
if let Some(on_abort) = on_abort {
|
||||
on_abort();
|
||||
}
|
||||
match tokio::time::timeout(cleanup_timeout, &mut waiter).await {
|
||||
Ok(Ok(resp)) => WaitOutcome::CancelledAfterResponse(resp),
|
||||
Ok(Err(_)) => WaitOutcome::WaiterClosed,
|
||||
@@ -669,7 +763,17 @@ async fn await_with_optional_cancel(
|
||||
Ok(resp) => WaitOutcome::Response(resp),
|
||||
Err(_) => WaitOutcome::WaiterClosed,
|
||||
},
|
||||
_ = &mut deadline => WaitOutcome::Timeout,
|
||||
_ = &mut deadline => {
|
||||
let Some(on_abort) = on_abort else {
|
||||
return WaitOutcome::Timeout;
|
||||
};
|
||||
on_abort();
|
||||
match tokio::time::timeout(cleanup_timeout, &mut waiter).await {
|
||||
Ok(Ok(resp)) => WaitOutcome::TimedOutAfterResponse(resp),
|
||||
Ok(Err(_)) => WaitOutcome::WaiterClosed,
|
||||
Err(_) => WaitOutcome::CleanupTimeout,
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
None => match tokio::time::timeout(timeout, &mut waiter).await {
|
||||
@@ -759,6 +863,7 @@ mod await_with_optional_cancel_tests {
|
||||
Duration::from_secs(1),
|
||||
rx,
|
||||
Some(&token),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
@@ -781,6 +886,7 @@ mod await_with_optional_cancel_tests {
|
||||
Duration::from_secs(1),
|
||||
rx,
|
||||
Some(&token),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
match outcome {
|
||||
@@ -795,9 +901,14 @@ mod await_with_optional_cancel_tests {
|
||||
async fn no_cancel_token_still_returns_response() {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
tx.send(dummy_response()).unwrap();
|
||||
let outcome =
|
||||
await_with_optional_cancel(Duration::from_secs(10), Duration::from_secs(1), rx, None)
|
||||
.await;
|
||||
let outcome = await_with_optional_cancel(
|
||||
Duration::from_secs(10),
|
||||
Duration::from_secs(1),
|
||||
rx,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(outcome, WaitOutcome::Response(_)));
|
||||
}
|
||||
|
||||
@@ -812,6 +923,7 @@ mod await_with_optional_cancel_tests {
|
||||
Duration::from_secs(1),
|
||||
rx,
|
||||
Some(&token),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
});
|
||||
@@ -835,10 +947,37 @@ mod await_with_optional_cancel_tests {
|
||||
Duration::from_millis(10),
|
||||
rx,
|
||||
Some(&token),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(outcome, WaitOutcome::CleanupTimeout));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lifecycle_timeout_forwards_abort_and_waits_for_cleanup_response() {
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
let token = AbortToken::new();
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let abort_forwarded = AtomicBool::new(false);
|
||||
let forward_abort = || abort_forwarded.store(true, Ordering::SeqCst);
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
tx.send(dummy_response()).unwrap();
|
||||
});
|
||||
|
||||
let outcome = await_with_optional_cancel(
|
||||
Duration::from_millis(5),
|
||||
Duration::from_millis(100),
|
||||
rx,
|
||||
Some(&token),
|
||||
Some(&forward_abort),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(abort_forwarded.load(Ordering::SeqCst));
|
||||
assert!(matches!(outcome, WaitOutcome::TimedOutAfterResponse(_)));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -901,6 +1040,8 @@ mod tool_job_session_id_tests {
|
||||
timeout: Duration::from_secs(1),
|
||||
respond: tx,
|
||||
inflight: None,
|
||||
lifecycle_cancel: None,
|
||||
cancel_cleanup_timeout: CANCEL_CLEANUP_TIMEOUT,
|
||||
session_id: SessionId("sess-A".into()),
|
||||
};
|
||||
assert_eq!(job.session_id.0, "sess-A");
|
||||
|
||||
@@ -12,13 +12,14 @@ use bsk_protocol::system::{BrowserStatusEntry, SessionStatusEntry};
|
||||
use bsk_protocol::tools::{
|
||||
SessionStartParams, SessionStartResult, SessionStopParams, SessionStopResult,
|
||||
};
|
||||
use bsk_protocol::{Frame, RequestFrame, ResponseBody, RpcError, RpcId};
|
||||
use bsk_protocol::{ErrorCode, Frame, Method, RequestFrame, ResponseBody, RpcError, RpcId};
|
||||
use rand::Rng;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::time::{Duration, timeout};
|
||||
|
||||
use super::abort::AbortToken;
|
||||
use super::browsers::{BrowserClient, BrowserId, BrowserRegistry, SelectError};
|
||||
use super::queue::{DispatchError, ToolQueueRegistry};
|
||||
use super::queue::{CANCEL_CLEANUP_TIMEOUT, DispatchError, ToolQueueRegistry};
|
||||
use super::session_interrupt::SessionInterruptRegistry;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
@@ -339,6 +340,14 @@ pub enum StartSessionError {
|
||||
IdExhausted,
|
||||
#[error("session creation timed out waiting for extension")]
|
||||
Timeout,
|
||||
#[error("session creation was cancelled")]
|
||||
Cancelled,
|
||||
#[error("session creation cancellation cleanup failed for {session_id}: {message}")]
|
||||
CleanupFailed {
|
||||
session_id: SessionId,
|
||||
agent_window_id: Option<i64>,
|
||||
message: String,
|
||||
},
|
||||
#[error("extension rejected tool.session_start: {0:?}")]
|
||||
ExtensionError(RpcError),
|
||||
#[error("transport closed while waiting for extension response")]
|
||||
@@ -354,6 +363,8 @@ impl StartSessionError {
|
||||
StartSessionError::AmbiguousBrowserLabel { .. } => "invalid_params",
|
||||
StartSessionError::IdExhausted => "protocol_error",
|
||||
StartSessionError::Timeout => "timeout",
|
||||
StartSessionError::Cancelled => "cancelled",
|
||||
StartSessionError::CleanupFailed { .. } => "protocol_error",
|
||||
StartSessionError::TransportClosed => "protocol_error",
|
||||
StartSessionError::ExtensionError(err) => match err.code {
|
||||
bsk_protocol::ErrorCode::Timeout => "timeout",
|
||||
@@ -407,6 +418,7 @@ pub struct AgentWindowOptions {
|
||||
/// On success the matching per-session dispatch queue (M6.5,
|
||||
/// design §5) is also spawned so that subsequent `tool.*` RPCs serialise
|
||||
/// against this session's Agent Window / ref-store.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn start_session(
|
||||
registry: &Arc<BrowserRegistry>,
|
||||
sessions: &Arc<SessionRegistry>,
|
||||
@@ -415,24 +427,32 @@ pub async fn start_session(
|
||||
window: AgentWindowOptions,
|
||||
connect_wait: Duration,
|
||||
timeout_dur: Duration,
|
||||
cancel: Option<AbortToken>,
|
||||
) -> Result<Session, StartSessionError> {
|
||||
let client: Arc<BrowserClient> = registry
|
||||
.select_with_connect_wait(requested, connect_wait)
|
||||
.await
|
||||
.map_err(|e| match e {
|
||||
SelectError::NoBrowserConnected => StartSessionError::NoBrowserConnected,
|
||||
SelectError::MultipleBrowsersOnline => StartSessionError::MultipleBrowsersOnline {
|
||||
browsers: snapshot_status_entries(registry, sessions),
|
||||
},
|
||||
SelectError::NotFound => StartSessionError::BrowserNotFound,
|
||||
SelectError::AmbiguousLabel {
|
||||
label,
|
||||
instance_ids,
|
||||
} => StartSessionError::AmbiguousBrowserLabel {
|
||||
label,
|
||||
instance_ids,
|
||||
},
|
||||
})?;
|
||||
let selection = registry.select_with_connect_wait(requested, connect_wait);
|
||||
tokio::pin!(selection);
|
||||
let selected = match cancel.as_ref() {
|
||||
Some(token) => tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => return Err(StartSessionError::Cancelled),
|
||||
selected = &mut selection => selected,
|
||||
},
|
||||
None => selection.await,
|
||||
};
|
||||
let client: Arc<BrowserClient> = selected.map_err(|e| match e {
|
||||
SelectError::NoBrowserConnected => StartSessionError::NoBrowserConnected,
|
||||
SelectError::MultipleBrowsersOnline => StartSessionError::MultipleBrowsersOnline {
|
||||
browsers: snapshot_status_entries(registry, sessions),
|
||||
},
|
||||
SelectError::NotFound => StartSessionError::BrowserNotFound,
|
||||
SelectError::AmbiguousLabel {
|
||||
label,
|
||||
instance_ids,
|
||||
} => StartSessionError::AmbiguousBrowserLabel {
|
||||
label,
|
||||
instance_ids,
|
||||
},
|
||||
})?;
|
||||
let session_id = sessions
|
||||
.reserve_id(client.id.clone(), SESSION_ID_MAX_RESERVE_ATTEMPTS, now_ms)
|
||||
.ok_or(StartSessionError::IdExhausted)?;
|
||||
@@ -453,22 +473,71 @@ pub async fn start_session(
|
||||
let mut pending = client.pending.lock().unwrap();
|
||||
pending.register(rpc_id.clone())
|
||||
};
|
||||
if cancel.as_ref().is_some_and(AbortToken::is_cancelled) {
|
||||
sessions.cancel_reservation(&session_id);
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
return Err(StartSessionError::Cancelled);
|
||||
}
|
||||
if client.sink.send(Frame::Request(request)).is_err() {
|
||||
sessions.cancel_reservation(&session_id);
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
return Err(StartSessionError::TransportClosed);
|
||||
}
|
||||
let response = match timeout(timeout_dur, waiter).await {
|
||||
Ok(Ok(resp)) => resp,
|
||||
Ok(Err(_)) => {
|
||||
let mut waiter = waiter;
|
||||
enum StartWaitOutcome {
|
||||
Response(bsk_protocol::ResponseFrame),
|
||||
WaiterClosed,
|
||||
Cancelled,
|
||||
Timeout,
|
||||
}
|
||||
let wait_outcome = match cancel.as_ref() {
|
||||
Some(token) => {
|
||||
let deadline = tokio::time::sleep(timeout_dur);
|
||||
tokio::pin!(deadline);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => StartWaitOutcome::Cancelled,
|
||||
response = &mut waiter => match response {
|
||||
Ok(response) => StartWaitOutcome::Response(response),
|
||||
Err(_) => StartWaitOutcome::WaiterClosed,
|
||||
},
|
||||
_ = &mut deadline => StartWaitOutcome::Timeout,
|
||||
}
|
||||
}
|
||||
None => match timeout(timeout_dur, &mut waiter).await {
|
||||
Ok(Ok(response)) => StartWaitOutcome::Response(response),
|
||||
Ok(Err(_)) => StartWaitOutcome::WaiterClosed,
|
||||
Err(_) => StartWaitOutcome::Timeout,
|
||||
},
|
||||
};
|
||||
let response = match wait_outcome {
|
||||
StartWaitOutcome::Response(response) => response,
|
||||
StartWaitOutcome::WaiterClosed => {
|
||||
sessions.cancel_reservation(&session_id);
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
return Err(StartSessionError::TransportClosed);
|
||||
}
|
||||
Err(_) => {
|
||||
sessions.cancel_reservation(&session_id);
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
return Err(StartSessionError::Timeout);
|
||||
StartWaitOutcome::Cancelled => {
|
||||
return Err(finish_aborted_start(
|
||||
&client,
|
||||
sessions,
|
||||
&session_id,
|
||||
&rpc_id,
|
||||
waiter,
|
||||
StartAbortReason::Cancelled,
|
||||
)
|
||||
.await);
|
||||
}
|
||||
StartWaitOutcome::Timeout => {
|
||||
return Err(finish_aborted_start(
|
||||
&client,
|
||||
sessions,
|
||||
&session_id,
|
||||
&rpc_id,
|
||||
waiter,
|
||||
StartAbortReason::Timeout,
|
||||
)
|
||||
.await);
|
||||
}
|
||||
};
|
||||
let agent_window_id = match response.body {
|
||||
@@ -504,6 +573,124 @@ pub async fn start_session(
|
||||
Ok(session)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum StartAbortReason {
|
||||
Cancelled,
|
||||
Timeout,
|
||||
}
|
||||
|
||||
impl StartAbortReason {
|
||||
fn error(self) -> StartSessionError {
|
||||
match self {
|
||||
Self::Cancelled => StartSessionError::Cancelled,
|
||||
Self::Timeout => StartSessionError::Timeout,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn finish_aborted_start(
|
||||
client: &Arc<BrowserClient>,
|
||||
sessions: &Arc<SessionRegistry>,
|
||||
session_id: &SessionId,
|
||||
rpc_id: &RpcId,
|
||||
mut waiter: tokio::sync::oneshot::Receiver<bsk_protocol::ResponseFrame>,
|
||||
reason: StartAbortReason,
|
||||
) -> StartSessionError {
|
||||
sessions.cancel_reservation(session_id);
|
||||
let cancel = Frame::Request(RequestFrame {
|
||||
id: format!("cancel-{rpc_id}"),
|
||||
method: Method::Cancel,
|
||||
params: Some(serde_json::json!({ "rpc_id": rpc_id })),
|
||||
});
|
||||
if client.sink.send(cancel).is_err() {
|
||||
client.pending.lock().unwrap().cancel(rpc_id);
|
||||
return StartSessionError::TransportClosed;
|
||||
}
|
||||
|
||||
match timeout(CANCEL_CLEANUP_TIMEOUT, &mut waiter).await {
|
||||
Ok(Ok(response)) => match response.body {
|
||||
ResponseBody::Err(err)
|
||||
if matches!(err.code, ErrorCode::Cancelled | ErrorCode::UserAborted) =>
|
||||
{
|
||||
reason.error()
|
||||
}
|
||||
ResponseBody::Err(err) => StartSessionError::ExtensionError(err),
|
||||
ResponseBody::Ok(value) => {
|
||||
let agent_window_id = serde_json::from_value::<SessionStartResult>(value)
|
||||
.ok()
|
||||
.and_then(|result| result.agent_window_id);
|
||||
match rollback_extension_session(client, session_id).await {
|
||||
Ok(()) => reason.error(),
|
||||
Err(message) => StartSessionError::CleanupFailed {
|
||||
session_id: session_id.clone(),
|
||||
agent_window_id,
|
||||
message,
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
Ok(Err(_)) => {
|
||||
client.pending.lock().unwrap().cancel(rpc_id);
|
||||
StartSessionError::TransportClosed
|
||||
}
|
||||
Err(_) => {
|
||||
// The cancel frame is already queued behind session_start. Even
|
||||
// if chrome.windows.create is still blocked, the extension will
|
||||
// observe the aborted signal and compensate before replying.
|
||||
client.pending.lock().unwrap().cancel(rpc_id);
|
||||
reason.error()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn rollback_extension_session(
|
||||
client: &Arc<BrowserClient>,
|
||||
session_id: &SessionId,
|
||||
) -> Result<(), String> {
|
||||
let rollback_id = next_rpc_id("sess-start-rollback");
|
||||
let waiter = {
|
||||
let mut pending = client.pending.lock().unwrap();
|
||||
pending.register(rollback_id.clone())
|
||||
};
|
||||
let request = RequestFrame {
|
||||
id: rollback_id.clone(),
|
||||
method: Method::ToolSessionStop,
|
||||
params: Some(
|
||||
serde_json::to_value(SessionStopParams {
|
||||
session_id: session_id.0.clone(),
|
||||
})
|
||||
.expect("serialize session_start rollback params"),
|
||||
),
|
||||
};
|
||||
if client.sink.send(Frame::Request(request)).is_err() {
|
||||
client.pending.lock().unwrap().cancel(&rollback_id);
|
||||
return Err("browser disconnected before rollback was queued".into());
|
||||
}
|
||||
let response = match timeout(CANCEL_CLEANUP_TIMEOUT, waiter).await {
|
||||
Ok(Ok(response)) => response,
|
||||
Ok(Err(_)) => return Err("browser disconnected during rollback".into()),
|
||||
Err(_) => {
|
||||
client.pending.lock().unwrap().cancel(&rollback_id);
|
||||
return Err(format!(
|
||||
"extension did not finish rollback within {CANCEL_CLEANUP_TIMEOUT:?}"
|
||||
));
|
||||
}
|
||||
};
|
||||
match response.body {
|
||||
ResponseBody::Ok(value) => {
|
||||
let result: SessionStopResult = serde_json::from_value(value)
|
||||
.map_err(|err| format!("invalid rollback response: {err}"))?;
|
||||
if result.return_failures.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err("rollback reported borrowed-tab return failures".into())
|
||||
}
|
||||
}
|
||||
ResponseBody::Err(err) if err.code == ErrorCode::NotFound => Ok(()),
|
||||
ResponseBody::Err(err) => Err(format!("rollback failed: {} ({:?})", err.message, err.code)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Tear down a single session id (round-trips `tool.session_stop` to the
|
||||
/// owning extension and unregisters on success). Also tears down the
|
||||
/// per-session dispatch queue so the worker task exits.
|
||||
@@ -514,6 +701,7 @@ pub async fn stop_session(
|
||||
interrupts: &Arc<SessionInterruptRegistry>,
|
||||
session_id: &SessionId,
|
||||
timeout_dur: Duration,
|
||||
cancel: Option<AbortToken>,
|
||||
) -> Result<SessionStopResult, StopSessionError> {
|
||||
let session = sessions.get(session_id).ok_or(StopSessionError::NotFound)?;
|
||||
if registry.get(&session.browser_id).is_none() {
|
||||
@@ -524,12 +712,17 @@ pub async fn stop_session(
|
||||
session_id: session_id.0.clone(),
|
||||
})
|
||||
.unwrap();
|
||||
// Internal callers (idle reaping and shutdown) do not have a CLI-owned
|
||||
// token, but still need timeout-driven WS cancellation so a late window
|
||||
// close cannot leave daemon state pointing at a vanished session.
|
||||
let lifecycle_cancel = Some(cancel.unwrap_or_default());
|
||||
match queues
|
||||
.dispatch_after_closing(
|
||||
session_id,
|
||||
bsk_protocol::Method::ToolSessionStop,
|
||||
params,
|
||||
timeout_dur,
|
||||
lifecycle_cancel,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -567,6 +760,10 @@ pub async fn stop_session(
|
||||
drop_session_local(queues, interrupts, session_id);
|
||||
return Ok(SessionStopResult::default());
|
||||
}
|
||||
if err.code == bsk_protocol::ErrorCode::Timeout {
|
||||
queues.reopen(session_id);
|
||||
return Err(StopSessionError::Timeout);
|
||||
}
|
||||
queues.reopen(session_id);
|
||||
Err(StopSessionError::ExtensionError(err))
|
||||
}
|
||||
|
||||
@@ -15,7 +15,6 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::sync::Mutex;
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use anyhow::{Context, Result};
|
||||
@@ -291,8 +290,7 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
|
||||
});
|
||||
|
||||
let started_at = Instant::now();
|
||||
let activity: Arc<Mutex<Instant>> = Arc::new(Mutex::new(started_at));
|
||||
let active_ipc_connections = Arc::new(AtomicUsize::new(0));
|
||||
let activity = Arc::new(Mutex::new(IpcActivityState::new(started_at)));
|
||||
let (ipc_shutdown_tx, ipc_shutdown_rx) = tokio::sync::oneshot::channel::<()>();
|
||||
let status = ipc::DaemonStatus {
|
||||
started_at,
|
||||
@@ -314,18 +312,14 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
|
||||
let handler = handler.clone();
|
||||
let ipc_open = {
|
||||
let activity = activity.clone();
|
||||
let active = active_ipc_connections.clone();
|
||||
move || {
|
||||
active.fetch_add(1, Ordering::SeqCst);
|
||||
record_activity(&activity);
|
||||
record_ipc_open(&activity);
|
||||
}
|
||||
};
|
||||
let ipc_close = {
|
||||
let activity = activity.clone();
|
||||
let active = active_ipc_connections.clone();
|
||||
move || {
|
||||
active.fetch_sub(1, Ordering::SeqCst);
|
||||
record_activity(&activity);
|
||||
record_ipc_close(&activity);
|
||||
}
|
||||
};
|
||||
tokio::spawn(ipc::serve(
|
||||
@@ -344,7 +338,6 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
|
||||
let idle_task = {
|
||||
let activity = activity.clone();
|
||||
let state = Arc::clone(&state);
|
||||
let active_ipc_connections = active_ipc_connections.clone();
|
||||
let daemon_idle = cfg.daemon_idle;
|
||||
tokio::spawn(async move {
|
||||
let tick = (daemon_idle / 4).max(Duration::from_millis(250));
|
||||
@@ -352,22 +345,19 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
|
||||
ticker.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Delay);
|
||||
loop {
|
||||
ticker.tick().await;
|
||||
let last = match activity.lock() {
|
||||
Ok(a) => *a,
|
||||
Err(p) => *p.into_inner(),
|
||||
};
|
||||
if last.elapsed() < daemon_idle {
|
||||
let ipc_is_idle =
|
||||
lock_activity(&activity).should_exit_for_idle(Instant::now(), daemon_idle);
|
||||
if !ipc_is_idle {
|
||||
continue;
|
||||
}
|
||||
// Bridge from M2/M3 (which only knew about connection
|
||||
// counts) to M4/M5 (browser + session registries):
|
||||
// hold the daemon alive while any IPC client is
|
||||
// connected, any browser is paired, or any session
|
||||
// is live (design §3.2).
|
||||
if active_ipc_connections.load(Ordering::SeqCst) > 0
|
||||
|| !state.browsers.is_empty()
|
||||
|| !state.sessions.is_empty()
|
||||
{
|
||||
// IPC liveness (open-connection count + idle window) is already
|
||||
// enforced above by `should_exit_for_idle`, which reads
|
||||
// `IpcActivityState` under one lock and returns true only when
|
||||
// there are zero connections AND the idle interval has elapsed.
|
||||
// Here we additionally hold the daemon alive while any browser
|
||||
// is paired or any session is live (design §3.2, M4/M5
|
||||
// registries).
|
||||
if !state.browsers.is_empty() || !state.sessions.is_empty() {
|
||||
continue;
|
||||
}
|
||||
info!(
|
||||
@@ -491,6 +481,7 @@ pub(crate) fn spawn_session_idle_reaper(state: Arc<DaemonState>) -> tokio::task:
|
||||
&state.session_interrupts,
|
||||
&session_id,
|
||||
Duration::from_secs(10),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -681,10 +672,55 @@ fn restart_start_args(cfg: &DaemonConfig) -> StartArgs {
|
||||
}
|
||||
}
|
||||
|
||||
fn record_activity(activity: &Arc<Mutex<Instant>>) {
|
||||
if let Ok(mut a) = activity.lock() {
|
||||
*a = Instant::now();
|
||||
#[derive(Debug)]
|
||||
struct IpcActivityState {
|
||||
last_activity: Instant,
|
||||
active_connections: usize,
|
||||
}
|
||||
|
||||
impl IpcActivityState {
|
||||
fn new(now: Instant) -> Self {
|
||||
Self {
|
||||
last_activity: now,
|
||||
active_connections: 0,
|
||||
}
|
||||
}
|
||||
|
||||
fn touch(&mut self, now: Instant) {
|
||||
self.last_activity = now;
|
||||
}
|
||||
|
||||
fn opened(&mut self, now: Instant) {
|
||||
self.active_connections = self.active_connections.saturating_add(1);
|
||||
self.last_activity = now;
|
||||
}
|
||||
|
||||
fn closed(&mut self, now: Instant) {
|
||||
self.active_connections = self.active_connections.saturating_sub(1);
|
||||
self.last_activity = now;
|
||||
}
|
||||
|
||||
fn should_exit_for_idle(&self, now: Instant, idle: Duration) -> bool {
|
||||
self.active_connections == 0 && now.saturating_duration_since(self.last_activity) >= idle
|
||||
}
|
||||
}
|
||||
|
||||
fn lock_activity(
|
||||
activity: &Arc<Mutex<IpcActivityState>>,
|
||||
) -> std::sync::MutexGuard<'_, IpcActivityState> {
|
||||
activity.lock().unwrap_or_else(|poison| poison.into_inner())
|
||||
}
|
||||
|
||||
fn record_activity(activity: &Arc<Mutex<IpcActivityState>>) {
|
||||
lock_activity(activity).touch(Instant::now());
|
||||
}
|
||||
|
||||
fn record_ipc_open(activity: &Arc<Mutex<IpcActivityState>>) {
|
||||
lock_activity(activity).opened(Instant::now());
|
||||
}
|
||||
|
||||
fn record_ipc_close(activity: &Arc<Mutex<IpcActivityState>>) {
|
||||
lock_activity(activity).closed(Instant::now());
|
||||
}
|
||||
|
||||
/// Initialise tracing for the daemon: write a daily-rotated file in
|
||||
@@ -1067,4 +1103,37 @@ mod tests {
|
||||
assert_eq!(args.session_idle, Some(Duration::from_secs(11)));
|
||||
assert_eq!(args.daemon_idle, Some(Duration::from_secs(22)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ipc_close_restarts_the_idle_window_atomically() {
|
||||
let started = Instant::now();
|
||||
let idle = Duration::from_secs(10);
|
||||
let mut activity = IpcActivityState::new(started);
|
||||
|
||||
activity.opened(started);
|
||||
let closed_at = started + Duration::from_secs(30);
|
||||
assert!(
|
||||
!activity.should_exit_for_idle(closed_at, idle),
|
||||
"an open IPC connection must hold the daemon alive"
|
||||
);
|
||||
|
||||
activity.closed(closed_at);
|
||||
assert_eq!(activity.active_connections, 0);
|
||||
assert_eq!(activity.last_activity, closed_at);
|
||||
assert!(!activity.should_exit_for_idle(closed_at, idle));
|
||||
assert!(!activity.should_exit_for_idle(closed_at + idle - Duration::from_millis(1), idle));
|
||||
assert!(activity.should_exit_for_idle(closed_at + idle, idle));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ipc_activity_keeps_the_daemon_alive_until_a_full_idle_interval_passes() {
|
||||
let started = Instant::now();
|
||||
let idle = Duration::from_secs(10);
|
||||
let mut activity = IpcActivityState::new(started);
|
||||
let request_at = started + Duration::from_secs(20);
|
||||
|
||||
activity.touch(request_at);
|
||||
assert!(!activity.should_exit_for_idle(request_at + Duration::from_secs(9), idle));
|
||||
assert!(activity.should_exit_for_idle(request_at + idle, idle));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,8 +34,8 @@ pub struct DaemonState {
|
||||
/// `stop_session` / browser disconnect.
|
||||
pub tool_queues: Arc<ToolQueueRegistry>,
|
||||
/// Per-rpc-id cancellation tokens for daemon-side long-runners
|
||||
/// (M9.3 — currently only `tool.wait_ms`). The CLI's `cancel
|
||||
/// { rpc_id }` consults this registry first.
|
||||
/// (`tool.wait_ms` plus `session.*` lifecycle calls). The CLI's
|
||||
/// `cancel { rpc_id }` consults this registry first.
|
||||
pub abort_registry: Arc<AbortRegistry>,
|
||||
/// Tracks `tool.*` RPCs that have been forwarded to an extension
|
||||
/// over WS but have not yet received a response. Indexed by the
|
||||
|
||||
@@ -325,6 +325,7 @@ async fn session_stop_fast_fails_while_tool_is_in_flight() {
|
||||
&state.session_interrupts,
|
||||
&sid,
|
||||
Duration::from_secs(5),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
@@ -366,6 +367,7 @@ async fn session_stop_fast_fails_while_tool_is_in_flight() {
|
||||
&state.session_interrupts,
|
||||
&sid,
|
||||
Duration::from_secs(5),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("session.stop ok after tool completes");
|
||||
|
||||
@@ -10,9 +10,14 @@ use std::time::Duration;
|
||||
|
||||
use bsk::daemon::{self, DaemonConfig};
|
||||
use bsk::ipc_client::IpcClient;
|
||||
use bsk_protocol::cancel::{CancelParams, CancelResult};
|
||||
use bsk_protocol::system::{HandshakeParams, HandshakeResult, StatusResult};
|
||||
use bsk_protocol::tools::{SessionStartParams, SessionStartResult, SessionStopParams};
|
||||
use bsk_protocol::{BrowserPeerInfo, Frame, Method, RequestFrame, ResponseBody, ResponseFrame};
|
||||
use bsk_protocol::tools::{
|
||||
SessionStartParams, SessionStartResult, SessionStopParams, SessionStopResult,
|
||||
};
|
||||
use bsk_protocol::{
|
||||
BrowserPeerInfo, ErrorCode, Frame, Method, RequestFrame, ResponseBody, ResponseFrame, RpcError,
|
||||
};
|
||||
use futures_util::{SinkExt, StreamExt};
|
||||
use rand::Rng;
|
||||
use tokio_tungstenite::tungstenite::handshake::client::generate_key;
|
||||
@@ -23,6 +28,9 @@ use support::{wait_for_browser_count, wait_for_no_sessions};
|
||||
|
||||
const TEST_EXT_ID: &str = "abcdefghijklmnopabcdefghijklmnop";
|
||||
|
||||
type TestWs =
|
||||
tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>>;
|
||||
|
||||
fn tempfile_path(prefix: &str) -> PathBuf {
|
||||
let mut p = std::env::temp_dir();
|
||||
let mut rng = rand::thread_rng();
|
||||
@@ -56,9 +64,7 @@ async fn spawn_daemon_with_session_idle(session_idle: Duration) -> (daemon::Daem
|
||||
(handle, sock)
|
||||
}
|
||||
|
||||
async fn connect_ext(
|
||||
addr: std::net::SocketAddr,
|
||||
) -> tokio_tungstenite::WebSocketStream<tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>> {
|
||||
async fn connect_ext(addr: std::net::SocketAddr) -> TestWs {
|
||||
let origin = format!("chrome-extension://{TEST_EXT_ID}");
|
||||
let url = format!("ws://{addr}/");
|
||||
let req = Request::builder()
|
||||
@@ -76,11 +82,7 @@ async fn connect_ext(
|
||||
ws
|
||||
}
|
||||
|
||||
async fn handshake_as_ext(
|
||||
ws: &mut tokio_tungstenite::WebSocketStream<
|
||||
tokio_tungstenite::MaybeTlsStream<tokio::net::TcpStream>,
|
||||
>,
|
||||
) -> HandshakeResult {
|
||||
async fn handshake_as_ext(ws: &mut TestWs) -> HandshakeResult {
|
||||
let params = HandshakeParams {
|
||||
client: "browser-skill-extension".into(),
|
||||
version: "0.1.0-dev.0".parse().unwrap(),
|
||||
@@ -114,6 +116,128 @@ async fn handshake_as_ext(
|
||||
}
|
||||
}
|
||||
|
||||
async fn next_extension_request(ws: &mut TestWs) -> RequestFrame {
|
||||
loop {
|
||||
let message = ws.next().await.expect("daemon closed websocket").unwrap();
|
||||
match message {
|
||||
Message::Text(text) => match serde_json::from_str::<Frame>(&text).unwrap() {
|
||||
Frame::Request(request) => return request,
|
||||
Frame::Response(_) | Frame::Event(_) => continue,
|
||||
},
|
||||
Message::Ping(payload) => ws.send(Message::Pong(payload)).await.unwrap(),
|
||||
Message::Close(_) => panic!("daemon closed websocket before sending request"),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_extension_response(ws: &mut TestWs, response: ResponseFrame) {
|
||||
ws.send(Message::Text(serde_json::to_string(&response).unwrap()))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
async fn acknowledge_extension_cancel(ws: &mut TestWs, cancel: RequestFrame, target_rpc_id: &str) {
|
||||
assert_eq!(cancel.method, Method::Cancel);
|
||||
let params: CancelParams = serde_json::from_value(cancel.params.unwrap()).unwrap();
|
||||
assert_eq!(params.rpc_id, target_rpc_id);
|
||||
send_extension_response(
|
||||
ws,
|
||||
ResponseFrame {
|
||||
id: cancel.id,
|
||||
body: ResponseBody::Ok(serde_json::to_value(CancelResult { cancelled: true }).unwrap()),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn respond_to_aborted_start(
|
||||
ws: &mut TestWs,
|
||||
start_seen: Option<tokio::sync::oneshot::Sender<()>>,
|
||||
agent_window_id: i64,
|
||||
) {
|
||||
let start = next_extension_request(ws).await;
|
||||
assert_eq!(start.method, Method::ToolSessionStart);
|
||||
let params: SessionStartParams = serde_json::from_value(start.params.clone().unwrap()).unwrap();
|
||||
if let Some(start_seen) = start_seen {
|
||||
start_seen.send(()).unwrap();
|
||||
}
|
||||
|
||||
let cancel = next_extension_request(ws).await;
|
||||
acknowledge_extension_cancel(ws, cancel, &start.id).await;
|
||||
|
||||
// Model the hardest race: chrome.windows.create completed just as
|
||||
// cancellation arrived, so the extension's original call succeeds.
|
||||
send_extension_response(
|
||||
ws,
|
||||
ResponseFrame {
|
||||
id: start.id,
|
||||
body: ResponseBody::Ok(
|
||||
serde_json::to_value(SessionStartResult {
|
||||
agent_window_id: Some(agent_window_id),
|
||||
})
|
||||
.unwrap(),
|
||||
),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
let rollback = next_extension_request(ws).await;
|
||||
assert_eq!(rollback.method, Method::ToolSessionStop);
|
||||
let rollback_params: SessionStopParams =
|
||||
serde_json::from_value(rollback.params.unwrap()).unwrap();
|
||||
assert_eq!(rollback_params.session_id, params.session_id);
|
||||
send_extension_response(
|
||||
ws,
|
||||
ResponseFrame {
|
||||
id: rollback.id,
|
||||
body: ResponseBody::Ok(serde_json::to_value(SessionStopResult::default()).unwrap()),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn respond_to_aborted_stop(
|
||||
ws: &mut TestWs,
|
||||
stop_seen: Option<tokio::sync::oneshot::Sender<()>>,
|
||||
) {
|
||||
let start = next_extension_request(ws).await;
|
||||
assert_eq!(start.method, Method::ToolSessionStart);
|
||||
send_extension_response(
|
||||
ws,
|
||||
ResponseFrame {
|
||||
id: start.id,
|
||||
body: ResponseBody::Ok(
|
||||
serde_json::to_value(SessionStartResult {
|
||||
agent_window_id: Some(4242),
|
||||
})
|
||||
.unwrap(),
|
||||
),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
let stop = next_extension_request(ws).await;
|
||||
assert_eq!(stop.method, Method::ToolSessionStop);
|
||||
if let Some(stop_seen) = stop_seen {
|
||||
stop_seen.send(()).unwrap();
|
||||
}
|
||||
let cancel = next_extension_request(ws).await;
|
||||
acknowledge_extension_cancel(ws, cancel, &stop.id).await;
|
||||
send_extension_response(
|
||||
ws,
|
||||
ResponseFrame {
|
||||
id: stop.id,
|
||||
body: ResponseBody::Err(RpcError {
|
||||
code: ErrorCode::Cancelled,
|
||||
message: "session_stop aborted before window close".into(),
|
||||
data: None,
|
||||
}),
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn session_start_stop_round_trip_via_ipc() {
|
||||
let (handle, sock) = spawn_daemon().await;
|
||||
@@ -250,6 +374,240 @@ async fn session_start_stop_round_trip_via_ipc() {
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelling_session_start_rolls_back_a_late_extension_success() {
|
||||
const START_RPC_ID: &str = "session-start-cancel";
|
||||
|
||||
let (handle, sock) = spawn_daemon().await;
|
||||
let mut ws = connect_ext(handle.ws_addr()).await;
|
||||
let _ = handshake_as_ext(&mut ws).await;
|
||||
let (start_seen_tx, start_seen_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
|
||||
let responder = tokio::spawn(async move {
|
||||
respond_to_aborted_start(&mut ws, Some(start_seen_tx), 4242).await;
|
||||
let _ = release_rx.await;
|
||||
});
|
||||
|
||||
let mut start_ipc = IpcClient::connect(&sock).await.unwrap();
|
||||
let start_call = tokio::spawn(async move {
|
||||
let outcome: Result<serde_json::Value, RpcError> = start_ipc
|
||||
.call_with_id(
|
||||
START_RPC_ID.to_string(),
|
||||
Method::SessionStart,
|
||||
Some(serde_json::json!({ "focused": false })),
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
outcome
|
||||
});
|
||||
|
||||
start_seen_rx.await.unwrap();
|
||||
let mut cancel_ipc = IpcClient::connect(&sock).await.unwrap();
|
||||
let cancel: CancelResult = cancel_ipc
|
||||
.call(
|
||||
"cancel-start",
|
||||
Method::Cancel,
|
||||
Some(CancelParams {
|
||||
rpc_id: START_RPC_ID.to_string(),
|
||||
}),
|
||||
Duration::from_secs(2),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(cancel.cancelled);
|
||||
|
||||
let start_error = start_call.await.unwrap().unwrap_err();
|
||||
assert_eq!(start_error.code, ErrorCode::Cancelled);
|
||||
|
||||
let mut status_ipc = IpcClient::connect(&sock).await.unwrap();
|
||||
let status: StatusResult = status_ipc
|
||||
.call::<(), _>(
|
||||
"status-after-start-cancel",
|
||||
Method::SystemStatus,
|
||||
None,
|
||||
Duration::from_secs(2),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(status.sessions.is_empty());
|
||||
assert_eq!(status.browsers[0].session_count, 0);
|
||||
|
||||
let _ = release_tx.send(());
|
||||
responder.await.unwrap();
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn timing_out_session_start_rolls_back_a_late_extension_success() {
|
||||
let (handle, _sock) = spawn_daemon().await;
|
||||
let state = handle.state();
|
||||
let mut ws = connect_ext(handle.ws_addr()).await;
|
||||
let _ = handshake_as_ext(&mut ws).await;
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
|
||||
let responder = tokio::spawn(async move {
|
||||
respond_to_aborted_start(&mut ws, None, 4343).await;
|
||||
let _ = release_rx.await;
|
||||
});
|
||||
|
||||
let start = bsk::daemon::sessions::start_session(
|
||||
&state.browsers,
|
||||
&state.sessions,
|
||||
&state.tool_queues,
|
||||
None,
|
||||
bsk::daemon::sessions::AgentWindowOptions::default(),
|
||||
Duration::ZERO,
|
||||
Duration::from_millis(20),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(
|
||||
start,
|
||||
Err(bsk::daemon::sessions::StartSessionError::Timeout)
|
||||
));
|
||||
assert!(state.sessions.is_empty());
|
||||
|
||||
let _ = release_tx.send(());
|
||||
responder.await.unwrap();
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelling_session_stop_keeps_the_session_retryable() {
|
||||
const STOP_RPC_ID: &str = "session-stop-cancel";
|
||||
|
||||
let (handle, sock) = spawn_daemon().await;
|
||||
let mut ws = connect_ext(handle.ws_addr()).await;
|
||||
let _ = handshake_as_ext(&mut ws).await;
|
||||
let (stop_seen_tx, stop_seen_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
|
||||
let responder = tokio::spawn(async move {
|
||||
respond_to_aborted_stop(&mut ws, Some(stop_seen_tx)).await;
|
||||
let _ = release_rx.await;
|
||||
});
|
||||
|
||||
let mut start_ipc = IpcClient::connect(&sock).await.unwrap();
|
||||
let start: serde_json::Value = start_ipc
|
||||
.call(
|
||||
"start-before-stop-cancel",
|
||||
Method::SessionStart,
|
||||
Some(serde_json::json!({ "focused": false })),
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let session_id = start["session_id"].as_str().unwrap().to_string();
|
||||
|
||||
let mut stop_ipc = IpcClient::connect(&sock).await.unwrap();
|
||||
let stop_session_id = session_id.clone();
|
||||
let stop_call = tokio::spawn(async move {
|
||||
let outcome: Result<serde_json::Value, RpcError> = stop_ipc
|
||||
.call_with_id(
|
||||
STOP_RPC_ID.to_string(),
|
||||
Method::SessionStop,
|
||||
Some(serde_json::json!({
|
||||
"session_id": stop_session_id,
|
||||
"all": false,
|
||||
})),
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
outcome
|
||||
});
|
||||
|
||||
stop_seen_rx.await.unwrap();
|
||||
let mut cancel_ipc = IpcClient::connect(&sock).await.unwrap();
|
||||
let cancel: CancelResult = cancel_ipc
|
||||
.call(
|
||||
"cancel-stop",
|
||||
Method::Cancel,
|
||||
Some(CancelParams {
|
||||
rpc_id: STOP_RPC_ID.to_string(),
|
||||
}),
|
||||
Duration::from_secs(2),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(cancel.cancelled);
|
||||
|
||||
let stop_error = stop_call.await.unwrap().unwrap_err();
|
||||
assert_eq!(stop_error.code, ErrorCode::Cancelled);
|
||||
|
||||
let mut status_ipc = IpcClient::connect(&sock).await.unwrap();
|
||||
let status: StatusResult = status_ipc
|
||||
.call::<(), _>(
|
||||
"status-after-stop-cancel",
|
||||
Method::SystemStatus,
|
||||
None,
|
||||
Duration::from_secs(2),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(status.sessions.len(), 1);
|
||||
assert_eq!(status.sessions[0].session_id, session_id);
|
||||
|
||||
let _ = release_tx.send(());
|
||||
responder.await.unwrap();
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn timing_out_session_stop_cancels_extension_teardown_and_keeps_session() {
|
||||
let (handle, sock) = spawn_daemon().await;
|
||||
let state = handle.state();
|
||||
let mut ws = connect_ext(handle.ws_addr()).await;
|
||||
let _ = handshake_as_ext(&mut ws).await;
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
|
||||
let responder = tokio::spawn(async move {
|
||||
respond_to_aborted_stop(&mut ws, None).await;
|
||||
let _ = release_rx.await;
|
||||
});
|
||||
|
||||
let mut start_ipc = IpcClient::connect(&sock).await.unwrap();
|
||||
let start: serde_json::Value = start_ipc
|
||||
.call(
|
||||
"start-before-stop-timeout",
|
||||
Method::SessionStart,
|
||||
Some(serde_json::json!({ "focused": false })),
|
||||
Duration::from_secs(5),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
let session_id =
|
||||
bsk::daemon::sessions::SessionId(start["session_id"].as_str().unwrap().to_string());
|
||||
|
||||
let stop = bsk::daemon::sessions::stop_session(
|
||||
&state.browsers,
|
||||
&state.sessions,
|
||||
&state.tool_queues,
|
||||
&state.session_interrupts,
|
||||
&session_id,
|
||||
Duration::from_millis(20),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(
|
||||
stop,
|
||||
Err(bsk::daemon::sessions::StopSessionError::Timeout)
|
||||
));
|
||||
assert!(state.sessions.get(&session_id).is_some());
|
||||
|
||||
let _ = release_tx.send(());
|
||||
responder.await.unwrap();
|
||||
handle.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn session_idle_timeout_stops_and_unregisters_session() {
|
||||
let (handle, _sock) = spawn_daemon_with_session_idle(Duration::from_millis(50)).await;
|
||||
|
||||
+1
-1
@@ -9,7 +9,7 @@
|
||||
"ext:dev": "pnpm --filter @browser-skill/extension dev",
|
||||
"ext:test": "pnpm --filter @browser-skill/extension test",
|
||||
"vom:test": "pnpm --filter @browser-skill/vom test",
|
||||
"lint": "biome check . && stylelint \"**/*.css\"",
|
||||
"lint": "biome check . && stylelint \"**/*.css\" && pnpm --filter @wxg-prc-cpg/browser-skill-dsh-plugin typecheck && pnpm --filter @wxg-prc-cpg/browser-skill-dsh-plugin test",
|
||||
"format": "biome format --write ."
|
||||
},
|
||||
"devDependencies": {
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
lib/
|
||||
src/client/bsk-ui.nomodule.css
|
||||
.bsk-ui.input.css
|
||||
.bsk-ui.raw.css
|
||||
src/skill-content.generated.ts
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2026 Tencent
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,195 @@
|
||||
# dsh-plugin-browserskill
|
||||
|
||||
npm: [`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)
|
||||
|
||||
A [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (dsh) tool plugin that exposes
|
||||
[BrowserSkill](https://github.com/Tencent/BrowserSkill) (`bsk`) browser automation to the model.
|
||||
|
||||
Each tool maps to one `bsk <cmd> --json` invocation: the plugin spawns the bsk CLI, parses its
|
||||
structured JSON output, and returns a canonical typed value. The bsk daemon, browser, and browser
|
||||
extension keep owning the actual browser control — this package is a thin, well-typed bridge.
|
||||
|
||||
## Tools
|
||||
|
||||
| Tool | bsk command | Purpose |
|
||||
| --- | --- | --- |
|
||||
| `browser_session_start` | `bsk session start` | Open an Agent Window session; optional initial URL, window size, and device emulation preset. Returns the session id and makes it the current session. |
|
||||
| `browser_session_stop` | `bsk session stop` | Stop a session (the current one by default) and close its Agent Window. Only plugin-created sessions can be stopped. |
|
||||
| `browser_session_list` | — (registry only) | List the sessions this plugin created, marking the current one. Foreign daemon sessions are never shown. |
|
||||
| `browser_navigate` | `bsk navigate` | Navigate the active tab, with `waitUntil` / timeout control. |
|
||||
| `browser_snapshot` | `bsk snapshot` | Indented aria-tree snapshot with `@eN` refs for interaction tools. |
|
||||
| `browser_observe` | `bsk observe` | Semantic VOM observation (read-only) with `@eN` refs. |
|
||||
| `browser_click` | `bsk click` | Click a snapshot ref or CSS selector (button / click-count options). |
|
||||
| `browser_fill` | `bsk fill` | Fill an input / textarea / contenteditable (clears first by default). |
|
||||
| `browser_press` | `bsk press` | Dispatch a key or combo, optionally focusing a target first. |
|
||||
| `browser_screenshot` | `bsk screenshot` | PNG capture of the tab or a ref-cropped element; inlines the image when the deployment supports image input, otherwise returns a file path. |
|
||||
| `browser_emulate` | `bsk emulate` | Apply or clear mobile device emulation on the active tab. |
|
||||
|
||||
## Agent skill (progressive disclosure)
|
||||
|
||||
Beyond the tools, the plugin publishes the **`browser-skill` agent skill** through the harness's
|
||||
official skill seam (`ctx.skills.register`): the catalog entry (name + routing description) is
|
||||
resident in `<available_skills>`, and the body is loaded only when the model invokes the `skill`
|
||||
tool. The body is assembled at build time from two parts, so there is exactly one source of
|
||||
truth: a dsh-specific prelude (`skill/prelude.md` — tool↔CLI map, owned-session semantics,
|
||||
plugin-only overrides) followed verbatim by the canonical CLI skill (`skill/SKILL.md` at the
|
||||
repo root — the same file `crates/bsk-cli/build.rs` mirrors for the CLI package; workflows,
|
||||
stop-when-done rules, refs usage, sandbox rules). Registration and every pre-step catalog
|
||||
snapshot are pure in-memory reads (no disk/process/daemon); compositions without the skill
|
||||
seam degrade silently.
|
||||
|
||||
## Multi-session model
|
||||
|
||||
One agent conversation can drive several browser sessions at once:
|
||||
|
||||
- `browser_session_start` returns the session id and makes it the **current session**.
|
||||
- Every operation tool accepts an optional `session` argument. When omitted, the call acts on the
|
||||
current session (the one most recently started or used); when given, that session becomes current.
|
||||
- Every tool result echoes the session it actually acted on, so the model never has to guess.
|
||||
- The number of concurrent sessions started through the plugin is capped (`maxSessions`, default 5).
|
||||
- Unloading the plugin stops every session it started and kills in-flight bsk processes.
|
||||
|
||||
**Ownership boundary**: the bsk daemon may be shared with other agents, terminals, or dsh
|
||||
instances. The plugin therefore only ever sees and operates on sessions it created itself —
|
||||
an explicit `session` argument naming a foreign or unknown id is rejected, `browser_session_list`
|
||||
shows plugin-created sessions only (no daemon-wide view), and stop/unload cleanup can never touch
|
||||
a session owned by another program.
|
||||
|
||||
## Installation
|
||||
|
||||
The plugin follows the standard dsh bundle layout (`dsh.bundle` manifest + `cordis.patch.yml`):
|
||||
|
||||
```sh
|
||||
dsh plugin --profile <name> add @wxg-prc-cpg/browser-skill-dsh-plugin
|
||||
dsh --profile <name>
|
||||
```
|
||||
|
||||
Prerequisite: the `bsk` CLI must be installed and on `PATH`, and the BrowserSkill browser extension
|
||||
(Chrome or Edge) must be connected — see the
|
||||
[BrowserSkill README](https://github.com/Tencent/BrowserSkill). When bsk is missing, tool calls fail
|
||||
with install guidance instead of a bare spawn error.
|
||||
|
||||
## Configuration
|
||||
|
||||
All fields are optional and validated through the plugin's Schemastery `Config`:
|
||||
|
||||
```yaml
|
||||
# cordis.patch.yml override example
|
||||
- insert:
|
||||
- id: browserskill
|
||||
name: "@wxg-prc-cpg/browser-skill-dsh-plugin"
|
||||
config:
|
||||
bskPath: bsk # path to the bsk binary (default: resolve from PATH)
|
||||
defaultTimeoutMs: 120000
|
||||
maxSessions: 5
|
||||
# observationEnabled: true # live PiP/overlay observation (below)
|
||||
# thumbnailIntervalMs: 1500 # frame cadence while a session is active
|
||||
# idleIntervalMs: 8000 # idle cadence / recent-activity window
|
||||
# lazyTools: true # reveal browser_* tools only after the skill is invoked
|
||||
```
|
||||
|
||||
- **`lazyTools` (default `true`)** — the final progressive-disclosure stage: the eleven
|
||||
`browser_*` tool schemas stay OUT of the system prompt (zero schema tokens) until the
|
||||
`browser-skill` skill is actually invoked — the skill catalog entry is the only
|
||||
advertisement. One successful invocation (model tool call, or a `/browser-skill` user
|
||||
gesture) registers the whole suite for the rest of the process; repeated invocations are
|
||||
no-ops, and sessions resumed with a past invocation in their durable log reveal the suite
|
||||
on entry. Set `false` for the legacy always-on registration.
|
||||
|
||||
## Observation overlay (PiP mini-window)
|
||||
|
||||
When the plugin runs inside the dsh Web UI, an **observation overlay** floats over the app
|
||||
(registered into the `shell.overlay` seat): a breathing thumbnail per owned session plus its
|
||||
current action and elapsed time. The card docks at the top-right of the content area (clear of
|
||||
the composer and the shell's header controls) and wears the **BrowserSkill product family's own
|
||||
look**: the overlay reuses `@browser-skill/ui` components (`Button`, `cn`) and its oklch design
|
||||
tokens (`--card`, `--primary`, `--destructive`, `--ring`, …), status dots spec'd after the
|
||||
extension popup's connection indicator, and Remix icons. The BSK utility sheet is compiled
|
||||
scoped under the `.bsk-obs` root class (`scripts/build-client-css.mjs`), so the overlay looks
|
||||
like the BrowserSkill extension without leaking a single selector into the host shell — and the
|
||||
shell's theme cannot bleed back in.
|
||||
|
||||
- **Lifecycle**: hidden while the plugin owns no sessions; appears on the first
|
||||
`browser_session_start`; disappears when all sessions stop (or the plugin unloads).
|
||||
- **Focus view**: status row (green/idle/red dot + session + action + mm:ss), the latest page
|
||||
frame (refreshes every ~1.5s while active, ~8s when idle; the last good frame stays on
|
||||
stage while the next one loads, and is kept on errors so the card does not flash),
|
||||
and a compact icon toolbar (Interrupt + Pop out, hover for the label).
|
||||
- **Interrupt**: one click kills the in-flight bsk command of the focus session (same semantics
|
||||
as the chat Stop button — the current action fails, the agent run may continue). Strip items
|
||||
carry their own hover interrupt button.
|
||||
- **Multi-session strip**: every session gets a tile (mini frame + id + status dot); focus
|
||||
auto-follows the most recently active session; clicking a tile pins focus (pin badge, click
|
||||
again to release); errored sessions get a red edge without stealing focus; sessions the daemon
|
||||
lost are greyed out; prolonged daemon/browser outage shows "browser unavailable" and greys
|
||||
the interrupt button until captures recover.
|
||||
- **Drag & resize**: drag the header to move the card; drag any of the four
|
||||
corners to resize (min 240×180, max 80% of the viewport; no visible grip).
|
||||
Both are remembered for the page lifetime.
|
||||
- **Pop out (PiP)**: upgrades the card into a native Document PiP window (requires a user
|
||||
gesture, per browser rules), sized from the current card; closing the PiP falls back to the
|
||||
in-page card with state intact. Browsers without Document PiP simply hide the button.
|
||||
- **Wire**: the host serves `GET /bsk-observation/state`, `GET /bsk-observation/events` (SSE),
|
||||
`POST /bsk-observation/interrupt`, and `GET /bsk-observation/thumbnail/<attachmentId>` over
|
||||
the dsh `webServer` route seam (dsh 0.1's Typert Remote pipeline is closed to out-of-tree
|
||||
packages). All commands for one session — tool calls and frame captures alike — run through a
|
||||
per-session FIFO, because the daemon accepts only one unfinished command per session.
|
||||
- **Trust model**: these routes expose live screenshots (and an interrupt write), so they
|
||||
replicate the browser-trust fence dsh applies to its own `/api` routes: the request Host
|
||||
must be a loopback authority (`localhost`, `127.0.0.0/8`, `[::1]`), a present Origin must
|
||||
match the Host, `sec-fetch-site: cross-site` is refused, and POST requires an
|
||||
`application/json` body (cross-site simple requests can never satisfy that). The channel is
|
||||
therefore built for **loopback-only serving** — binding the dsh web server to `0.0.0.0` and
|
||||
reaching it through a LAN address will (deliberately) fail the fence; do not put these
|
||||
routes behind a non-loopback reverse proxy without adding your own authentication.
|
||||
- Configure with `observationEnabled` / `thumbnailIntervalMs` / `idleIntervalMs`.
|
||||
|
||||
## Behavior notes
|
||||
|
||||
- **Cancellation**: aborting a tool call (`exec.signal`) kills the underlying bsk child process,
|
||||
matching BrowserSkill's cooperative tool-cancellation model.
|
||||
- **UI cards**: calls render as terminal cards (command line as title, output as the completed
|
||||
card). Screenshots additionally attach the image itself when the host mounts an attachment store
|
||||
and the active model route declares image input; otherwise the PNG path is returned.
|
||||
- **Web UI toolview (browser half)**: the package is dual-face. `dsh.client` (platform `web`) ships
|
||||
`lib/client.js`, which registers a keyed `tool.call.toolview` view for `browser_screenshot`. The
|
||||
custom view keeps the terminal block (command + output) and, when the settled result carries an
|
||||
image block, resolves the durable attachment through the client session's authorized
|
||||
`readAttachment` RPC and renders it with the shared `MessageImage` thumbnail/lightbox atoms.
|
||||
Every other `browser_*` tool keeps the stock terminal card. The bundle follows the dsh client
|
||||
contract: a CJS closure factory handed to `window.__ModuleLoader__.load`, platform modules
|
||||
(`react`, `dsh-client-ui-*`) external, everything else inlined, CSS Modules compiled by
|
||||
lightningcss.
|
||||
- **Errors**: non-zero bsk exits surface the CLI's JSON error envelope (`code`, `message`, `hint`)
|
||||
so the model gets the daemon's actionable guidance.
|
||||
- **Long-running work** (e.g. `bsk record`) is not backgrounded via `ctx.jobs` yet — tracked as a
|
||||
follow-up.
|
||||
|
||||
## Development
|
||||
|
||||
```sh
|
||||
pnpm install
|
||||
pnpm --filter @wxg-prc-cpg/browser-skill-dsh-plugin typecheck
|
||||
pnpm --filter @wxg-prc-cpg/browser-skill-dsh-plugin test # unit tests mock bsk; no browser needed
|
||||
pnpm --filter @wxg-prc-cpg/browser-skill-dsh-plugin build # tsdown -> lib/
|
||||
```
|
||||
|
||||
## Publishing
|
||||
|
||||
The GitHub Actions workflow **Release dsh plugin** publishes this package to npm
|
||||
as `@wxg-prc-cpg/browser-skill-dsh-plugin`. The Cordis plugin id stays
|
||||
`dsh-plugin-browserskill`.
|
||||
|
||||
Trigger it by pushing a tag that matches `package.json`'s `version`:
|
||||
|
||||
```sh
|
||||
git tag dsh-plugin-v0.1.0
|
||||
git push origin dsh-plugin-v0.1.0
|
||||
```
|
||||
|
||||
Or run the workflow from the Actions tab (`workflow_dispatch`). The job reads the
|
||||
`NPM_TOKEN` secret from the `npm-publish` GitHub Environment.
|
||||
|
||||
## License
|
||||
|
||||
MIT
|
||||
@@ -0,0 +1,5 @@
|
||||
- insert:
|
||||
- id: browserskill
|
||||
# Must be the published package name — the loader imports this specifier.
|
||||
# Quoted because YAML reserves a leading '@' in plain scalars.
|
||||
name: "@wxg-prc-cpg/browser-skill-dsh-plugin"
|
||||
@@ -0,0 +1,130 @@
|
||||
{
|
||||
"name": "@wxg-prc-cpg/browser-skill-dsh-plugin",
|
||||
"description": "DeepSeek Harness tool plugin that exposes BrowserSkill (bsk) browser automation to the model",
|
||||
"version": "0.1.1",
|
||||
"license": "MIT",
|
||||
"publishConfig": {
|
||||
"access": "public",
|
||||
"registry": "https://registry.npmjs.org/"
|
||||
},
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/Tencent/BrowserSkill.git",
|
||||
"directory": "packages/dsh-plugin-browserskill"
|
||||
},
|
||||
"homepage": "https://github.com/Tencent/BrowserSkill/tree/main/packages/dsh-plugin-browserskill#readme",
|
||||
"bugs": {
|
||||
"url": "https://github.com/Tencent/BrowserSkill/issues"
|
||||
},
|
||||
"keywords": [
|
||||
"dsh-plugin",
|
||||
"deepseek-harness",
|
||||
"browserskill",
|
||||
"browser-automation"
|
||||
],
|
||||
"type": "module",
|
||||
"main": "lib/index.mjs",
|
||||
"types": "lib/index.d.mts",
|
||||
"exports": {
|
||||
".": {
|
||||
"types": "./lib/index.d.mts",
|
||||
"default": "./lib/index.mjs"
|
||||
},
|
||||
"./client": {
|
||||
"default": "./lib/client.cjs"
|
||||
},
|
||||
"./package.json": "./package.json"
|
||||
},
|
||||
"files": [
|
||||
"lib",
|
||||
"cordis.patch.yml",
|
||||
"LICENSE"
|
||||
],
|
||||
"dsh": {
|
||||
"bundle": {
|
||||
"patch": "./cordis.patch.yml"
|
||||
},
|
||||
"client": {
|
||||
"platform": "web",
|
||||
"inject": [
|
||||
"@deepseek-ai/dsh-client-runtime",
|
||||
"@deepseek-ai/dsh-client-ui-tool",
|
||||
"@deepseek-ai/dsh-client-ui-layout"
|
||||
]
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"build": "node scripts/build-client-css.mjs && node scripts/build-skill-content.mjs && tsdown",
|
||||
"typecheck": "pnpm run build:skill && tsc --noEmit",
|
||||
"test": "pnpm run build:css && pnpm run build:skill && vitest run",
|
||||
"build:css": "node scripts/build-client-css.mjs",
|
||||
"build:skill": "node scripts/build-skill-content.mjs",
|
||||
"prepack": "pnpm run build",
|
||||
"lint:publint": "publint"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@deepseek-ai/cordis": "^4.0.1",
|
||||
"@deepseek-ai/dsh-attachment": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/dsh-client-runtime": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/dsh-client-ui-attachment": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/dsh-client-ui-primitives": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/dsh-client-ui-tool": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/dsh-llm": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/dsh-tools": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/schemastery": "^3.18.1",
|
||||
"react": "^18.2.0"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"@deepseek-ai/dsh-attachment": {
|
||||
"optional": true
|
||||
},
|
||||
"@deepseek-ai/dsh-client-runtime": {
|
||||
"optional": true
|
||||
},
|
||||
"@deepseek-ai/dsh-client-ui-attachment": {
|
||||
"optional": true
|
||||
},
|
||||
"@deepseek-ai/dsh-client-ui-primitives": {
|
||||
"optional": true
|
||||
},
|
||||
"@deepseek-ai/dsh-client-ui-tool": {
|
||||
"optional": true
|
||||
},
|
||||
"@deepseek-ai/dsh-llm": {
|
||||
"optional": true
|
||||
},
|
||||
"react": {
|
||||
"optional": true
|
||||
}
|
||||
},
|
||||
"devDependencies": {
|
||||
"@deepseek-ai/cordis": "^4.0.1",
|
||||
"@deepseek-ai/dsh-agent": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/dsh-attachment": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/dsh-client-runtime": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/dsh-client-ui-attachment": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/dsh-client-ui-layout": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/dsh-client-ui-primitives": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/dsh-client-ui-tool": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/dsh-llm": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/dsh-tools": "^0.1.0-rc.6",
|
||||
"@deepseek-ai/schemastery": "^3.18.1",
|
||||
"@tailwindcss/cli": "^4.3.3",
|
||||
"@testing-library/react": "^16.1.0",
|
||||
"@types/node": "^22.10.0",
|
||||
"@types/react": "~18.3.1",
|
||||
"@types/react-dom": "^18.3.0",
|
||||
"happy-dom": "^15.11.7",
|
||||
"lightningcss": "^1.30.0",
|
||||
"postcss": "^8.5.0",
|
||||
"postcss-prefix-selector": "^2.1.1",
|
||||
"react": "^18.2.0",
|
||||
"react-dom": "^18.2.0",
|
||||
"tailwindcss": "^4.3.3",
|
||||
"tsdown": "^0.22.14",
|
||||
"typescript": "^5.8.0",
|
||||
"vitest": "^4.1.6",
|
||||
"@browser-skill/ui": "workspace:*",
|
||||
"@remixicon/react": "^4.9.0"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Build the scoped BrowserSkill-UI utility sheet for the dsh client bundle.
|
||||
*
|
||||
* The overlay reuses @browser-skill/ui components (shadcn-style, Tailwind v4
|
||||
* utility classes). Those classes need a stylesheet — but a global utility
|
||||
* sheet inside the dsh page would collide with the host UI, so the compiled
|
||||
* output is scope-prefixed under `.bsk-obs` (the overlay root class):
|
||||
* 1. tailwindcss compiles the utilities used by packages/ui + our client;
|
||||
* 2. postcss-prefix-selector nests every rule under .bsk-obs.
|
||||
* Token values themselves live in src/client/bsk-tokens.nomodule.css
|
||||
* (hand-mirrored from packages/ui/src/styles/tailwind.css).
|
||||
*
|
||||
* Output: src/client/bsk-ui.nomodule.css (gitignored build artifact).
|
||||
*/
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import postcss from "postcss";
|
||||
import prefixer from "postcss-prefix-selector";
|
||||
|
||||
const pkgRoot = dirname(dirname(fileURLToPath(import.meta.url)));
|
||||
const input = join(pkgRoot, ".bsk-ui.input.css");
|
||||
const rawOut = join(pkgRoot, ".bsk-ui.raw.css");
|
||||
const finalOut = join(pkgRoot, "src/client/bsk-ui.nomodule.css");
|
||||
|
||||
// The canonical sheet can't be imported directly (its own `@import
|
||||
// "tailwindcss"` must resolve from a package that has it), so we inline the
|
||||
// canonical @theme block read from packages/ui at build time.
|
||||
const canonical = readFileSync(join(pkgRoot, "..", "ui", "src", "styles", "tailwind.css"), "utf8");
|
||||
const themeBlock = canonical.match(/@theme inline \{[\s\S]*?\n\}/)?.[0];
|
||||
if (themeBlock === undefined) throw new Error("canonical @theme inline block not found");
|
||||
writeFileSync(
|
||||
input,
|
||||
[
|
||||
'@import "tailwindcss";',
|
||||
"@custom-variant dark (&:is(.dark *));",
|
||||
themeBlock,
|
||||
'@source "../ui/src";',
|
||||
'@source "./src/client";',
|
||||
"",
|
||||
].join("\n"),
|
||||
);
|
||||
|
||||
execFileSync(
|
||||
process.execPath,
|
||||
[
|
||||
join(pkgRoot, "node_modules", "@tailwindcss", "cli", "dist", "index.mjs"),
|
||||
"-i",
|
||||
input,
|
||||
"-o",
|
||||
rawOut,
|
||||
],
|
||||
{ cwd: pkgRoot, stdio: "inherit" },
|
||||
);
|
||||
|
||||
const result = await postcss([
|
||||
prefixer({
|
||||
prefix: ".bsk-obs",
|
||||
transform(prefix, selector, prefixedSelector) {
|
||||
// Root-level anchors collapse onto the scope root itself.
|
||||
if (selector === ":root" || selector === "html" || selector === "body") return prefix;
|
||||
if (selector.startsWith(prefix)) return selector;
|
||||
return prefixedSelector;
|
||||
},
|
||||
}),
|
||||
]).process(readFileSync(rawOut, "utf8"), { from: rawOut });
|
||||
|
||||
mkdirSync(dirname(finalOut), { recursive: true });
|
||||
writeFileSync(finalOut, result.css);
|
||||
console.log(`bsk-ui sheet: ${result.css.length} bytes -> ${finalOut}`);
|
||||
@@ -0,0 +1,44 @@
|
||||
// Assemble the plugin's skill content: a dsh-specific routing prelude, the
|
||||
// canonical CLI skill body verbatim (repo-root skill/SKILL.md — the same source
|
||||
// crates/bsk-cli's build.rs mirrors), then a dsh routing reminder postlude. The
|
||||
// original's YAML frontmatter is stripped: name/description are supplied by the
|
||||
// registration.
|
||||
// Output: src/skill-content.generated.ts (gitignored, built before tsdown and
|
||||
// before vitest) so the markdown ships inlined in the bundle and registration
|
||||
// is a pure in-memory read — never a per-step disk access.
|
||||
import { readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const here = dirname(fileURLToPath(import.meta.url));
|
||||
const pkg = join(here, "..");
|
||||
const prelude = readFileSync(join(pkg, "skill", "prelude.md"), "utf8");
|
||||
const upstream = readFileSync(join(pkg, "..", "..", "skill", "SKILL.md"), "utf8");
|
||||
const postlude = readFileSync(join(pkg, "skill", "postlude.md"), "utf8");
|
||||
|
||||
function stripFrontmatter(markdown) {
|
||||
if (!markdown.startsWith("---\n")) return markdown;
|
||||
const end = markdown.indexOf("\n---\n", 4);
|
||||
return end === -1 ? markdown : markdown.slice(end + 5).replace(/^\s*\n/, "");
|
||||
}
|
||||
|
||||
const NAME = "browser-skill";
|
||||
const DESCRIPTION =
|
||||
"Browser automation against the user's logged-in Chromium through this plugin's injected browser_* tools; never invoke bsk through bash or shell. " +
|
||||
"tools (bsk engine): visit and read pages, fill forms, click through flows, scrape data, " +
|
||||
"smoke-test UI changes. Use when a task needs a real browser; requires the bsk CLI and the " +
|
||||
"browser-skill extension connected.";
|
||||
|
||||
const content = `${prelude.trim()}\n\n${stripFrontmatter(upstream).trim()}\n\n${postlude.trim()}\n`;
|
||||
|
||||
const banner =
|
||||
"// GENERATED by scripts/build-skill-content.mjs — do not edit.\n" +
|
||||
"// Sources: skill/prelude.md + ../../../skill/SKILL.md + skill/postlude.md.\n";
|
||||
|
||||
writeFileSync(
|
||||
join(pkg, "src", "skill-content.generated.ts"),
|
||||
`${banner}export const BSK_SKILL_NAME = ${JSON.stringify(NAME)};\n` +
|
||||
`export const BSK_SKILL_DESCRIPTION = ${JSON.stringify(DESCRIPTION)};\n` +
|
||||
`export const BSK_SKILL_MARKDOWN = ${JSON.stringify(content)};\n`,
|
||||
);
|
||||
console.log(`skill-content.generated.ts written (${content.length} chars)`);
|
||||
@@ -0,0 +1,10 @@
|
||||
---
|
||||
|
||||
## DSH routing reminder
|
||||
|
||||
The CLI-oriented reference above describes behavior, not the execution mechanism in
|
||||
DeepSeek Harness. For every browser operation, call the injected `browser_*` model
|
||||
tool. Do not translate these examples into `bash` commands and do not execute `bsk`
|
||||
directly; the plugin invokes `bsk` internally. In particular, always begin with
|
||||
`browser_session_start`, pass its returned session id to subsequent `browser_*`
|
||||
tools, and finish with `browser_session_stop`.
|
||||
@@ -0,0 +1,71 @@
|
||||
# browser-skill (dsh edition)
|
||||
|
||||
## DSH tool-routing requirement (highest priority)
|
||||
|
||||
This skill is running inside DeepSeek Harness, where the plugin has injected
|
||||
structured `browser_*` model tools. **Always call those injected `browser_*` tools
|
||||
for browser work. Never use `bash`, `shell`, `exec`, or another tool to invoke the
|
||||
`bsk` CLI directly.** The CLI command names shown below are only conceptual
|
||||
mappings for understanding the tools; they are not instructions to run commands.
|
||||
If a `browser_*` tool is available, using the CLI directly is incorrect because it
|
||||
bypasses plugin session ownership, observation UI, tool cards, and cleanup.
|
||||
|
||||
Use the injected tool whose name matches the operation, for example:
|
||||
`browser_session_start`, `browser_navigate`, `browser_observe`, `browser_snapshot`,
|
||||
`browser_click`, `browser_fill`, `browser_press`, `browser_screenshot`, and
|
||||
`browser_session_stop`. Pass arguments as the tool schema specifies (not CLI flags).
|
||||
|
||||
Drive the user's **real Chromium browser** (logins, cookies, Agent Window isolation)
|
||||
through this plugin's `browser_*` tools. Each injected tool is implemented by the
|
||||
plugin using the `bsk` engine internally — the model must not invoke that engine
|
||||
itself. The workflow, judgment rules, and safety constraints remain identical to
|
||||
the canonical CLI skill.
|
||||
|
||||
> **Tool availability**: invoking this skill (which you just did) makes the plugin's
|
||||
> `browser_*` tools available for the rest of the session — they are injected on
|
||||
> successful skill load. Until then they are deliberately hidden, so do not attempt
|
||||
> a browser action before loading this skill.
|
||||
|
||||
## Tool ↔ CLI map
|
||||
|
||||
The tools below are thin, structured wrappers over the bsk CLI verbs of the same
|
||||
name; `session` parameters take the 4-letter session id (no `--session` flag
|
||||
spelling here):
|
||||
|
||||
| dsh tool | bsk equivalent |
|
||||
|---|---|
|
||||
| `browser_session_start` / `browser_session_stop` / `browser_session_list` | `bsk session start/stop/list` |
|
||||
| `browser_navigate` | `bsk navigate` |
|
||||
| `browser_snapshot` / `browser_observe` | `bsk snapshot` / `bsk observe` |
|
||||
| `browser_click` / `browser_fill` / `browser_press` | `bsk click` / `bsk fill` / `bsk press` |
|
||||
| `browser_screenshot` | `bsk screenshot` |
|
||||
| `browser_emulate` | `bsk emulate` |
|
||||
|
||||
(`bsk console` / `bsk network` have no tool counterpart in this plugin yet.)
|
||||
|
||||
Plugin-specific semantics that override or narrow the CLI rules below:
|
||||
|
||||
- **Owned sessions only**: this plugin sees and acts on just the sessions IT
|
||||
started via `browser_session_start`. `browser_session_list` shows only those
|
||||
(with the current marker); passing a session id created by another program
|
||||
(or person) sharing the bsk daemon is refused outright — never retry those
|
||||
with the same id.
|
||||
- **Current session**: omitted `session` parameters resolve to the most
|
||||
recently started or used OWNED session. With several sessions in flight,
|
||||
pass `session` explicitly to avoid surprises.
|
||||
- **Screenshots**: when the model route accepts images, `browser_screenshot`
|
||||
returns the PNG inline as an image attachment; otherwise it returns the
|
||||
saved file path.
|
||||
- **`browser_emulate mobile`**: requires `width`+`height` — the bsk daemon
|
||||
refuses `--mobile` without viewport dimensions.
|
||||
- **Live observation**: in the dsh Web UI, every owned session is watched by
|
||||
the floating overlay (thumbnails, action, interrupt) — observational side
|
||||
traffic is free and never steals your command turn; just keep acting.
|
||||
|
||||
The canonical CLI skill body follows verbatim (source of truth: `skill/SKILL.md`
|
||||
in the BrowserSkill repo, mirrored by build into `crates/bsk-cli/skill/SKILL.md`).
|
||||
Read `bsk <verb> --session <id>` as "the `browser_<verb>` tool with
|
||||
`session: <id>`"; everything else — when to use, the stop-when-done rules, the
|
||||
interaction loop, refs, sandbox rules — applies unchanged.
|
||||
|
||||
---
|
||||
@@ -0,0 +1,370 @@
|
||||
/* Observation overlay skin, BrowserSkill family: colors/radius ride the BSK
|
||||
design tokens (--card/--border/--primary/--destructive/--ring/--radius,
|
||||
defined on the .bsk-obs scope root by bsk-tokens.nomodule.css — values
|
||||
mirrored from packages/ui/src/styles/tailwind.css). Layout-only here;
|
||||
component chrome (buttons etc.) comes from @browser-skill/ui plus the
|
||||
scope-prefixed utility sheet. The shell overlay layer is click-through, so
|
||||
the card and capsule opt back into pointer events themselves. */
|
||||
|
||||
.card {
|
||||
position: fixed;
|
||||
z-index: 40;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
pointer-events: auto;
|
||||
background: var(--card);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: var(--radius);
|
||||
box-shadow: 0 8px 28px rgb(0 0 0 / 18%);
|
||||
overflow: hidden;
|
||||
color: var(--card-foreground);
|
||||
}
|
||||
|
||||
.body {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
height: 100%;
|
||||
min-height: 0;
|
||||
font-size: 12px;
|
||||
background: var(--card);
|
||||
color: var(--card-foreground);
|
||||
}
|
||||
|
||||
.header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
padding: 6px 8px;
|
||||
cursor: move;
|
||||
user-select: none;
|
||||
touch-action: none;
|
||||
border-bottom: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.status-text {
|
||||
flex: 1;
|
||||
min-width: 0;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.icon-button {
|
||||
flex: none;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
border: none;
|
||||
background: none;
|
||||
padding: 2px;
|
||||
border-radius: calc(var(--radius) - 6px);
|
||||
font: inherit;
|
||||
color: var(--muted-foreground);
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.icon-button:hover {
|
||||
color: var(--foreground);
|
||||
background: var(--accent);
|
||||
}
|
||||
|
||||
.stage {
|
||||
position: relative;
|
||||
flex: 1;
|
||||
min-height: 0;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
background: var(--background);
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
/* Fade only on first mount. The <img> is reused across frames (src swap,
|
||||
no React key) so this animation must not replay on every breath. */
|
||||
.thumb {
|
||||
max-width: 100%;
|
||||
max-height: 100%;
|
||||
object-fit: contain;
|
||||
animation: bsk-obs-fade-in 240ms ease-out;
|
||||
}
|
||||
@keyframes bsk-obs-fade-in {
|
||||
from {
|
||||
opacity: 0;
|
||||
}
|
||||
|
||||
to {
|
||||
opacity: 1;
|
||||
}
|
||||
}
|
||||
|
||||
.placeholder {
|
||||
color: var(--muted-foreground);
|
||||
font-size: 12px;
|
||||
}
|
||||
|
||||
.badge {
|
||||
position: absolute;
|
||||
top: 6px;
|
||||
right: 6px;
|
||||
width: 16px;
|
||||
height: 16px;
|
||||
border-radius: 50%;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
color: oklch(0.7 0.19 60);
|
||||
border: 1px solid oklch(0.7 0.19 60);
|
||||
background: var(--card);
|
||||
}
|
||||
|
||||
.actions {
|
||||
position: relative;
|
||||
z-index: 3;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: 4px;
|
||||
|
||||
/* A little inset so icons are not flush with the resize corners. */
|
||||
padding: 4px 8px;
|
||||
min-height: 32px;
|
||||
border-top: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.tool-wrap {
|
||||
position: relative;
|
||||
display: inline-flex;
|
||||
}
|
||||
|
||||
.tool-button {
|
||||
flex: none;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
width: 28px;
|
||||
height: 28px;
|
||||
padding: 0;
|
||||
border: none;
|
||||
border-radius: calc(var(--radius) - 4px);
|
||||
background: none;
|
||||
color: var(--muted-foreground);
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.tool-button:disabled {
|
||||
opacity: 0.4;
|
||||
cursor: default;
|
||||
}
|
||||
|
||||
.tool-button:focus-visible {
|
||||
outline: 2px solid var(--ring);
|
||||
outline-offset: 2px;
|
||||
}
|
||||
|
||||
.tool-danger {
|
||||
color: var(--destructive);
|
||||
}
|
||||
|
||||
/* Hover the wrap (not just :hover on the button) so the tooltip target and
|
||||
the left-corner resize overlay still light the icon up. */
|
||||
.tool-wrap:hover .tool-button:not(:disabled) {
|
||||
color: var(--foreground);
|
||||
background: var(--accent);
|
||||
}
|
||||
|
||||
.tool-wrap:hover .tool-danger:not(:disabled) {
|
||||
color: var(--destructive);
|
||||
background: color-mix(in oklch, var(--destructive) 18%, var(--card));
|
||||
}
|
||||
|
||||
.tool-wrap:hover .tool-button:disabled {
|
||||
background: var(--accent);
|
||||
}
|
||||
|
||||
.hint {
|
||||
position: absolute;
|
||||
bottom: calc(100% + 4px);
|
||||
left: 0;
|
||||
width: max-content;
|
||||
max-width: 220px;
|
||||
padding: 4px 6px;
|
||||
border-radius: calc(var(--radius) - 4px);
|
||||
background: var(--card);
|
||||
border: 1px solid var(--border);
|
||||
color: var(--muted-foreground);
|
||||
font-size: 11px;
|
||||
line-height: 1.35;
|
||||
box-shadow: 0 4px 14px rgb(0 0 0 / 16%);
|
||||
z-index: 41;
|
||||
}
|
||||
|
||||
.hint[data-align="end"] {
|
||||
left: auto;
|
||||
right: 0;
|
||||
}
|
||||
|
||||
/* Invisible hit targets — no grip glyph; all four corners resize. */
|
||||
.resize-handle {
|
||||
position: absolute;
|
||||
width: 16px;
|
||||
height: 16px;
|
||||
touch-action: none;
|
||||
z-index: 2;
|
||||
}
|
||||
|
||||
.resize-handle[data-corner="nw"] {
|
||||
top: 0;
|
||||
left: 0;
|
||||
cursor: nwse-resize;
|
||||
}
|
||||
|
||||
.resize-handle[data-corner="ne"] {
|
||||
top: 0;
|
||||
right: 0;
|
||||
cursor: nesw-resize;
|
||||
}
|
||||
|
||||
.resize-handle[data-corner="sw"] {
|
||||
bottom: 0;
|
||||
left: 0;
|
||||
cursor: nesw-resize;
|
||||
}
|
||||
|
||||
.resize-handle[data-corner="se"] {
|
||||
bottom: 0;
|
||||
right: 0;
|
||||
cursor: nwse-resize;
|
||||
}
|
||||
|
||||
.capsule {
|
||||
position: fixed;
|
||||
right: 16px;
|
||||
top: 64px;
|
||||
z-index: 40;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 6px;
|
||||
padding: 6px 12px;
|
||||
pointer-events: auto;
|
||||
border-radius: 999px;
|
||||
border: 1px solid var(--border);
|
||||
background: var(--card);
|
||||
box-shadow: 0 4px 14px rgb(0 0 0 / 16%);
|
||||
font-size: 12px;
|
||||
color: var(--muted-foreground);
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.capsule:hover {
|
||||
background: var(--accent);
|
||||
}
|
||||
|
||||
.strip {
|
||||
display: flex;
|
||||
gap: 6px;
|
||||
padding: 4px 8px;
|
||||
overflow-x: auto;
|
||||
border-top: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.strip-item {
|
||||
position: relative;
|
||||
flex: none;
|
||||
border-radius: calc(var(--radius) - 6px);
|
||||
border: 1px solid var(--border);
|
||||
}
|
||||
|
||||
.strip-item[data-focused] {
|
||||
border-color: var(--primary);
|
||||
}
|
||||
|
||||
.strip-item[data-state="error"] {
|
||||
border-color: var(--destructive);
|
||||
}
|
||||
|
||||
.strip-item[data-state="dead"] {
|
||||
opacity: 0.45;
|
||||
}
|
||||
|
||||
.strip-main {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 4px;
|
||||
padding: 3px 6px;
|
||||
border: none;
|
||||
border-radius: calc(var(--radius) - 6px);
|
||||
background: none;
|
||||
font: inherit;
|
||||
font-size: 11px;
|
||||
color: var(--muted-foreground);
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.strip-main:hover {
|
||||
background: var(--accent);
|
||||
}
|
||||
|
||||
.strip-thumb {
|
||||
width: 28px;
|
||||
height: 20px;
|
||||
overflow: hidden;
|
||||
border-radius: 3px;
|
||||
background: var(--background);
|
||||
display: inline-flex;
|
||||
}
|
||||
|
||||
.strip-thumb img {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
object-fit: cover;
|
||||
}
|
||||
|
||||
.strip-thumb-empty {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
|
||||
.strip-id {
|
||||
max-width: 64px;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
}
|
||||
|
||||
.pin-badge {
|
||||
color: var(--primary);
|
||||
}
|
||||
|
||||
.strip-interrupt {
|
||||
position: absolute;
|
||||
top: -7px;
|
||||
right: -7px;
|
||||
width: 16px;
|
||||
height: 16px;
|
||||
border-radius: 50%;
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
border: 1px solid var(--destructive);
|
||||
background: var(--card);
|
||||
color: var(--destructive);
|
||||
padding: 0;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.strip-interrupt:disabled {
|
||||
opacity: 0.4;
|
||||
cursor: default;
|
||||
}
|
||||
|
||||
.icon-button:focus-visible,
|
||||
.capsule:focus-visible,
|
||||
.strip-main:focus-visible,
|
||||
.strip-interrupt:focus-visible {
|
||||
outline: 2px solid var(--ring);
|
||||
outline-offset: 2px;
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.thumb {
|
||||
animation: none;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,612 @@
|
||||
// Observation overlay: the default in-app carrier for live session watching.
|
||||
// Expanded = draggable/resizable floating card (top-right, out of the
|
||||
// composer's way); collapsed = a status capsule; "pop out" upgrades the same
|
||||
// content to a native Document PiP window (user gesture required by the
|
||||
// browser). Multi-session renders a meeting-style strip under the focus view.
|
||||
// Visuals follow the BrowserSkill product family: @browser-skill/ui components
|
||||
// and oklch tokens on a .bsk-obs scope root, so the card reads as BSK's own
|
||||
// surface without leaking styles into (or inheriting themes from) the shell.
|
||||
|
||||
import { cn } from "@browser-skill/ui";
|
||||
import {
|
||||
RiArrowDownSLine,
|
||||
RiErrorWarningLine,
|
||||
RiPictureInPicture2Line,
|
||||
RiPushpinFill,
|
||||
RiStopCircleLine,
|
||||
} from "@remixicon/react";
|
||||
|
||||
// remixicon's component types target @types/react 19 while the dsh shell
|
||||
// runs react 18 — a compile-time-only recast keeps the 18 typecheck happy.
|
||||
type IconComponent = (props: { size?: number | string; className?: string }) => ReactNode;
|
||||
const asIcon = (component: unknown): IconComponent => component as IconComponent;
|
||||
const IconStop = asIcon(RiStopCircleLine);
|
||||
const IconPip = asIcon(RiPictureInPicture2Line);
|
||||
const IconDown = asIcon(RiArrowDownSLine);
|
||||
const IconWarn = asIcon(RiErrorWarningLine);
|
||||
const IconPin = asIcon(RiPushpinFill);
|
||||
|
||||
import {
|
||||
type ReactNode,
|
||||
type PointerEvent as ReactPointerEvent,
|
||||
useCallback,
|
||||
useEffect,
|
||||
useRef,
|
||||
useState,
|
||||
useSyncExternalStore,
|
||||
} from "react";
|
||||
import { createPortal } from "react-dom";
|
||||
import type { SessionObservation } from "../observation";
|
||||
import css from "./ObservationOverlay.module.css";
|
||||
import type { ObservationClientStore } from "./observation-store";
|
||||
|
||||
/** Minimal Document PiP surface (TS lib.dom lacks it). */
|
||||
interface DocumentPip {
|
||||
requestWindow(options?: { width?: number; height?: number }): Promise<Window>;
|
||||
}
|
||||
|
||||
interface Point {
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
interface Size {
|
||||
w: number;
|
||||
h: number;
|
||||
}
|
||||
|
||||
const DEFAULT_SIZE: Size = { w: 320, h: 240 };
|
||||
const MIN_SIZE: Size = { w: 240, h: 180 };
|
||||
const EDGE_MARGIN = 16;
|
||||
/** Default dock: top-right, just under the shell's top bar (no spacing tokens exist in dsh 0.1). */
|
||||
const TOP_OFFSET = 64;
|
||||
|
||||
function pipApi(): DocumentPip | undefined {
|
||||
if (typeof window === "undefined") return undefined;
|
||||
return (window as unknown as { documentPictureInPicture?: DocumentPip }).documentPictureInPicture;
|
||||
}
|
||||
|
||||
function clampSize(size: Size, viewport: Size): Size {
|
||||
const maxW = viewport.w * 0.8;
|
||||
const maxH = viewport.h * 0.8;
|
||||
return {
|
||||
w: Math.min(Math.max(size.w, MIN_SIZE.w), maxW),
|
||||
h: Math.min(Math.max(size.h, MIN_SIZE.h), maxH),
|
||||
};
|
||||
}
|
||||
|
||||
function clampPos(pos: Point, size: Size, viewport: Size): Point {
|
||||
return {
|
||||
x: Math.min(Math.max(pos.x, 0), Math.max(0, viewport.w - size.w)),
|
||||
y: Math.min(Math.max(pos.y, 0), Math.max(0, viewport.h - size.h)),
|
||||
};
|
||||
}
|
||||
|
||||
export type ResizeCorner = "nw" | "ne" | "sw" | "se";
|
||||
|
||||
/** Grow/shrink from one corner, keeping the opposite corner planted. */
|
||||
export function applyResize(
|
||||
base: Point & Size,
|
||||
corner: ResizeCorner,
|
||||
dx: number,
|
||||
dy: number,
|
||||
viewport: Size,
|
||||
): { pos: Point; size: Size } {
|
||||
const nextW = corner === "ne" || corner === "se" ? base.w + dx : base.w - dx;
|
||||
const nextH = corner === "sw" || corner === "se" ? base.h + dy : base.h - dy;
|
||||
const size = clampSize({ w: nextW, h: nextH }, viewport);
|
||||
const x = corner === "nw" || corner === "sw" ? base.x + base.w - size.w : base.x;
|
||||
const y = corner === "nw" || corner === "ne" ? base.y + base.h - size.h : base.y;
|
||||
return { pos: clampPos({ x, y }, size, viewport), size };
|
||||
}
|
||||
|
||||
const CORNER_LABEL: Record<ResizeCorner, string> = {
|
||||
nw: "top left",
|
||||
ne: "top right",
|
||||
sw: "bottom left",
|
||||
se: "bottom right",
|
||||
};
|
||||
|
||||
function formatElapsed(sinceMs: number, nowMs: number): string {
|
||||
const total = Math.max(0, Math.floor((nowMs - sinceMs) / 1000));
|
||||
const mm = String(Math.floor(total / 60)).padStart(2, "0");
|
||||
const ss = String(total % 60).padStart(2, "0");
|
||||
return `${mm}:${ss}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Auto-follow focus: the most recently touched session, but never yank focus
|
||||
* to a session whose latest action failed (errors flag the strip item, they
|
||||
* do not steal the stage) or one already reported dead.
|
||||
*/
|
||||
export function focusOf(sessions: readonly SessionObservation[]): SessionObservation | undefined {
|
||||
if (sessions.length === 0) return undefined;
|
||||
const byRecency = [...sessions].sort((a, b) => b.since - a.since);
|
||||
const healthy = byRecency.find((s) => s.lastError === undefined && s.dead !== true);
|
||||
return healthy ?? byRecency[0];
|
||||
}
|
||||
|
||||
export function statusOf(obs: SessionObservation): "active" | "idle" | "error" {
|
||||
if (obs.lastError !== undefined && obs.action === "idle") return "error";
|
||||
return obs.action === "idle" ? "idle" : "active";
|
||||
}
|
||||
|
||||
/** Compact toolbar icon: no label, hover bubble for the name / semantics. */
|
||||
function IconAction(props: {
|
||||
label: string;
|
||||
hint: string;
|
||||
disabled?: boolean;
|
||||
danger?: boolean;
|
||||
align?: "start" | "end";
|
||||
onClick: () => void;
|
||||
children: ReactNode;
|
||||
}) {
|
||||
const [open, setOpen] = useState(false);
|
||||
return (
|
||||
<span
|
||||
className={css["tool-wrap"]}
|
||||
onPointerEnter={() => setOpen(true)}
|
||||
onPointerLeave={() => setOpen(false)}
|
||||
>
|
||||
<button
|
||||
type="button"
|
||||
className={cn(css["tool-button"], props.danger === true && css["tool-danger"])}
|
||||
disabled={props.disabled}
|
||||
aria-label={props.label}
|
||||
onClick={props.onClick}
|
||||
>
|
||||
{props.children}
|
||||
</button>
|
||||
{open ? (
|
||||
<span className={css.hint} data-align={props.align} role="tooltip">
|
||||
{props.hint}
|
||||
</span>
|
||||
) : null}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
/** Flat status dot, specced after the BSK popup's ConnectionStatusIndicator. */
|
||||
function StatusDot({ state }: { state: "active" | "idle" | "error" | "dead" }) {
|
||||
const color =
|
||||
state === "active"
|
||||
? "bg-emerald-500"
|
||||
: state === "error"
|
||||
? "bg-red-500"
|
||||
: state === "dead"
|
||||
? "bg-amber-500"
|
||||
: "bg-muted-foreground/40";
|
||||
return (
|
||||
<span
|
||||
className={cn("size-2 shrink-0 rounded-full ring-2 ring-background", color)}
|
||||
data-state={state}
|
||||
aria-hidden
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
/** One strip item: mini frame, id, status dot, hover interrupt, pin toggle. */
|
||||
function StripItem(props: {
|
||||
store: ObservationClientStore;
|
||||
obs: SessionObservation;
|
||||
pinned: boolean;
|
||||
focused: boolean;
|
||||
onTogglePin: (sessionId: string) => void;
|
||||
}) {
|
||||
const { store, obs, pinned, focused, onTogglePin } = props;
|
||||
const [hover, setHover] = useState(false);
|
||||
const thumbId = obs.thumbnailAttachmentId;
|
||||
useEffect(() => {
|
||||
store.ensureThumbnail(thumbId);
|
||||
}, [store, thumbId]);
|
||||
const thumb = store.getSnapshot().displayFrames[obs.sessionId];
|
||||
const state = obs.dead === true ? "dead" : statusOf(obs);
|
||||
return (
|
||||
<div
|
||||
className={css["strip-item"]}
|
||||
data-state={state}
|
||||
data-focused={focused || undefined}
|
||||
data-pinned={pinned || undefined}
|
||||
onPointerEnter={() => setHover(true)}
|
||||
onPointerLeave={() => setHover(false)}
|
||||
>
|
||||
<button
|
||||
type="button"
|
||||
className={css["strip-main"]}
|
||||
aria-label={`${pinned ? "Unpin" : "Pin"} session ${obs.sessionId}`}
|
||||
aria-pressed={pinned}
|
||||
onClick={() => onTogglePin(obs.sessionId)}
|
||||
>
|
||||
<span className={css["strip-thumb"]}>
|
||||
{thumb?.url !== undefined ? (
|
||||
<img src={thumb.url} alt="" />
|
||||
) : (
|
||||
<span className={css["strip-thumb-empty"]} />
|
||||
)}
|
||||
</span>
|
||||
<span className={css["strip-id"]}>{obs.sessionId}</span>
|
||||
<StatusDot state={state} />
|
||||
{pinned ? <IconPin size={9} className={css["pin-badge"]} aria-label="pinned" /> : null}
|
||||
</button>
|
||||
{hover && !pinned && obs.dead !== true ? (
|
||||
<button
|
||||
type="button"
|
||||
className={css["strip-interrupt"]}
|
||||
aria-label={`Interrupt session ${obs.sessionId}`}
|
||||
disabled={obs.action === "idle"}
|
||||
onClick={(event) => {
|
||||
event.stopPropagation();
|
||||
void store.interrupt(obs.sessionId);
|
||||
}}
|
||||
>
|
||||
<IconStop size={10} />
|
||||
</button>
|
||||
) : null}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/** The floating card / PiP shared content. */
|
||||
function OverlayBody(props: {
|
||||
store: ObservationClientStore;
|
||||
focus: SessionObservation | undefined;
|
||||
sessions: readonly SessionObservation[];
|
||||
available: boolean;
|
||||
pinnedId: string | null;
|
||||
onTogglePin: (sessionId: string) => void;
|
||||
now: number;
|
||||
onPopOut?: (() => void) | undefined;
|
||||
onCollapse?: (() => void) | undefined;
|
||||
inPip: boolean;
|
||||
onHeaderPointerDown?: (event: ReactPointerEvent<HTMLDivElement>) => void;
|
||||
}) {
|
||||
const {
|
||||
store,
|
||||
focus,
|
||||
sessions,
|
||||
available,
|
||||
pinnedId,
|
||||
onTogglePin,
|
||||
now,
|
||||
onPopOut,
|
||||
onCollapse,
|
||||
inPip,
|
||||
onHeaderPointerDown,
|
||||
} = props;
|
||||
const [interrupting, setInterrupting] = useState(false);
|
||||
|
||||
const thumbId = focus?.thumbnailAttachmentId;
|
||||
useEffect(() => {
|
||||
store.ensureThumbnail(thumbId);
|
||||
}, [store, thumbId]);
|
||||
// Paint the last good frame while the next attachment decodes — swapping
|
||||
// to a placeholder (and remounting <img> with a fade) is what made the
|
||||
// card flash on every breath / action-end capture.
|
||||
const thumb =
|
||||
focus !== undefined ? store.getSnapshot().displayFrames[focus.sessionId] : undefined;
|
||||
const displayUrl = thumb?.url;
|
||||
|
||||
const canInterrupt =
|
||||
available &&
|
||||
!interrupting &&
|
||||
focus !== undefined &&
|
||||
focus.action !== "idle" &&
|
||||
focus.dead !== true;
|
||||
const onInterrupt = (): void => {
|
||||
if (!canInterrupt || focus === undefined) return;
|
||||
setInterrupting(true);
|
||||
void store.interrupt(focus.sessionId).finally(() => setInterrupting(false));
|
||||
};
|
||||
|
||||
const statusText = !available
|
||||
? "browser unavailable"
|
||||
: focus === undefined
|
||||
? "no session"
|
||||
: `${focus.sessionId} · ${focus.action === "idle" ? "idle" : focus.action} · ${formatElapsed(focus.since, now)}`;
|
||||
const state = !available ? "error" : focus !== undefined ? statusOf(focus) : "idle";
|
||||
|
||||
return (
|
||||
<div className={cn(css.body, "bsk-obs")} data-state={state} data-in-pip={inPip || undefined}>
|
||||
<div
|
||||
className={css.header}
|
||||
data-testid="obs-header"
|
||||
onPointerDown={onHeaderPointerDown}
|
||||
role="presentation"
|
||||
>
|
||||
<StatusDot state={state === "error" ? "error" : state === "active" ? "active" : "idle"} />
|
||||
<span className={css["status-text"]}>{statusText}</span>
|
||||
{onCollapse !== undefined ? (
|
||||
<button
|
||||
type="button"
|
||||
className={css["icon-button"]}
|
||||
aria-label="Collapse"
|
||||
onClick={onCollapse}
|
||||
>
|
||||
<IconDown size={14} />
|
||||
</button>
|
||||
) : null}
|
||||
</div>
|
||||
<div className={css.stage}>
|
||||
{displayUrl !== undefined ? (
|
||||
<img
|
||||
className={css.thumb}
|
||||
src={displayUrl}
|
||||
alt={`session ${focus?.sessionId ?? ""} view`}
|
||||
/>
|
||||
) : (
|
||||
<div className={css.placeholder}>
|
||||
{!available
|
||||
? "last frame kept"
|
||||
: thumb?.status === "error"
|
||||
? "frame unavailable"
|
||||
: "waiting for page"}
|
||||
</div>
|
||||
)}
|
||||
{thumb?.status === "error" ? (
|
||||
<span className={css.badge} aria-label="thumbnail failed">
|
||||
<IconWarn size={12} />
|
||||
</span>
|
||||
) : null}
|
||||
</div>
|
||||
{sessions.length >= 2 ? (
|
||||
<div className={css.strip} data-testid="obs-strip" role="list">
|
||||
{sessions.map((obs) => (
|
||||
<StripItem
|
||||
key={obs.sessionId}
|
||||
store={store}
|
||||
obs={obs}
|
||||
pinned={pinnedId === obs.sessionId}
|
||||
focused={focus?.sessionId === obs.sessionId}
|
||||
onTogglePin={onTogglePin}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
) : null}
|
||||
<div className={css.actions}>
|
||||
<IconAction
|
||||
label={interrupting ? "Interrupting…" : "Interrupt the current browser action"}
|
||||
hint="Stop the current browser action."
|
||||
disabled={!canInterrupt}
|
||||
danger
|
||||
onClick={onInterrupt}
|
||||
>
|
||||
<IconStop size={16} />
|
||||
</IconAction>
|
||||
{onPopOut !== undefined ? (
|
||||
<IconAction
|
||||
label="Pop out into a mini window"
|
||||
hint="Pop out into a mini window"
|
||||
align="end"
|
||||
onClick={onPopOut}
|
||||
>
|
||||
<IconPip size={16} />
|
||||
</IconAction>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
/** Clone the host document's style/link nodes into a PiP window. */
|
||||
function cloneStylesInto(pipWindow: Window): void {
|
||||
for (const node of document.querySelectorAll('link[rel="stylesheet"], style')) {
|
||||
pipWindow.document.head.appendChild(node.cloneNode(true));
|
||||
}
|
||||
}
|
||||
|
||||
export function ObservationOverlay({ store }: { store: ObservationClientStore }) {
|
||||
const snapshot = useSyncExternalStore(store.subscribe, store.getSnapshot);
|
||||
useEffect(() => {
|
||||
store.start();
|
||||
return () => store.stop();
|
||||
}, [store]);
|
||||
|
||||
const [collapsed, setCollapsed] = useState(false);
|
||||
const [pos, setPos] = useState<Point | null>(null);
|
||||
const [size, setSize] = useState<Size>(DEFAULT_SIZE);
|
||||
const [pipWindow, setPipWindow] = useState<Window | null>(null);
|
||||
const [pinnedId, setPinnedId] = useState<string | null>(null);
|
||||
const dragRef = useRef<{
|
||||
kind: "move" | "resize";
|
||||
corner?: ResizeCorner;
|
||||
startX: number;
|
||||
startY: number;
|
||||
base: Point & Size;
|
||||
} | null>(null);
|
||||
|
||||
// Elapsed-time ticker: 1s while anything is active.
|
||||
const anyActive = snapshot.sessions.some((s) => s.action !== "idle");
|
||||
const [now, setNow] = useState(() => Date.now());
|
||||
useEffect(() => {
|
||||
if (!anyActive) return;
|
||||
const timer = setInterval(() => setNow(Date.now()), 1000);
|
||||
return () => clearInterval(timer);
|
||||
}, [anyActive]);
|
||||
|
||||
// Focus: the pinned session wins while it still exists; otherwise auto-follow.
|
||||
const pinned =
|
||||
pinnedId !== null ? snapshot.sessions.find((s) => s.sessionId === pinnedId) : undefined;
|
||||
const focus = pinned ?? focusOf(snapshot.sessions);
|
||||
const onTogglePin = useCallback((sessionId: string) => {
|
||||
setPinnedId((current) => (current === sessionId ? null : sessionId));
|
||||
}, []);
|
||||
|
||||
const viewport = (): Size => ({ w: window.innerWidth, h: window.innerHeight });
|
||||
|
||||
const onPointerMove = useCallback((event: PointerEvent) => {
|
||||
const drag = dragRef.current;
|
||||
if (drag === null) return;
|
||||
const dx = event.clientX - drag.startX;
|
||||
const dy = event.clientY - drag.startY;
|
||||
if (drag.kind === "move") {
|
||||
setPos(
|
||||
clampPos(
|
||||
{ x: drag.base.x + dx, y: drag.base.y + dy },
|
||||
{ w: drag.base.w, h: drag.base.h },
|
||||
viewport(),
|
||||
),
|
||||
);
|
||||
} else if (drag.corner !== undefined) {
|
||||
const next = applyResize(drag.base, drag.corner, dx, dy, viewport());
|
||||
setPos(next.pos);
|
||||
setSize(next.size);
|
||||
}
|
||||
}, []);
|
||||
const onPointerUp = useCallback(() => {
|
||||
dragRef.current = null;
|
||||
document.removeEventListener("pointermove", onPointerMove);
|
||||
document.removeEventListener("pointerup", onPointerUp);
|
||||
}, [onPointerMove]);
|
||||
|
||||
const cardOrigin = (): Point & Size => {
|
||||
const vp = viewport();
|
||||
return {
|
||||
x: pos?.x ?? vp.w - size.w - EDGE_MARGIN,
|
||||
y: pos?.y ?? TOP_OFFSET,
|
||||
w: size.w,
|
||||
h: size.h,
|
||||
};
|
||||
};
|
||||
|
||||
const beginMove = (event: ReactPointerEvent) => {
|
||||
event.preventDefault();
|
||||
const rect = (
|
||||
event.currentTarget.closest("[data-obs-card]") as HTMLElement | null
|
||||
)?.getBoundingClientRect();
|
||||
const origin = cardOrigin();
|
||||
const base = {
|
||||
x: rect?.left ?? origin.x,
|
||||
y: rect?.top ?? origin.y,
|
||||
w: rect !== undefined && rect.width > 0 ? rect.width : origin.w,
|
||||
h: rect !== undefined && rect.height > 0 ? rect.height : origin.h,
|
||||
};
|
||||
dragRef.current = { kind: "move", startX: event.clientX, startY: event.clientY, base };
|
||||
setPos({ x: base.x, y: base.y });
|
||||
document.addEventListener("pointermove", onPointerMove);
|
||||
document.addEventListener("pointerup", onPointerUp);
|
||||
};
|
||||
|
||||
const beginResize = (corner: ResizeCorner) => (event: ReactPointerEvent) => {
|
||||
event.preventDefault();
|
||||
event.stopPropagation();
|
||||
const rect = (
|
||||
event.currentTarget.closest("[data-obs-card]") as HTMLElement | null
|
||||
)?.getBoundingClientRect();
|
||||
const origin = cardOrigin();
|
||||
const base = {
|
||||
x: rect?.left ?? origin.x,
|
||||
y: rect?.top ?? origin.y,
|
||||
w: rect !== undefined && rect.width > 0 ? rect.width : origin.w,
|
||||
h: rect !== undefined && rect.height > 0 ? rect.height : origin.h,
|
||||
};
|
||||
dragRef.current = {
|
||||
kind: "resize",
|
||||
corner,
|
||||
startX: event.clientX,
|
||||
startY: event.clientY,
|
||||
base,
|
||||
};
|
||||
setPos({ x: base.x, y: base.y });
|
||||
document.addEventListener("pointermove", onPointerMove);
|
||||
document.addEventListener("pointerup", onPointerUp);
|
||||
};
|
||||
|
||||
const popOut = async (): Promise<void> => {
|
||||
const pip = pipApi();
|
||||
if (pip === undefined) return;
|
||||
try {
|
||||
const win = await pip.requestWindow({ width: size.w, height: size.h });
|
||||
cloneStylesInto(win);
|
||||
win.addEventListener("pagehide", () => setPipWindow(null));
|
||||
setPipWindow(win);
|
||||
} catch {
|
||||
// requestWindow rejects without a user gesture (or when the window was
|
||||
// closed mid-request): stay on the in-page card, no state change.
|
||||
}
|
||||
};
|
||||
|
||||
// Hidden while no owned session exists (and no PiP is up).
|
||||
if (snapshot.sessions.length === 0 && pipWindow === null) return null;
|
||||
|
||||
const body = (
|
||||
<OverlayBody
|
||||
store={store}
|
||||
focus={focus}
|
||||
sessions={snapshot.sessions}
|
||||
available={snapshot.available}
|
||||
pinnedId={pinned !== undefined ? pinnedId : null}
|
||||
onTogglePin={onTogglePin}
|
||||
now={now}
|
||||
inPip={pipWindow !== null}
|
||||
onPopOut={pipWindow === null && pipApi() !== undefined ? () => void popOut() : undefined}
|
||||
onCollapse={pipWindow === null ? () => setCollapsed(true) : undefined}
|
||||
onHeaderPointerDown={pipWindow === null ? beginMove : undefined}
|
||||
/>
|
||||
);
|
||||
|
||||
if (pipWindow !== null) {
|
||||
return createPortal(body, pipWindow.document.body);
|
||||
}
|
||||
|
||||
if (collapsed) {
|
||||
const state = focus !== undefined ? statusOf(focus) : "idle";
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
className={cn(css.capsule, "bsk-obs")}
|
||||
data-state={state}
|
||||
aria-label="Expand browser observation overlay"
|
||||
onClick={() => setCollapsed(false)}
|
||||
>
|
||||
<StatusDot state={state} />
|
||||
<span className={css["capsule-text"]}>
|
||||
{snapshot.sessions.length} session{snapshot.sessions.length === 1 ? "" : "s"}
|
||||
{focus !== undefined && focus.action !== "idle"
|
||||
? ` · ${focus.action} · ${formatElapsed(focus.since, now)}`
|
||||
: ""}
|
||||
</span>
|
||||
</button>
|
||||
);
|
||||
}
|
||||
|
||||
const style: React.CSSProperties =
|
||||
pos !== null
|
||||
? { left: pos.x, top: pos.y, width: size.w, height: size.h }
|
||||
: { right: EDGE_MARGIN, top: TOP_OFFSET, width: size.w, height: size.h };
|
||||
return (
|
||||
<div className={cn(css.card, "bsk-obs")} style={style} data-obs-card data-testid="obs-card">
|
||||
{body}
|
||||
{(["nw", "ne", "sw", "se"] as const).map((corner) => (
|
||||
<div
|
||||
key={corner}
|
||||
className={css["resize-handle"]}
|
||||
data-corner={corner}
|
||||
data-testid={`obs-resize-${corner}`}
|
||||
aria-label={`Resize overlay from the ${CORNER_LABEL[corner]}`}
|
||||
role="separator"
|
||||
aria-valuenow={size.w}
|
||||
aria-valuetext={`${Math.round(size.w)} by ${Math.round(size.h)} pixels`}
|
||||
aria-valuemin={MIN_SIZE.w}
|
||||
aria-valuemax={Math.round(viewport().w * 0.8)}
|
||||
tabIndex={corner === "se" ? 0 : -1}
|
||||
onPointerDown={beginResize(corner)}
|
||||
onKeyDown={
|
||||
corner === "se"
|
||||
? (event) => {
|
||||
const step = 16;
|
||||
const dx =
|
||||
event.key === "ArrowRight" ? step : event.key === "ArrowLeft" ? -step : 0;
|
||||
const dy = event.key === "ArrowDown" ? step : event.key === "ArrowUp" ? -step : 0;
|
||||
if (dx === 0 && dy === 0) return;
|
||||
event.preventDefault();
|
||||
const next = applyResize(cardOrigin(), "se", dx, dy, viewport());
|
||||
setPos(next.pos);
|
||||
setSize(next.size);
|
||||
}
|
||||
: undefined
|
||||
}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
/* browser_screenshot toolview: disclosure row over a terminal block plus the
|
||||
screenshot image. Semantic tokens only (--dsw-alias-*), no literal colors. */
|
||||
|
||||
.card {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
}
|
||||
|
||||
.summary {
|
||||
min-width: 0;
|
||||
margin-left: 6px;
|
||||
overflow: hidden;
|
||||
text-overflow: ellipsis;
|
||||
white-space: nowrap;
|
||||
color: var(--dsw-alias-label-secondary);
|
||||
}
|
||||
|
||||
.card[data-state="error"] .summary {
|
||||
color: var(--dsw-alias-label-error);
|
||||
}
|
||||
|
||||
.body {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 8px;
|
||||
padding: 4px 0 4px 22px;
|
||||
}
|
||||
|
||||
.image-wrap {
|
||||
display: flex;
|
||||
}
|
||||
|
||||
.inspect-button {
|
||||
align-self: flex-start;
|
||||
border: none;
|
||||
background: none;
|
||||
padding: 0;
|
||||
font: inherit;
|
||||
font-size: 12px;
|
||||
color: var(--dsw-alias-label-tertiary);
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.inspect-button:hover {
|
||||
color: var(--dsw-alias-label-secondary);
|
||||
}
|
||||
|
||||
.inspect-button:focus-visible {
|
||||
outline: 2px solid var(--dsw-alias-border-focus);
|
||||
outline-offset: 2px;
|
||||
border-radius: 2px;
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.inspect-button {
|
||||
transition: none;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
// browser_screenshot keyed toolview: the terminal card's command line and
|
||||
// output stay exactly as the host renders them, and a result carrying an
|
||||
// image block additionally renders the screenshot itself (thumbnail with the
|
||||
// shared click-to-open lightbox). Pure function of the frozen call/result
|
||||
// slice — live and replay paths render identically.
|
||||
|
||||
import type { ImageAttachmentRef } from "@deepseek-ai/dsh-attachment";
|
||||
import {
|
||||
type ImageLoader,
|
||||
MessageImage,
|
||||
type MessageImageLabels,
|
||||
} from "@deepseek-ai/dsh-client-ui-attachment";
|
||||
import {
|
||||
DisclosureRow,
|
||||
StateDot,
|
||||
type StateDotState,
|
||||
TerminalBlock,
|
||||
type TerminalBlockLabels,
|
||||
} from "@deepseek-ai/dsh-client-ui-primitives";
|
||||
import type { ToolCallViewProps } from "@deepseek-ai/dsh-client-ui-tool/client";
|
||||
import { useState } from "react";
|
||||
import css from "./ScreenshotToolView.module.css";
|
||||
|
||||
/** Resolved loader bound to the owning session at registration time. */
|
||||
export type ScreenshotImageLoader = (attachment: ImageAttachmentRef) => Promise<string>;
|
||||
|
||||
export type ScreenshotToolViewProps = ToolCallViewProps & {
|
||||
loadImage: ScreenshotImageLoader;
|
||||
};
|
||||
|
||||
type ViewState = "running" | "ok" | "error";
|
||||
|
||||
interface ViewModel {
|
||||
readonly state: ViewState;
|
||||
readonly command: string;
|
||||
readonly output: string | null;
|
||||
readonly image: ImageAttachmentRef | null;
|
||||
readonly summary: string;
|
||||
}
|
||||
|
||||
const TERMINAL_LABELS: Partial<TerminalBlockLabels> = {
|
||||
running: "Running",
|
||||
failed: "Failed",
|
||||
done: "Done",
|
||||
copy: "Copy",
|
||||
copied: "Copied",
|
||||
noOutput: "No output",
|
||||
collapseAria: "Collapse output",
|
||||
collapse: "Collapse",
|
||||
expandAria: (hidden) => `Expand the remaining ${hidden} output lines`,
|
||||
expand: (hidden) => `… ${hidden} more lines`,
|
||||
};
|
||||
|
||||
const IMAGE_LABELS: MessageImageLabels = {
|
||||
image: "screenshot",
|
||||
open: "Open the original screenshot",
|
||||
openNamed: (label) => `Open screenshot ${label}`,
|
||||
loading: "Loading…",
|
||||
loadFailed: "Load failed — retry",
|
||||
lightbox: { dialog: "Screenshot preview", close: "Close preview" },
|
||||
};
|
||||
|
||||
function firstLine(text: string): string {
|
||||
const newline = text.indexOf("\n");
|
||||
return newline === -1 ? text : text.slice(0, newline);
|
||||
}
|
||||
|
||||
/** Rebuild the command line from the logged arguments (mirrors the host presenter). */
|
||||
function commandOf(argsRaw: string, callId: string): string {
|
||||
try {
|
||||
const args = JSON.parse(argsRaw) as { session?: unknown; ref?: unknown };
|
||||
const parts = ["bsk", "screenshot", "--session"];
|
||||
parts.push(
|
||||
typeof args.session === "string" && args.session !== "" ? args.session : "(current)",
|
||||
);
|
||||
if (typeof args.ref === "string" && args.ref !== "") parts.push("--ref", args.ref);
|
||||
return parts.join(" ");
|
||||
} catch {
|
||||
// Streaming can expose a truncated JSON prefix; it is still the best label.
|
||||
return argsRaw === "" ? `bsk screenshot (${callId})` : `bsk screenshot ${firstLine(argsRaw)}`;
|
||||
}
|
||||
}
|
||||
|
||||
/** Derive the display model from the frozen block only. */
|
||||
export function viewModelOf(block: ToolCallViewProps["block"]): ViewModel {
|
||||
const settled = "kind" in block;
|
||||
const argsRaw = (settled ? block.call?.argsRaw : block.argsRaw) ?? "";
|
||||
const command = commandOf(argsRaw, block.callId);
|
||||
if (!settled) {
|
||||
return { state: "running", command, output: null, image: null, summary: command };
|
||||
}
|
||||
const textBlock = block.content.find((item) => item.type === "text");
|
||||
const imageBlock = block.content.find((item) => item.type === "image");
|
||||
const output = textBlock !== undefined && textBlock.type === "text" ? textBlock.text : null;
|
||||
const image =
|
||||
imageBlock !== undefined && imageBlock.type === "image" ? imageBlock.attachment : null;
|
||||
const state: ViewState = block.isError ? "error" : "ok";
|
||||
const summary = output !== null ? firstLine(output) : command;
|
||||
return { state, command, output, image, summary };
|
||||
}
|
||||
|
||||
function dotState(state: ViewState): StateDotState {
|
||||
switch (state) {
|
||||
case "running":
|
||||
return "ongoing";
|
||||
case "error":
|
||||
return "error";
|
||||
default:
|
||||
return "done";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Render one browser_screenshot call: a disclosure row over a terminal block
|
||||
* (command + output) plus the screenshot image when the result carries one.
|
||||
*/
|
||||
export function ScreenshotToolView({ block, cwd, inspect, loadImage }: ScreenshotToolViewProps) {
|
||||
const model = viewModelOf(block);
|
||||
const [expanded, setExpanded] = useState(false);
|
||||
const expandable = model.state === "running" || model.output !== null || model.image !== null;
|
||||
const open = expanded && expandable;
|
||||
return (
|
||||
<div className={css.card} data-tool="browser_screenshot" data-state={model.state}>
|
||||
<DisclosureRow
|
||||
icon={<StateDot state={dotState(model.state)} />}
|
||||
title="Screenshot"
|
||||
open={open}
|
||||
expandable={expandable}
|
||||
onToggle={() => setExpanded((value) => !value)}
|
||||
expandOnRowClick
|
||||
previewChevron
|
||||
collapsedContent={<span className={css.summary}>{model.summary}</span>}
|
||||
>
|
||||
<div className={css.body}>
|
||||
<TerminalBlock
|
||||
command={model.command}
|
||||
cwd={cwd ?? undefined}
|
||||
output={model.output ?? undefined}
|
||||
running={model.state === "running"}
|
||||
labels={TERMINAL_LABELS}
|
||||
/>
|
||||
{model.image !== null ? (
|
||||
<div className={css["image-wrap"]}>
|
||||
<MessageImage
|
||||
attachment={model.image}
|
||||
load={loadImage}
|
||||
variant="single"
|
||||
labels={IMAGE_LABELS}
|
||||
/>
|
||||
</div>
|
||||
) : null}
|
||||
{inspect !== undefined ? (
|
||||
<button type="button" className={css["inspect-button"]} onClick={inspect}>
|
||||
Inspect
|
||||
</button>
|
||||
) : null}
|
||||
</div>
|
||||
</DisclosureRow>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
/*
|
||||
* BrowserSkill UI design tokens, scoped to the observation overlay root.
|
||||
* VALUES MIRROR packages/ui/src/styles/tailwind.css (:root and .dark) — keep
|
||||
* in sync with that file; it is the canonical source. Scoped here (instead of
|
||||
* :root) so the dsh host theme can neither leak in nor be polluted by us.
|
||||
* Dark follows prefers-color-scheme, same as the extension popup.
|
||||
*/
|
||||
.bsk-obs {
|
||||
color-scheme: light dark;
|
||||
--background: oklch(0.99 0.005 50);
|
||||
--foreground: oklch(0.14 0.03 50);
|
||||
--card: oklch(0.995 0.003 50 / 0.82);
|
||||
--card-foreground: oklch(0.14 0.03 50);
|
||||
--primary: oklch(0.62 0.22 45);
|
||||
--primary-foreground: oklch(0.99 0.01 50);
|
||||
--secondary: oklch(0.96 0.04 65);
|
||||
--secondary-foreground: oklch(0.25 0.08 65);
|
||||
--muted: oklch(0.96 0.01 50);
|
||||
--muted-foreground: oklch(0.55 0.04 50);
|
||||
--accent: oklch(0.96 0.04 65);
|
||||
--accent-foreground: oklch(0.25 0.08 65);
|
||||
--destructive: oklch(0.6 0.25 25);
|
||||
--destructive-foreground: oklch(0.99 0.01 50);
|
||||
--border: oklch(0.9 0.025 60);
|
||||
--input: oklch(0.9 0.025 60);
|
||||
--ring: oklch(0.62 0.22 45);
|
||||
--radius: 0.75rem;
|
||||
}
|
||||
@media (prefers-color-scheme: dark) {
|
||||
.bsk-obs {
|
||||
--background: oklch(0.2 0.02 50);
|
||||
--foreground: oklch(0.98 0.01 50);
|
||||
--card: oklch(0.24 0.02 50 / 0.82);
|
||||
--card-foreground: oklch(0.98 0.01 50);
|
||||
--primary: oklch(0.7 0.18 45);
|
||||
--primary-foreground: oklch(0.14 0.03 50);
|
||||
--secondary: oklch(0.32 0.03 60);
|
||||
--secondary-foreground: oklch(0.98 0.01 50);
|
||||
--muted: oklch(0.3 0.02 50);
|
||||
--muted-foreground: oklch(0.75 0.03 50);
|
||||
--accent: oklch(0.32 0.03 60);
|
||||
--accent-foreground: oklch(0.98 0.01 50);
|
||||
--destructive: oklch(0.45 0.2 25);
|
||||
--destructive-foreground: oklch(0.98 0.01 50);
|
||||
--border: oklch(0.35 0.02 60);
|
||||
--input: oklch(0.35 0.02 60);
|
||||
--ring: oklch(0.7 0.18 45);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
/**
|
||||
* dsh-plugin-browserskill browser half: the `browser_screenshot` keyed
|
||||
* toolview plus the observation overlay (live thumbnails + interrupt) floating
|
||||
* over the shell via the `shell.overlay` seat.
|
||||
*/
|
||||
|
||||
import type { ImageAttachmentRef } from "@deepseek-ai/dsh-attachment";
|
||||
import type { ClientContext, ISessions, SessionId } from "@deepseek-ai/dsh-client-runtime/client";
|
||||
// Type-only: pulls the 'shell.overlay' SlotMap merge into scope.
|
||||
import type {} from "@deepseek-ai/dsh-client-ui-layout/client";
|
||||
import type { ToolCallViewProps } from "@deepseek-ai/dsh-client-ui-tool/client";
|
||||
import { createElement } from "react";
|
||||
// Scope-prefixed BSK design tokens and utility sheet (injected verbatim as
|
||||
// <style> tags; selectors stay unhashed so `cn(..., "bsk-obs")` roots match).
|
||||
import "./bsk-tokens.nomodule.css";
|
||||
import "./bsk-ui.nomodule.css";
|
||||
import { ObservationOverlay } from "./ObservationOverlay";
|
||||
import { type EventSourceLike, ObservationClientStore } from "./observation-store";
|
||||
import { ScreenshotToolView } from "./ScreenshotToolView";
|
||||
|
||||
/** Required services: slots, session-scoped attachment reads, and the overlay seat. */
|
||||
export const inject = ["slots", "sessions"];
|
||||
|
||||
/** Resolve one durable image attachment into a browser blob URL. */
|
||||
async function loadSessionImage(
|
||||
sessions: ISessions,
|
||||
sessionId: SessionId,
|
||||
attachment: ImageAttachmentRef,
|
||||
): Promise<string> {
|
||||
const session = sessions.binding(sessionId)?.session;
|
||||
if (session === undefined) {
|
||||
throw new Error(`screenshot toolview: session "${String(sessionId)}" is not bound`);
|
||||
}
|
||||
const result = await session.readAttachment(attachment.attachmentId);
|
||||
if (!result.ok) {
|
||||
throw new Error(
|
||||
`screenshot toolview: readAttachment failed: ${result.error.code}: ${result.error.message}`,
|
||||
);
|
||||
}
|
||||
const bytes = Uint8Array.from(result.value.data);
|
||||
return URL.createObjectURL(
|
||||
new Blob([bytes.buffer as ArrayBuffer], { type: result.value.attachment.mediaType }),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Thumbnail loader for the overlay: frames are plugin-owned runtime data
|
||||
* (never referenced by a session log, so the session-authorized RPC refuses
|
||||
* them), served by the host over the plugin's own route.
|
||||
*/
|
||||
async function overlayImageLoader(attachmentId: string): Promise<string> {
|
||||
const res = await fetch(`/bsk-observation/thumbnail/${encodeURIComponent(attachmentId)}`);
|
||||
if (!res.ok) throw new Error(`thumbnail fetch failed: ${res.status}`);
|
||||
return URL.createObjectURL(await res.blob());
|
||||
}
|
||||
|
||||
/**
|
||||
* Client plugin body: register the keyed toolview and the observation overlay.
|
||||
* @param ctx - client root context.
|
||||
*/
|
||||
export function apply(ctx: ClientContext): void {
|
||||
const sessions = ctx.get("sessions") as unknown as ISessions;
|
||||
ctx.slots.inject("tool.call.toolview", () =>
|
||||
ctx.slots.register(
|
||||
{ name: "tool.call.toolview", key: "browser_screenshot" },
|
||||
(props: ToolCallViewProps) =>
|
||||
createElement(ScreenshotToolView, {
|
||||
...props,
|
||||
loadImage: (attachment: ImageAttachmentRef) =>
|
||||
loadSessionImage(sessions, props.sessionId, attachment),
|
||||
}),
|
||||
),
|
||||
);
|
||||
const store = new ObservationClientStore({
|
||||
fetchFn: (url, init) => fetch(url, init),
|
||||
eventSourceFactory: (url) => new EventSource(url) as unknown as EventSourceLike,
|
||||
loadImage: overlayImageLoader,
|
||||
});
|
||||
ctx.slots.inject("shell.overlay", () =>
|
||||
ctx.slots.register({ name: "shell.overlay", id: "bsk-observation" }, () =>
|
||||
createElement(ObservationOverlay, { store }),
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,295 @@
|
||||
/**
|
||||
* Client-side data layer for the observation overlay: initial state fetch,
|
||||
* SSE increment stream (with a state refetch on every (re)open — events that
|
||||
* fired while the stream was down are otherwise lost forever), on-demand
|
||||
* thumbnail blob loading through the session-authorized readAttachment path,
|
||||
* and the interrupt call. The last ready blob per session is held until its
|
||||
* replacement decodes so the overlay can swap frames in place. All I/O is
|
||||
* injected so tests never touch a network.
|
||||
*/
|
||||
|
||||
import type { ObservationEvent, SessionObservation } from "../observation";
|
||||
|
||||
export interface EventSourceLike {
|
||||
onmessage: ((event: { data: string }) => void) | null;
|
||||
/** Fires on the initial connect AND on every automatic reconnect. */
|
||||
onopen?: (() => void) | null;
|
||||
close(): void;
|
||||
}
|
||||
|
||||
export interface ObservationClientDeps {
|
||||
fetchFn: (
|
||||
url: string,
|
||||
init?: { method?: string; body?: string; headers?: Record<string, string> },
|
||||
) => Promise<{
|
||||
ok: boolean;
|
||||
json(): Promise<unknown>;
|
||||
}>;
|
||||
eventSourceFactory: (url: string) => EventSourceLike;
|
||||
/** Resolve one attachment id to a displayable blob URL. */
|
||||
loadImage: (attachmentId: string) => Promise<string>;
|
||||
}
|
||||
|
||||
export interface ThumbnailState {
|
||||
status: "loading" | "ready" | "error";
|
||||
url?: string;
|
||||
}
|
||||
|
||||
export interface OverlaySnapshot {
|
||||
readonly sessions: readonly SessionObservation[];
|
||||
/** Whether the SSE increment stream is currently subscribed. */
|
||||
readonly subscribed: boolean;
|
||||
readonly thumbnails: Readonly<Record<string, ThumbnailState>>;
|
||||
/**
|
||||
* sessionId → the frame the overlay should paint. While a newer attachment
|
||||
* is still loading (or failed), this keeps the last good URL so the card
|
||||
* never drops to a placeholder between breaths.
|
||||
*/
|
||||
readonly displayFrames: Readonly<Record<string, ThumbnailState>>;
|
||||
/** False when the host reports the browser/daemon as unreachable. */
|
||||
readonly available: boolean;
|
||||
}
|
||||
|
||||
const STATE_URL = "/bsk-observation/state";
|
||||
const EVENTS_URL = "/bsk-observation/events";
|
||||
const INTERRUPT_URL = "/bsk-observation/interrupt";
|
||||
|
||||
function revoke(url: string | undefined): void {
|
||||
if (url !== undefined && typeof URL.revokeObjectURL === "function") {
|
||||
URL.revokeObjectURL(url);
|
||||
}
|
||||
}
|
||||
|
||||
export class ObservationClientStore {
|
||||
private sessions = new Map<string, SessionObservation>();
|
||||
private thumbs = new Map<string, ThumbnailState>();
|
||||
/** sessionId → the attachment id currently advertised for it. */
|
||||
private readonly thumbBySession = new Map<string, string>();
|
||||
/** sessionId → last successfully decoded frame (held across the next load). */
|
||||
private readonly lastReady = new Map<string, { attachmentId: string; url: string }>();
|
||||
private listeners = new Set<() => void>();
|
||||
private events: EventSourceLike | undefined;
|
||||
private snapshot: OverlaySnapshot = {
|
||||
sessions: [],
|
||||
subscribed: false,
|
||||
thumbnails: {},
|
||||
displayFrames: {},
|
||||
available: true,
|
||||
};
|
||||
private available = true;
|
||||
private started = false;
|
||||
|
||||
constructor(private readonly deps: ObservationClientDeps) {}
|
||||
|
||||
subscribe = (listener: () => void): (() => void) => {
|
||||
this.listeners.add(listener);
|
||||
return () => this.listeners.delete(listener);
|
||||
};
|
||||
|
||||
getSnapshot = (): OverlaySnapshot => this.snapshot;
|
||||
|
||||
private publish(): void {
|
||||
this.snapshot = {
|
||||
sessions: [...this.sessions.values()],
|
||||
subscribed: this.events !== undefined,
|
||||
thumbnails: Object.fromEntries(this.thumbs),
|
||||
displayFrames: this.buildDisplayFrames(),
|
||||
available: this.available,
|
||||
};
|
||||
for (const listener of [...this.listeners]) listener();
|
||||
}
|
||||
|
||||
/** Frame the overlay should paint for one session (last good while next loads). */
|
||||
private frameFor(sessionId: string): ThumbnailState | undefined {
|
||||
const attachmentId = this.thumbBySession.get(sessionId);
|
||||
const current = attachmentId !== undefined ? this.thumbs.get(attachmentId) : undefined;
|
||||
if (current?.status === "ready" && current.url !== undefined) return current;
|
||||
const held = this.lastReady.get(sessionId);
|
||||
if (held !== undefined) {
|
||||
return {
|
||||
status: current?.status === "error" ? "error" : "ready",
|
||||
url: held.url,
|
||||
};
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
private buildDisplayFrames(): Record<string, ThumbnailState> {
|
||||
const frames: Record<string, ThumbnailState> = {};
|
||||
for (const sessionId of this.sessions.keys()) {
|
||||
const frame = this.frameFor(sessionId);
|
||||
if (frame !== undefined) frames[sessionId] = frame;
|
||||
}
|
||||
return frames;
|
||||
}
|
||||
|
||||
private sessionOf(attachmentId: string): string | undefined {
|
||||
for (const [sessionId, id] of this.thumbBySession) {
|
||||
if (id === attachmentId) return sessionId;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** (Re)pull the full state: initial load and every SSE (re)open. */
|
||||
private refreshState(): void {
|
||||
void this.deps
|
||||
.fetchFn(STATE_URL)
|
||||
.then(async (res) => {
|
||||
if (!res.ok) return;
|
||||
const body = (await res.json()) as {
|
||||
sessions?: SessionObservation[];
|
||||
available?: boolean;
|
||||
};
|
||||
const sessions = body.sessions ?? [];
|
||||
this.sessions = new Map(sessions.map((s) => [s.sessionId, s]));
|
||||
// Prune thumbnails of sessions that vanished while we were away.
|
||||
const alive = new Set(sessions.map((s) => s.sessionId));
|
||||
for (const sessionId of [...this.thumbBySession.keys()]) {
|
||||
if (!alive.has(sessionId)) this.dropThumb(sessionId);
|
||||
}
|
||||
for (const s of sessions) this.trackThumb(s.sessionId, s.thumbnailAttachmentId);
|
||||
if (typeof body.available === "boolean") this.available = body.available;
|
||||
this.publish();
|
||||
})
|
||||
.catch(() => {});
|
||||
}
|
||||
|
||||
/** Initial fetch + SSE subscription. Idempotent. */
|
||||
start(): void {
|
||||
if (this.started) return;
|
||||
this.started = true;
|
||||
this.refreshState();
|
||||
const events = this.deps.eventSourceFactory(EVENTS_URL);
|
||||
events.onmessage = (message) => {
|
||||
let event: ObservationEvent;
|
||||
try {
|
||||
event = JSON.parse(message.data) as ObservationEvent;
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
this.apply(event);
|
||||
};
|
||||
events.onopen = () => {
|
||||
// Reconnects lose every event fired during the outage — resync.
|
||||
if (this.events === events) this.refreshState();
|
||||
};
|
||||
this.events = events;
|
||||
this.publish();
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.events?.close();
|
||||
this.events = undefined;
|
||||
this.started = false;
|
||||
for (const thumb of this.thumbs.values()) revoke(thumb.url);
|
||||
this.thumbs.clear();
|
||||
this.thumbBySession.clear();
|
||||
this.lastReady.clear();
|
||||
this.sessions.clear();
|
||||
this.publish();
|
||||
}
|
||||
|
||||
/** Forget one session's tracked + held frames, revoking their blob URLs. */
|
||||
private dropThumb(sessionId: string): void {
|
||||
const attachmentId = this.thumbBySession.get(sessionId);
|
||||
const held = this.lastReady.get(sessionId);
|
||||
this.thumbBySession.delete(sessionId);
|
||||
this.lastReady.delete(sessionId);
|
||||
if (attachmentId !== undefined) {
|
||||
revoke(this.thumbs.get(attachmentId)?.url);
|
||||
this.thumbs.delete(attachmentId);
|
||||
}
|
||||
if (held !== undefined && held.attachmentId !== attachmentId) {
|
||||
revoke(held.url);
|
||||
this.thumbs.delete(held.attachmentId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Track the frame a session currently advertises. The last *ready* blob is
|
||||
* kept until the replacement decodes — dropping it on the upsert is what
|
||||
* made the overlay flash a placeholder between breaths.
|
||||
*/
|
||||
private trackThumb(sessionId: string, attachmentId: string | undefined): void {
|
||||
const previous = this.thumbBySession.get(sessionId);
|
||||
if (previous === attachmentId) return;
|
||||
if (attachmentId !== undefined) this.thumbBySession.set(sessionId, attachmentId);
|
||||
else this.thumbBySession.delete(sessionId);
|
||||
const heldId = this.lastReady.get(sessionId)?.attachmentId;
|
||||
if (previous !== undefined && previous !== heldId) {
|
||||
revoke(this.thumbs.get(previous)?.url);
|
||||
this.thumbs.delete(previous);
|
||||
}
|
||||
}
|
||||
|
||||
private apply(event: ObservationEvent): void {
|
||||
if (event.type === "reset") {
|
||||
this.sessions.clear();
|
||||
for (const thumb of this.thumbs.values()) revoke(thumb.url);
|
||||
this.thumbs.clear();
|
||||
this.thumbBySession.clear();
|
||||
this.lastReady.clear();
|
||||
} else if (event.type === "remove" && event.session !== undefined) {
|
||||
this.sessions.delete(event.session.sessionId);
|
||||
this.dropThumb(event.session.sessionId);
|
||||
} else if (event.type === "upsert" && event.session !== undefined) {
|
||||
this.sessions.set(event.session.sessionId, event.session);
|
||||
this.trackThumb(event.session.sessionId, event.session.thumbnailAttachmentId);
|
||||
} else if (event.type === "availability") {
|
||||
this.available = event.available;
|
||||
}
|
||||
this.publish();
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure a thumbnail load is in flight for one attachment reference. New
|
||||
* frames replace the old URL only after they decode; failures keep the last
|
||||
* good frame (the caller renders `status: 'error'` as a small badge over
|
||||
* the old image).
|
||||
*/
|
||||
ensureThumbnail(attachmentId: string | undefined): void {
|
||||
if (attachmentId === undefined || this.thumbs.has(attachmentId)) return;
|
||||
this.thumbs.set(attachmentId, { status: "loading" });
|
||||
this.publish();
|
||||
this.deps.loadImage(attachmentId).then(
|
||||
(url) => {
|
||||
if (!this.thumbs.has(attachmentId)) {
|
||||
// Replaced or removed while loading: never resurrect (or leak) it.
|
||||
revoke(url);
|
||||
return;
|
||||
}
|
||||
this.thumbs.set(attachmentId, { status: "ready", url });
|
||||
const sessionId = this.sessionOf(attachmentId);
|
||||
if (sessionId !== undefined) {
|
||||
const previous = this.lastReady.get(sessionId);
|
||||
if (previous !== undefined && previous.attachmentId !== attachmentId) {
|
||||
revoke(previous.url);
|
||||
this.thumbs.delete(previous.attachmentId);
|
||||
}
|
||||
this.lastReady.set(sessionId, { attachmentId, url });
|
||||
}
|
||||
this.publish();
|
||||
},
|
||||
() => {
|
||||
this.thumbs.set(attachmentId, { status: "error" });
|
||||
this.publish();
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/** Interrupt the current or named session's in-flight call. */
|
||||
async interrupt(sessionId?: string): Promise<boolean> {
|
||||
try {
|
||||
const res = await this.deps.fetchFn(INTERRUPT_URL, {
|
||||
method: "POST",
|
||||
headers: { "content-type": "application/json" },
|
||||
body: JSON.stringify(sessionId === undefined ? {} : { sessionId }),
|
||||
});
|
||||
if (!res.ok) return false;
|
||||
const body = (await res.json()) as { interrupted?: boolean };
|
||||
return body.interrupted === true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
declare module "*.module.css" {
|
||||
const classes: Record<string, string>;
|
||||
export default classes;
|
||||
}
|
||||
|
||||
declare module "*.css";
|
||||
@@ -0,0 +1,83 @@
|
||||
/**
|
||||
* Screenshot delivery. The canonical tool value stays plain JSON (a PNG file
|
||||
* path plus pixel metadata); when the host mounts a durable attachment store
|
||||
* AND the calling route declares image input, the PNG bytes are additionally
|
||||
* committed through `ctx.attachments` so the render step can attach the image
|
||||
* itself to the tool result. Any uncertainty (no store, unknown route,
|
||||
* text-only model) falls back to the path-only form.
|
||||
*/
|
||||
|
||||
import type { Context } from "@deepseek-ai/cordis";
|
||||
import type { ImageAttachmentRef } from "@deepseek-ai/dsh-attachment";
|
||||
import type { ToolExecution } from "@deepseek-ai/dsh-tools";
|
||||
|
||||
/** Structural view of the attachment seam (avoids a runtime dependency). */
|
||||
interface AttachmentLike {
|
||||
imageLimits: {
|
||||
mediaTypes: readonly string[];
|
||||
maxImageBytes: number;
|
||||
maxMessageImageBytes: number;
|
||||
};
|
||||
saveImage(input: {
|
||||
data: Uint8Array;
|
||||
mediaType: string;
|
||||
name?: string;
|
||||
}): Promise<ImageAttachmentRef>;
|
||||
}
|
||||
|
||||
interface LlmLike {
|
||||
resolveModelInfo(
|
||||
provider: string,
|
||||
model: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ inputModalities?: readonly string[] }>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Commit screenshot bytes to the host attachment store when the composition
|
||||
* supports durable images on the current model route.
|
||||
* @returns the durable reference, or undefined to stay in path-only mode.
|
||||
*/
|
||||
export async function trySaveScreenshot(
|
||||
ctx: Context,
|
||||
exec: ToolExecution,
|
||||
data: Uint8Array,
|
||||
name: string,
|
||||
): Promise<ImageAttachmentRef | undefined> {
|
||||
const attachments = ctx.get("attachments") as AttachmentLike | undefined;
|
||||
if (attachments === undefined) return undefined;
|
||||
if (!attachments.imageLimits.mediaTypes.includes("image/png")) return undefined;
|
||||
if (
|
||||
data.byteLength >
|
||||
Math.min(attachments.imageLimits.maxImageBytes, attachments.imageLimits.maxMessageImageBytes)
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
if (!(await isImageCapableRoute(ctx, exec))) return undefined;
|
||||
try {
|
||||
return await attachments.saveImage({ data, mediaType: "image/png", name });
|
||||
} catch {
|
||||
// A store hiccup must not fail the tool call; the PNG path still works.
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Best-effort check that the calling route's model accepts image input,
|
||||
* mirroring dsh-tool-fs `read_image`. Unknown route / missing llm service
|
||||
* answers false (refuse to attach) so history never gains an image block a
|
||||
* text-only adapter cannot replay.
|
||||
*/
|
||||
async function isImageCapableRoute(ctx: Context, exec: ToolExecution): Promise<boolean> {
|
||||
const llm = ctx.get("llm") as LlmLike | undefined;
|
||||
const provider =
|
||||
exec.agent?.session.requestHeader()?.config.provider ?? exec.agent?.options.provider;
|
||||
const model = exec.agent?.session.requestHeader()?.config.model ?? exec.agent?.options.model;
|
||||
if (llm === undefined || provider === undefined || model === undefined) return false;
|
||||
try {
|
||||
const info = await llm.resolveModelInfo(provider, model, exec.signal);
|
||||
return info.inputModalities?.includes("image") ?? false;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
/**
|
||||
* dsh-plugin-browserskill — a DeepSeek Harness bundle that registers
|
||||
* BrowserSkill (`bsk`) browser automation as model-visible tools.
|
||||
*
|
||||
* Each tool spawns `bsk <cmd> --json`, parses the structured output, and
|
||||
* returns a canonical JSON value. The plugin tracks the sessions it starts so
|
||||
* one agent conversation can drive several browsers at once; unloading the
|
||||
* plugin stops every tracked session and kills any in-flight bsk children.
|
||||
*
|
||||
* @module dsh-plugin-browserskill
|
||||
*/
|
||||
|
||||
import type { Context } from "@deepseek-ai/cordis";
|
||||
import Schema from "@deepseek-ai/schemastery";
|
||||
import { armLazyTools } from "./lazy-tools";
|
||||
import { ObservationService } from "./observation";
|
||||
import { registerObservationRoutes } from "./observation-http";
|
||||
import { KeyedExecutor } from "./queue";
|
||||
import { type BskRunner, createBskRunner } from "./runner";
|
||||
import { SessionRegistry } from "./sessions";
|
||||
import { registerBskSkill } from "./skill";
|
||||
import { type PluginConfig, registerTools } from "./tools";
|
||||
|
||||
export const name = "dsh-plugin-browserskill";
|
||||
export const inject = ["tools"];
|
||||
|
||||
/** Runtime configuration schema (validated and defaulted by Cordis). */
|
||||
export const Config = Schema.object({
|
||||
bskPath: Schema.string()
|
||||
.default("bsk")
|
||||
.description("Path to the bsk CLI binary (defaults to resolving `bsk` from PATH)."),
|
||||
defaultTimeoutMs: Schema.number()
|
||||
.default(120_000)
|
||||
.description("Default per-command timeout in milliseconds."),
|
||||
maxSessions: Schema.number()
|
||||
.default(5)
|
||||
.description("Maximum number of concurrent browser sessions started through this plugin."),
|
||||
observationEnabled: Schema.boolean()
|
||||
.default(true)
|
||||
.description("Track per-session observation state (action/url/thumbnail) for the PiP overlay."),
|
||||
thumbnailIntervalMs: Schema.number()
|
||||
.default(1500)
|
||||
.description("Thumbnail refresh cadence for active sessions (milliseconds)."),
|
||||
idleIntervalMs: Schema.number()
|
||||
.default(8000)
|
||||
.description("Thumbnail refresh cadence for idle sessions; also the recent-activity window."),
|
||||
lazyTools: Schema.boolean()
|
||||
.default(true)
|
||||
.description(
|
||||
"Reveal the browser_* tools only after the browser-skill skill is invoked (default true); " +
|
||||
"false registers the full suite at load.",
|
||||
),
|
||||
});
|
||||
|
||||
export type Config = PluginConfig;
|
||||
|
||||
/** Test seams: swap the process runner (unit tests never spawn a real bsk). */
|
||||
export interface ApplyOptions {
|
||||
runnerFactory?: (bskPath: string) => BskRunner;
|
||||
}
|
||||
|
||||
export function apply(
|
||||
ctx: Context,
|
||||
config: Partial<PluginConfig> = {},
|
||||
options: ApplyOptions = {},
|
||||
): void {
|
||||
const resolved: PluginConfig = {
|
||||
bskPath: config.bskPath ?? "bsk",
|
||||
defaultTimeoutMs: config.defaultTimeoutMs ?? 120_000,
|
||||
maxSessions: config.maxSessions ?? 5,
|
||||
observationEnabled: config.observationEnabled ?? true,
|
||||
thumbnailIntervalMs: config.thumbnailIntervalMs ?? 1500,
|
||||
idleIntervalMs: config.idleIntervalMs ?? 8000,
|
||||
lazyTools: config.lazyTools ?? true,
|
||||
};
|
||||
const runner = options.runnerFactory?.(resolved.bskPath) ?? createBskRunner(resolved.bskPath);
|
||||
const registry = new SessionRegistry(resolved.maxSessions);
|
||||
const queue = new KeyedExecutor();
|
||||
const observation = new ObservationService({
|
||||
ctx,
|
||||
runner,
|
||||
registry,
|
||||
queue,
|
||||
options: {
|
||||
enabled: resolved.observationEnabled,
|
||||
thumbnailIntervalMs: resolved.thumbnailIntervalMs,
|
||||
idleIntervalMs: resolved.idleIntervalMs,
|
||||
},
|
||||
});
|
||||
|
||||
// Progressive disclosure of the BSK agent skill (catalog entry resident,
|
||||
// body on demand) through the official skill seam; silent no-op when the
|
||||
// composition lacks it. With lazyTools on, the skill entry is initially the
|
||||
// ONLY model-visible advertisement — the tool suite reveals itself on a
|
||||
// successful skill invocation (or a session whose history already has one).
|
||||
const unregisterSkill = registerBskSkill(ctx);
|
||||
const removeSuite = resolved.lazyTools
|
||||
? armLazyTools(ctx, () =>
|
||||
registerTools({ ctx, runner, registry, config: resolved, observation, queue }),
|
||||
)
|
||||
: registerTools({ ctx, runner, registry, config: resolved, observation, queue });
|
||||
// Route registration rides ctx.inject: the webServer service may be provided
|
||||
// AFTER this plugin loads, and in headless compositions it never appears (the
|
||||
// callback simply never runs, leaving the rest of the plugin unaffected).
|
||||
let removeRoutes: () => void = () => {};
|
||||
ctx.inject(["webServer"], (injected) => {
|
||||
removeRoutes = registerObservationRoutes(injected, observation);
|
||||
return () => removeRoutes();
|
||||
});
|
||||
|
||||
// Non-blocking install probe: warn early when bsk is missing instead of
|
||||
// failing the first tool call with a bare spawn error. Uses --version on
|
||||
// purpose — it answers without starting the daemon (`bsk status` would
|
||||
// ensure-spawn one, an expensive side effect for a probe).
|
||||
runner.run(["--version"], { timeoutMs: 10_000 }).then(
|
||||
() => {},
|
||||
(error: unknown) => {
|
||||
const detail = error instanceof Error ? error.message : String(error);
|
||||
console.warn(
|
||||
`[${name}] bsk probe failed (${detail}); browser tools will report install guidance until the bsk CLI is available`,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
// Unload cleanup: kill in-flight children, then stop every session this
|
||||
// plugin OWNS (created via browser_session_start). Referenced or unknown
|
||||
// sessions belonging to other programs on the shared daemon are never
|
||||
// touched; per-stop failures (already stopped externally, daemon restart)
|
||||
// are swallowed so one stale handle cannot abort the rest.
|
||||
ctx.effect(() => {
|
||||
return () => {
|
||||
removeSuite();
|
||||
unregisterSkill();
|
||||
removeRoutes();
|
||||
observation.dispose();
|
||||
runner.killAll();
|
||||
const stops = registry
|
||||
.ownedIds()
|
||||
.map((sessionId) =>
|
||||
runner.run(["session", "stop", sessionId], { timeoutMs: 15_000 }).catch(() => {}),
|
||||
);
|
||||
return Promise.all(stops).then(() => {});
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export type { ObservationEvent, ObservationOptions, SessionObservation } from "./observation";
|
||||
export { ObservationService } from "./observation";
|
||||
export { registerObservationRoutes } from "./observation-http";
|
||||
export { KeyedExecutor } from "./queue";
|
||||
export type { BskRunner, BskRunResult, SpawnImpl } from "./runner";
|
||||
export { BskError, createBskRunner } from "./runner";
|
||||
export { SessionRegistry } from "./sessions";
|
||||
export type { PluginConfig, ToolDeps } from "./tools";
|
||||
export { registerTools } from "./tools";
|
||||
@@ -0,0 +1,153 @@
|
||||
/**
|
||||
* Lazy tool-schema injection ("progressive disclosure", final stage): with
|
||||
* `lazyTools` on (the default), the 11 browser_* tool schemas stay OUT of the
|
||||
* system prompt until the `browser-skill` skill has actually been invoked —
|
||||
* the skill catalog entry is the only advertisement. One successful
|
||||
* invocation (model tool call, or a user's `/browser-skill` gesture) reveals
|
||||
* the whole suite for the rest of the process lifetime; repeated invocations
|
||||
* are idempotent no-ops. Session resume is covered by scanning durable
|
||||
* session events for a past successful invocation (tool/call + tool/result
|
||||
* pair, or a skill-invocation sourced message) when a session is entered.
|
||||
*
|
||||
* Verified against dsh 0.1 (recorded in the PR ticket):
|
||||
* - `tools/result(exec, result)`: exec carries normalized `name`/`arguments`,
|
||||
* result is discriminated by `isError` — the model-invocation trigger.
|
||||
* - There is NO official global tool-visibility switch (`ctx.tools.restrict`
|
||||
* is agent-scoped and throws from a plain host context), so conditional
|
||||
* registration + this hook is the intended pattern; the registry supports
|
||||
* mid-flight register/dispose with an unfiltered `tools/change` notice, and
|
||||
* tool-skill's per-step catalog digest treats visibility changes as a
|
||||
* first-class cache-invalidation input — the suite simply appears in the
|
||||
* NEXT step's assembly.
|
||||
* - Durable events: `tool/call` {callId, name, arguments(JSON string)} pairs
|
||||
* with `tool/result` {message: {callId, isError}} — the history signal.
|
||||
*/
|
||||
|
||||
import type { Context } from "@deepseek-ai/cordis";
|
||||
import { BSK_SKILL_NAME } from "./skill-content.generated";
|
||||
|
||||
/** Structural view of the pieces of the session seam we consume. */
|
||||
interface SessionLike {
|
||||
events: readonly SessionEventLike[];
|
||||
}
|
||||
interface SessionEventLike {
|
||||
type: string;
|
||||
data?: unknown;
|
||||
}
|
||||
interface SessionsLike {
|
||||
list(): SessionLike[];
|
||||
}
|
||||
|
||||
interface ToolResultExecutionLike {
|
||||
name: string;
|
||||
arguments?: unknown;
|
||||
}
|
||||
|
||||
/** Parse a tool arguments payload that may be normalized (object) or raw JSON. */
|
||||
function skillNameOf(args: unknown): string | undefined {
|
||||
if (typeof args === "string") {
|
||||
try {
|
||||
return skillNameOf(JSON.parse(args));
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
if (typeof args === "object" && args !== null && "name" in args) {
|
||||
const name = (args as { name?: unknown }).name;
|
||||
return typeof name === "string" ? name : undefined;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function isSkillInvocationMessage(data: unknown): boolean {
|
||||
if (typeof data !== "object" || data === null) return false;
|
||||
const source = (data as { source?: unknown }).source;
|
||||
if (typeof source !== "object" || source === null) return false;
|
||||
const { kind, name } = source as { kind?: unknown; name?: unknown };
|
||||
return kind === "skill-invocation" && name === BSK_SKILL_NAME;
|
||||
}
|
||||
|
||||
/**
|
||||
* A durable log proves the skill was successfully invoked when a successful
|
||||
* `tool/result` pairs a `tool/call` for skill/browser-skill — or when a
|
||||
* `/browser-skill` user gesture landed as a skill-invocation message.
|
||||
*/
|
||||
export function hasSuccessfulSkillInvocation(events: readonly SessionEventLike[]): boolean {
|
||||
const skillCallIds = new Set<unknown>();
|
||||
for (const event of events) {
|
||||
if (event.type !== "tool/call") continue;
|
||||
const data = event.data as { callId?: unknown; name?: unknown; arguments?: unknown };
|
||||
if (data?.name === "skill" && skillNameOf(data.arguments) === BSK_SKILL_NAME) {
|
||||
skillCallIds.add(data.callId);
|
||||
}
|
||||
}
|
||||
for (const event of events) {
|
||||
if (event.type === "tool/result") {
|
||||
const message = (event.data as { message?: unknown })?.message;
|
||||
if (typeof message !== "object" || message === null) continue;
|
||||
const { callId, isError } = message as { callId?: unknown; isError?: unknown };
|
||||
if (isError === false && skillCallIds.has(callId)) return true;
|
||||
}
|
||||
if (isSkillInvocationMessage(event.data)) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Arm the lazy reveal. Returns a disposer tearing down listeners and — when
|
||||
* the reveal already happened — the tool suite itself.
|
||||
* @param registerSuite - registers the 11 browser_* tools, returns their disposer.
|
||||
*/
|
||||
export function armLazyTools(ctx: Context, registerSuite: () => () => void): () => void {
|
||||
let suiteDisposer: (() => void) | undefined;
|
||||
const disposers: (() => void)[] = [];
|
||||
const ensureSuite = (): void => {
|
||||
if (suiteDisposer !== undefined) return;
|
||||
try {
|
||||
suiteDisposer = registerSuite();
|
||||
} catch (error) {
|
||||
// A failed reveal must not strand the plugin: stay hidden, log, retry on
|
||||
// the next trigger instead of latching a half-registered suite.
|
||||
suiteDisposer = undefined;
|
||||
console.warn(
|
||||
`[dsh-plugin-browserskill] lazy tool registration failed: ${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
// Live trigger: a successful model invocation of skill/browser-skill.
|
||||
const onToolResult = (exec: ToolResultExecutionLike, result: { isError: boolean }): void => {
|
||||
if (result.isError) return;
|
||||
if (exec.name !== "skill") return;
|
||||
if (skillNameOf(exec.arguments) === BSK_SKILL_NAME) ensureSuite();
|
||||
};
|
||||
disposers.push(ctx.on("tools/result" as never, onToolResult as never));
|
||||
|
||||
const scanSession = (session: SessionLike): void => {
|
||||
try {
|
||||
if (hasSuccessfulSkillInvocation(session.events)) ensureSuite();
|
||||
} catch {
|
||||
// A session object that cannot be read must never break plugin startup.
|
||||
}
|
||||
};
|
||||
|
||||
// Live gesture/append feed: covers /browser-skill user gestures (no tool
|
||||
// call happens on that path) landing as skill-invocation messages.
|
||||
const onSessionEvent = (_session: SessionLike, event: SessionEventLike): void => {
|
||||
if (isSkillInvocationMessage(event?.data)) ensureSuite();
|
||||
};
|
||||
disposers.push(ctx.on("session/event" as never, onSessionEvent as never));
|
||||
|
||||
// History restore: sessions entered from now on, plus any already live.
|
||||
const onSessionCreated = (session: SessionLike): void => scanSession(session);
|
||||
disposers.push(ctx.on("session/created" as never, onSessionCreated as never));
|
||||
const sessions = ctx.get("sessions") as SessionsLike | null | undefined;
|
||||
if (sessions != null && typeof sessions.list === "function") {
|
||||
for (const session of sessions.list()) scanSession(session);
|
||||
}
|
||||
|
||||
return () => {
|
||||
for (const dispose of disposers.splice(0)) dispose();
|
||||
suiteDisposer?.();
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
/**
|
||||
* HTTP/SSE exposure of the ObservationService for the client half. The Typert
|
||||
* Remote pipeline and the forwarded-event allowlist are both closed to
|
||||
* out-of-tree packages in dsh 0.1 (generation runs over dsh's own ts.Program;
|
||||
* the event allowlist lives in dsh-api-remotes), so the plugin serves its
|
||||
* observation channel through the documented `webServer` route seam instead:
|
||||
*
|
||||
* GET /bsk-observation/state → { sessions, available }
|
||||
* GET /bsk-observation/events → SSE stream of ObservationEvent
|
||||
* POST /bsk-observation/interrupt → body {sessionId?} → {interrupted: boolean}
|
||||
* GET /bsk-observation/thumbnail/<attachmentId> → image bytes
|
||||
*
|
||||
* Routes exist only when a webServer service is mounted (web composition);
|
||||
* headless deployments skip registration entirely.
|
||||
*/
|
||||
|
||||
import type { IncomingMessage, ServerResponse } from "node:http";
|
||||
import type { Context } from "@deepseek-ai/cordis";
|
||||
import type { ObservationService } from "./observation";
|
||||
|
||||
/** Structural view of the dsh-host-webserver route seam. */
|
||||
interface WebServerLike {
|
||||
register(route: {
|
||||
kind: "exact" | "prefix";
|
||||
path: string;
|
||||
handler: (req: IncomingMessage, res: ServerResponse) => void | Promise<void>;
|
||||
}): () => void;
|
||||
}
|
||||
|
||||
const ROUTE_BASE = "/bsk-observation";
|
||||
const SSE_HEARTBEAT_MS = 15_000;
|
||||
|
||||
function sendJson(res: ServerResponse, status: number, body: unknown): void {
|
||||
res.writeHead(status, { "content-type": "application/json; charset=utf-8" });
|
||||
res.end(JSON.stringify(body));
|
||||
}
|
||||
|
||||
/**
|
||||
* Browser-trust fence, mirroring the one dsh applies to its /api routes (ours
|
||||
* live outside that prefix, so the checks are replicated here):
|
||||
* - Host must be a loopback authority (localhost / 127.0.0.0/8 / [::1]) — the
|
||||
* observation channel exposes live screenshots and must never answer a LAN
|
||||
* or DNS-rebound name;
|
||||
* - a present Origin must be same-host with the Host header (blocks cross-site
|
||||
* reads), and `sec-fetch-site: cross-site` is refused outright;
|
||||
* - POST must be `application/json` — anything a cross-site *simple request*
|
||||
* can send (form/plain) never reaches the handler, which kills CSRF.
|
||||
*/
|
||||
function fenceViolation(req: IncomingMessage): string | undefined {
|
||||
const host = req.headers.host ?? "";
|
||||
const hostname = /^\[.*\](?::\d+)?$/.test(host)
|
||||
? host.slice(1, host.indexOf("]"))
|
||||
: host.split(":")[0];
|
||||
const isLoopback =
|
||||
hostname === "localhost" ||
|
||||
hostname.endsWith(".localhost") ||
|
||||
hostname === "::1" ||
|
||||
/^127\.\d{1,3}\.\d{1,3}\.\d{1,3}$/.test(hostname);
|
||||
if (!isLoopback) return "host is not a loopback authority";
|
||||
const origin = req.headers.origin;
|
||||
if (origin !== undefined && origin !== "null") {
|
||||
let originHost: string | undefined;
|
||||
try {
|
||||
originHost = new URL(origin).host;
|
||||
} catch {
|
||||
return "unparseable Origin header";
|
||||
}
|
||||
if (originHost !== host) return "Origin does not match Host";
|
||||
}
|
||||
if (req.headers["sec-fetch-site"] === "cross-site") return "sec-fetch-site: cross-site";
|
||||
if (req.method === "POST") {
|
||||
const contentType = req.headers["content-type"] ?? "";
|
||||
if (!/^\s*application\/json\s*(;|$)/.test(contentType)) {
|
||||
return "POST requires an application/json body";
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/** Run the fence; returns true when the request was rejected (handled). */
|
||||
function fenceRejected(req: IncomingMessage, res: ServerResponse): boolean {
|
||||
const violation = fenceViolation(req);
|
||||
if (violation === undefined) return false;
|
||||
sendJson(res, 403, { error: `forbidden: ${violation}` });
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Register the observation routes. No-op (with a console note) when the
|
||||
* composition has no web server.
|
||||
* @returns disposer removing the routes.
|
||||
*/
|
||||
export function registerObservationRoutes(
|
||||
ctx: Context,
|
||||
observation: ObservationService,
|
||||
): () => void {
|
||||
const webServer = ctx.get("webServer") as WebServerLike | undefined;
|
||||
if (webServer === undefined) {
|
||||
return () => {};
|
||||
}
|
||||
const disposers = [
|
||||
webServer.register({
|
||||
kind: "exact",
|
||||
path: `${ROUTE_BASE}/state`,
|
||||
handler: (req, res) => {
|
||||
if (req.method !== "GET") {
|
||||
sendJson(res, 405, { error: "method not allowed" });
|
||||
return;
|
||||
}
|
||||
if (fenceRejected(req, res)) return;
|
||||
sendJson(res, 200, {
|
||||
sessions: observation.getState(),
|
||||
available: observation.isAvailable(),
|
||||
});
|
||||
},
|
||||
}),
|
||||
webServer.register({
|
||||
kind: "exact",
|
||||
path: `${ROUTE_BASE}/events`,
|
||||
handler: (req, res) => {
|
||||
if (req.method !== "GET") {
|
||||
sendJson(res, 405, { error: "method not allowed" });
|
||||
return;
|
||||
}
|
||||
if (fenceRejected(req, res)) return;
|
||||
res.writeHead(200, {
|
||||
"content-type": "text/event-stream",
|
||||
"cache-control": "no-cache",
|
||||
connection: "keep-alive",
|
||||
});
|
||||
const unsubscribe = observation.subscribe((event) => {
|
||||
res.write(`data: ${JSON.stringify(event)}\n\n`);
|
||||
});
|
||||
const heartbeat = setInterval(() => res.write(": heartbeat\n\n"), SSE_HEARTBEAT_MS);
|
||||
res.on("close", () => {
|
||||
clearInterval(heartbeat);
|
||||
unsubscribe();
|
||||
});
|
||||
},
|
||||
}),
|
||||
webServer.register({
|
||||
kind: "exact",
|
||||
path: `${ROUTE_BASE}/interrupt`,
|
||||
handler: (req, res) => {
|
||||
if (req.method !== "POST") {
|
||||
sendJson(res, 405, { error: "method not allowed" });
|
||||
return;
|
||||
}
|
||||
if (fenceRejected(req, res)) return;
|
||||
let body = "";
|
||||
req.on("data", (chunk: Buffer | string) => {
|
||||
body += chunk;
|
||||
});
|
||||
req.on("end", () => {
|
||||
let sessionId: string | undefined;
|
||||
try {
|
||||
const parsed = JSON.parse(body || "{}") as { sessionId?: unknown };
|
||||
if (typeof parsed.sessionId === "string" && parsed.sessionId !== "") {
|
||||
sessionId = parsed.sessionId;
|
||||
}
|
||||
} catch {
|
||||
sendJson(res, 400, { error: "invalid JSON body" });
|
||||
return;
|
||||
}
|
||||
sendJson(res, 200, { interrupted: observation.interrupt(sessionId) });
|
||||
});
|
||||
},
|
||||
}),
|
||||
webServer.register({
|
||||
kind: "prefix",
|
||||
path: `${ROUTE_BASE}/thumbnail`,
|
||||
handler: async (req, res) => {
|
||||
if (req.method !== "GET") {
|
||||
sendJson(res, 405, { error: "method not allowed" });
|
||||
return;
|
||||
}
|
||||
if (fenceRejected(req, res)) return;
|
||||
const pathname = decodeURIComponent(new URL(req.url ?? "/", "http://x").pathname);
|
||||
const attachmentId = pathname.slice(`${ROUTE_BASE}/thumbnail/`.length);
|
||||
const frame = await observation.readThumbnail(attachmentId);
|
||||
if (frame === undefined) {
|
||||
sendJson(res, 404, { error: "unknown thumbnail" });
|
||||
return;
|
||||
}
|
||||
res.writeHead(200, {
|
||||
"content-type": frame.mediaType,
|
||||
"cache-control": "no-cache",
|
||||
});
|
||||
res.end(Buffer.from(frame.data));
|
||||
},
|
||||
}),
|
||||
];
|
||||
return () => {
|
||||
for (const dispose of disposers) dispose();
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,414 @@
|
||||
/**
|
||||
* ObservationService: per-owned-session live observation state for the PiP
|
||||
* overlay — current action, page url, and a breathing thumbnail identified by
|
||||
* an ephemeral frame id. Frames live in a per-session 2-slot ring in process
|
||||
* memory (never the durable attachment store). Strict ownership boundary
|
||||
* applies throughout: only sessions in the plugin's SessionRegistry (owned by
|
||||
* construction) ever get an observation entry, and interrupt/kill paths can
|
||||
* only reach children this plugin spawned.
|
||||
*
|
||||
* Observation traffic isolation: thumbnail captures run through the runner
|
||||
* directly (never through the tool-level instrumentation), emit no action
|
||||
* events, and never move the registry's current pointer.
|
||||
*/
|
||||
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
import { readFile, unlink } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { Context } from "@deepseek-ai/cordis";
|
||||
import type { KeyedExecutor } from "./queue";
|
||||
import type { BskRunner } from "./runner";
|
||||
import type { SessionRegistry } from "./sessions";
|
||||
|
||||
/** One owned session's live observation record (wire-stable shape). */
|
||||
export interface SessionObservation {
|
||||
sessionId: string;
|
||||
/** Last settled page URL, when any navigation completed. */
|
||||
url?: string;
|
||||
/** Current action ('idle' when nothing is in flight). */
|
||||
action: string;
|
||||
/** Epoch ms when the current action started (elapsed time is client-side). */
|
||||
since: number;
|
||||
/** Latest thumbnail id (ephemeral, plugin-owned; never bytes on the wire). */
|
||||
thumbnailAttachmentId?: string;
|
||||
/** Summary of the most recent failed action (drives the red status dot). */
|
||||
lastError?: string;
|
||||
/**
|
||||
* The daemon reports this session as gone (e.g. daemon restart): the strip
|
||||
* greys it out until it is stopped/removed. No more frames are requested.
|
||||
*/
|
||||
dead?: boolean;
|
||||
}
|
||||
|
||||
/** Incremental event carried to subscribers (SSE on the wire). */
|
||||
export type ObservationEvent =
|
||||
| { type: "upsert" | "remove" | "reset"; session?: SessionObservation }
|
||||
| { type: "availability"; available: boolean };
|
||||
|
||||
export interface ObservationOptions {
|
||||
enabled: boolean;
|
||||
/** Fast cadence while a session is active or recently was. */
|
||||
thumbnailIntervalMs: number;
|
||||
/** Slow cadence for idle sessions; also the "recently active" window. */
|
||||
idleIntervalMs: number;
|
||||
}
|
||||
|
||||
/** Injectable clock/scheduler bits for tests. */
|
||||
export interface ObservationScheduler {
|
||||
setTimeout: (fn: () => void, ms: number) => unknown;
|
||||
clearTimeout: (handle: unknown) => void;
|
||||
now: () => number;
|
||||
}
|
||||
|
||||
const DEFAULT_SCHEDULER: ObservationScheduler = {
|
||||
setTimeout: (fn, ms) => {
|
||||
const timer = setTimeout(fn, ms);
|
||||
// Never hold the event loop open for a frame refresh.
|
||||
if (typeof timer === "object" && typeof timer.unref === "function") timer.unref();
|
||||
return timer;
|
||||
},
|
||||
clearTimeout: (h) => clearTimeout(h as Parameters<typeof clearTimeout>[0]),
|
||||
now: () => Date.now(),
|
||||
};
|
||||
|
||||
/** Max consecutive capture failures before a session drops to the idle cadence. */
|
||||
const FAILURE_BACKOFF_THRESHOLD = 3;
|
||||
/** Current + previous frame, so an in-flight HTTP fetch of the old id still lands. */
|
||||
const FRAME_RING_SIZE = 2;
|
||||
|
||||
export class ObservationService {
|
||||
private readonly scratchNamespace = randomUUID();
|
||||
private readonly observations = new Map<string, SessionObservation>();
|
||||
private readonly listeners = new Set<(event: ObservationEvent) => void>();
|
||||
private readonly captureTimers = new Map<string, unknown>();
|
||||
private readonly captureInFlight = new Set<string>();
|
||||
private readonly captureFailures = new Map<string, number>();
|
||||
private readonly lastActivity = new Map<string, number>();
|
||||
/** Ephemeral frame id → PNG bytes. Only the live ring members are present. */
|
||||
private readonly frames = new Map<string, { data: Uint8Array; mediaType: string }>();
|
||||
/** sessionId → oldest-first frame ids, length ≤ FRAME_RING_SIZE. */
|
||||
private readonly rings = new Map<string, string[]>();
|
||||
/** sessionId → sha256 of the current frame (skip republish when unchanged). */
|
||||
private readonly hashes = new Map<string, string>();
|
||||
/** sessionId → last issued sequence number. */
|
||||
private readonly seqs = new Map<string, number>();
|
||||
/** Consecutive capture failures across ALL sessions (daemon-level signal). */
|
||||
private globalFailures = 0;
|
||||
private available = true;
|
||||
private disposed = false;
|
||||
|
||||
constructor(
|
||||
private readonly deps: {
|
||||
ctx: Context;
|
||||
runner: BskRunner;
|
||||
registry: SessionRegistry;
|
||||
queue: KeyedExecutor;
|
||||
options: ObservationOptions;
|
||||
scheduler?: ObservationScheduler;
|
||||
},
|
||||
) {}
|
||||
|
||||
private get scheduler(): ObservationScheduler {
|
||||
return this.deps.scheduler ?? DEFAULT_SCHEDULER;
|
||||
}
|
||||
|
||||
/** All current entries (client initial/resync snapshot). */
|
||||
getState(): SessionObservation[] {
|
||||
return [...this.observations.values()].map((entry) => ({ ...entry }));
|
||||
}
|
||||
|
||||
/** Whether the browser side looks reachable (drives the "browser unavailable" state). */
|
||||
isAvailable(): boolean {
|
||||
return this.available;
|
||||
}
|
||||
|
||||
private setAvailable(available: boolean): void {
|
||||
if (this.available === available) return;
|
||||
this.available = available;
|
||||
this.emit({ type: "availability", available });
|
||||
}
|
||||
|
||||
/** Subscribe to incremental changes; returns an unsubscribe function. */
|
||||
subscribe(listener: (event: ObservationEvent) => void): () => void {
|
||||
this.listeners.add(listener);
|
||||
return () => this.listeners.delete(listener);
|
||||
}
|
||||
|
||||
private emit(event: ObservationEvent): void {
|
||||
for (const listener of [...this.listeners]) {
|
||||
try {
|
||||
listener(event);
|
||||
} catch {
|
||||
// A faulty subscriber must not starve the others.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private put(entry: SessionObservation): void {
|
||||
this.observations.set(entry.sessionId, entry);
|
||||
this.emit({ type: "upsert", session: { ...entry } });
|
||||
}
|
||||
|
||||
/** Register a fresh owned session (called from browser_session_start). */
|
||||
addSession(sessionId: string, url?: string): void {
|
||||
if (!this.deps.options.enabled || this.disposed) return;
|
||||
this.put({
|
||||
sessionId,
|
||||
...(url !== undefined ? { url } : {}),
|
||||
action: "idle",
|
||||
since: this.scheduler.now(),
|
||||
});
|
||||
this.lastActivity.set(sessionId, this.scheduler.now());
|
||||
this.scheduleCapture(sessionId, 0);
|
||||
}
|
||||
|
||||
/** Drop a session (called from browser_session_stop). */
|
||||
removeSession(sessionId: string): void {
|
||||
this.cancelCapture(sessionId);
|
||||
this.captureFailures.delete(sessionId);
|
||||
this.lastActivity.delete(sessionId);
|
||||
this.dropSessionFrames(sessionId);
|
||||
if (this.observations.delete(sessionId)) {
|
||||
this.emit({
|
||||
type: "remove",
|
||||
session: { sessionId, action: "idle", since: this.scheduler.now() },
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/** Mark an action starting on a session (tool entry instrumentation). */
|
||||
beginAction(sessionId: string, action: string): void {
|
||||
if (!this.deps.options.enabled || this.disposed) return;
|
||||
const entry = this.observations.get(sessionId);
|
||||
if (entry === undefined || entry.dead === true) return;
|
||||
const now = this.scheduler.now();
|
||||
this.lastActivity.set(sessionId, now);
|
||||
// A fresh action clears the previous error marker (red dot means "last action failed").
|
||||
const next: SessionObservation = { ...entry, action, since: now };
|
||||
delete next.lastError;
|
||||
this.put(next);
|
||||
}
|
||||
|
||||
/**
|
||||
* Mark the current action settling (tool exit instrumentation). Triggers an
|
||||
* immediate thumbnail refresh — action-driven first, timer as fallback.
|
||||
*/
|
||||
endAction(sessionId: string, error?: string): void {
|
||||
if (!this.deps.options.enabled || this.disposed) return;
|
||||
const entry = this.observations.get(sessionId);
|
||||
if (entry === undefined || entry.dead === true) return;
|
||||
const now = this.scheduler.now();
|
||||
this.lastActivity.set(sessionId, now);
|
||||
const next: SessionObservation = { ...entry, action: "idle", since: now };
|
||||
if (error !== undefined) next.lastError = error;
|
||||
else delete next.lastError;
|
||||
this.put(next);
|
||||
this.scheduleCapture(sessionId, 0);
|
||||
}
|
||||
|
||||
/** Record the settled page URL (navigate success / start with url). */
|
||||
setUrl(sessionId: string, url: string): void {
|
||||
if (!this.deps.options.enabled || this.disposed) return;
|
||||
const entry = this.observations.get(sessionId);
|
||||
if (entry === undefined) return;
|
||||
this.put({ ...entry, url });
|
||||
}
|
||||
|
||||
/**
|
||||
* Interrupt the in-flight call of one session (default: the registry's
|
||||
* current). Kills exactly the bsk children this plugin spawned for that
|
||||
* session — same user-visible semantics as the chat Stop button (the in-flight
|
||||
* tool call fails; the agent flow may continue).
|
||||
* @returns whether an in-flight call was actually interrupted.
|
||||
*/
|
||||
interrupt(sessionId?: string): boolean {
|
||||
const target = sessionId ?? this.deps.registry.current();
|
||||
if (target === undefined || !this.deps.registry.isOwned(target)) return false;
|
||||
return this.deps.runner.killFor(target) > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read one captured thumbnail from the in-process ring. Powers the plugin's
|
||||
* own HTTP thumbnail route — frames are plugin-owned runtime data, never
|
||||
* referenced by any session log, so the session-authorized client RPC
|
||||
* cannot serve them.
|
||||
*/
|
||||
async readThumbnail(
|
||||
attachmentId: string,
|
||||
): Promise<{ data: Uint8Array; mediaType: string } | undefined> {
|
||||
const frame = this.frames.get(attachmentId);
|
||||
if (frame === undefined) return undefined;
|
||||
return { data: frame.data, mediaType: frame.mediaType };
|
||||
}
|
||||
|
||||
/** Tear down all state and timers (plugin dispose). */
|
||||
dispose(): void {
|
||||
this.disposed = true;
|
||||
for (const sessionId of [...this.captureTimers.keys()]) this.cancelCapture(sessionId);
|
||||
this.captureTimers.clear();
|
||||
this.captureInFlight.clear();
|
||||
this.captureFailures.clear();
|
||||
this.lastActivity.clear();
|
||||
this.frames.clear();
|
||||
this.rings.clear();
|
||||
this.hashes.clear();
|
||||
this.seqs.clear();
|
||||
this.observations.clear();
|
||||
this.emit({ type: "reset" });
|
||||
this.listeners.clear();
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Thumbnail loop
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
private cancelCapture(sessionId: string): void {
|
||||
const timer = this.captureTimers.get(sessionId);
|
||||
if (timer !== undefined) {
|
||||
this.scheduler.clearTimeout(timer);
|
||||
this.captureTimers.delete(sessionId);
|
||||
}
|
||||
}
|
||||
|
||||
/** Schedule the next capture for a session; `delayMs` 0 means "as soon as the event loop allows". */
|
||||
private scheduleCapture(sessionId: string, delayMs?: number): void {
|
||||
if (!this.deps.options.enabled || this.disposed) return;
|
||||
const entry = this.observations.get(sessionId);
|
||||
if (entry === undefined || entry.dead === true) return;
|
||||
this.cancelCapture(sessionId);
|
||||
const now = this.scheduler.now();
|
||||
const lastSeen = this.lastActivity.get(sessionId) ?? 0;
|
||||
const failures = this.captureFailures.get(sessionId) ?? 0;
|
||||
const { thumbnailIntervalMs, idleIntervalMs } = this.deps.options;
|
||||
const cadence =
|
||||
now - lastSeen < idleIntervalMs && failures < FAILURE_BACKOFF_THRESHOLD
|
||||
? thumbnailIntervalMs
|
||||
: idleIntervalMs;
|
||||
const delay = delayMs ?? cadence;
|
||||
const timer = this.scheduler.setTimeout(() => {
|
||||
this.captureTimers.delete(sessionId);
|
||||
void this.capture(sessionId);
|
||||
}, delay);
|
||||
this.captureTimers.set(sessionId, timer);
|
||||
}
|
||||
|
||||
/**
|
||||
* Capture one frame: `bsk screenshot --json` through the runner, bytes into
|
||||
* the in-process ring, id onto the observation. Runs OUTSIDE the tool
|
||||
* instrumentation on purpose — no action events, no registry writes.
|
||||
* Failures keep the previous frame and back off silently.
|
||||
*/
|
||||
private async capture(sessionId: string): Promise<void> {
|
||||
if (this.disposed) return;
|
||||
const current = this.observations.get(sessionId);
|
||||
if (current === undefined || current.dead === true) return;
|
||||
if (this.captureInFlight.has(sessionId)) return;
|
||||
this.captureInFlight.add(sessionId);
|
||||
// Stable within this service, isolated from captures owned by other instances.
|
||||
const sessionKey = createHash("sha256").update(sessionId).digest("hex").slice(0, 16);
|
||||
const outPath = join(tmpdir(), `bsk-obs-${this.scratchNamespace}-${sessionKey}.png`);
|
||||
let writtenPath = outPath;
|
||||
try {
|
||||
const result = await this.deps.queue.run(sessionId, () =>
|
||||
this.deps.runner.run(["screenshot", "--session", sessionId, "--out", outPath], {
|
||||
timeoutMs: 15_000,
|
||||
tag: `observation:${sessionId}`,
|
||||
}),
|
||||
);
|
||||
if (result.code !== 0) {
|
||||
let code: string | undefined;
|
||||
try {
|
||||
code = (JSON.parse(result.stdout) as { code?: string }).code;
|
||||
} catch {
|
||||
code = undefined;
|
||||
}
|
||||
if (code === "session_not_found") {
|
||||
this.markDead(sessionId);
|
||||
return;
|
||||
}
|
||||
throw new Error(`screenshot exited ${result.code}`);
|
||||
}
|
||||
const reply = JSON.parse(result.stdout) as { path?: string };
|
||||
writtenPath = reply.path ?? outPath;
|
||||
const data = await readFile(writtenPath);
|
||||
const entry = this.observations.get(sessionId);
|
||||
if (entry === undefined) return;
|
||||
this.captureFailures.delete(sessionId);
|
||||
this.globalFailures = 0;
|
||||
this.setAvailable(true);
|
||||
this.publishFrame(sessionId, entry, data);
|
||||
} catch {
|
||||
// Silent by design: keep the previous frame, count toward backoff.
|
||||
this.captureFailures.set(sessionId, (this.captureFailures.get(sessionId) ?? 0) + 1);
|
||||
this.globalFailures += 1;
|
||||
if (this.globalFailures >= FAILURE_BACKOFF_THRESHOLD) this.setAvailable(false);
|
||||
} finally {
|
||||
await unlink(writtenPath).catch(() => {});
|
||||
this.captureInFlight.delete(sessionId);
|
||||
if (!this.disposed && this.observations.has(sessionId)) this.scheduleCapture(sessionId);
|
||||
}
|
||||
}
|
||||
|
||||
/** Insert a new frame, or no-op when the PNG bytes match the current one. */
|
||||
private publishFrame(sessionId: string, entry: SessionObservation, data: Uint8Array): void {
|
||||
const hash = createHash("sha256").update(data).digest("hex");
|
||||
if (this.hashes.get(sessionId) === hash) return;
|
||||
const seq = (this.seqs.get(sessionId) ?? 0) + 1;
|
||||
this.seqs.set(sessionId, seq);
|
||||
const id = `obs-${sessionId}-${seq}`;
|
||||
this.frames.set(id, { data, mediaType: "image/png" });
|
||||
this.hashes.set(sessionId, hash);
|
||||
const ring = this.rings.get(sessionId) ?? [];
|
||||
ring.push(id);
|
||||
while (ring.length > FRAME_RING_SIZE) {
|
||||
const evicted = ring.shift();
|
||||
if (evicted !== undefined) this.frames.delete(evicted);
|
||||
}
|
||||
this.rings.set(sessionId, ring);
|
||||
this.put({ ...entry, thumbnailAttachmentId: id });
|
||||
}
|
||||
|
||||
private dropSessionFrames(sessionId: string): void {
|
||||
for (const id of this.rings.get(sessionId) ?? []) this.frames.delete(id);
|
||||
this.rings.delete(sessionId);
|
||||
this.hashes.delete(sessionId);
|
||||
this.seqs.delete(sessionId);
|
||||
}
|
||||
|
||||
/** Mark a session dead: grey it out, stop asking for frames, keep the entry for the strip. */
|
||||
private markDead(sessionId: string): void {
|
||||
const entry = this.observations.get(sessionId);
|
||||
if (entry === undefined || entry.dead === true) return;
|
||||
this.cancelCapture(sessionId);
|
||||
this.put({ ...entry, dead: true, action: "idle" });
|
||||
}
|
||||
}
|
||||
|
||||
/** Map a bsk command label onto its observation action verb. */
|
||||
export function actionForLabel(label: string): string {
|
||||
switch (label) {
|
||||
case "session start":
|
||||
return "starting";
|
||||
case "session stop":
|
||||
return "stopping";
|
||||
case "navigate":
|
||||
return "navigating";
|
||||
case "snapshot":
|
||||
return "snapshotting";
|
||||
case "observe":
|
||||
return "observing";
|
||||
case "click":
|
||||
return "clicking";
|
||||
case "fill":
|
||||
return "filling";
|
||||
case "press":
|
||||
return "pressing";
|
||||
case "screenshot":
|
||||
return "capturing";
|
||||
case "emulate":
|
||||
return "emulating";
|
||||
default:
|
||||
return label;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
/**
|
||||
* Per-key FIFO executor. The bsk daemon serializes commands per session (a
|
||||
* second command while one is unfinished is rejected), so every plugin
|
||||
* command — model-facing tool calls AND observation captures — funnels
|
||||
* through one queue per session. A queued task rejects early if its abort
|
||||
* signal fires before it starts; once running, cancellation is owned by the
|
||||
* task's own signal handling (the runner kills the child).
|
||||
*/
|
||||
|
||||
export class KeyedExecutor {
|
||||
private readonly tails = new Map<string, Promise<void>>();
|
||||
|
||||
/** Run `fn` after every previously queued task for `key` settled. */
|
||||
run<T>(key: string, fn: () => Promise<T>, signal?: AbortSignal): Promise<T> {
|
||||
const previous = this.tails.get(key) ?? Promise.resolve();
|
||||
const task = new Promise<T>((resolve, reject) => {
|
||||
let settled = false;
|
||||
const onAbort = () => {
|
||||
if (!settled) {
|
||||
settled = true;
|
||||
reject(abortError());
|
||||
}
|
||||
};
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
void previous.then(() => {
|
||||
if (settled) return;
|
||||
if (signal?.aborted) {
|
||||
settled = true;
|
||||
reject(abortError());
|
||||
return;
|
||||
}
|
||||
// Running now: cancellation flows through the task's own signal.
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
fn().then(
|
||||
(value) => {
|
||||
settled = true;
|
||||
resolve(value);
|
||||
},
|
||||
(error: unknown) => {
|
||||
settled = true;
|
||||
reject(error);
|
||||
},
|
||||
);
|
||||
});
|
||||
});
|
||||
// The stored tail never rejects, so one failure never strands the queue;
|
||||
// and it chains BOTH the previous tail and this task, so a task aborted
|
||||
// while still queued cannot let a later task overlap the one before it.
|
||||
const tail = Promise.allSettled([previous, task]).then(() => {});
|
||||
this.tails.set(key, tail);
|
||||
// Drop the entry once the queue drains, so the map tracks live keys only.
|
||||
void tail.then(() => {
|
||||
if (this.tails.get(key) === tail) this.tails.delete(key);
|
||||
});
|
||||
return task;
|
||||
}
|
||||
}
|
||||
|
||||
function abortError(): Error {
|
||||
const error = new Error("tool call aborted");
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
/**
|
||||
* Process runner for the `bsk` CLI. Every model-facing tool in this plugin
|
||||
* maps to one `bsk <cmd> --json` invocation: spawn the child, capture
|
||||
* stdout/stderr, honor the dsh cancellation signal by killing the child, and
|
||||
* map the CLI's JSON error envelope onto a thrown `BskError`.
|
||||
*/
|
||||
|
||||
import { type ChildProcess, spawn } from "node:child_process";
|
||||
|
||||
/** Shape of the JSON error envelope `bsk --json` prints on failure. */
|
||||
export interface BskErrorBody {
|
||||
code?: string;
|
||||
message?: string;
|
||||
hint?: string;
|
||||
exit_code?: number;
|
||||
data?: unknown;
|
||||
}
|
||||
|
||||
/** A failed `bsk` invocation (non-zero exit, timeout, or spawn failure). */
|
||||
export class BskError extends Error {
|
||||
readonly code?: string;
|
||||
readonly hint?: string;
|
||||
readonly exitCode?: number | null;
|
||||
readonly timedOut: boolean;
|
||||
|
||||
constructor(
|
||||
message: string,
|
||||
options: { code?: string; hint?: string; exitCode?: number | null; timedOut?: boolean } = {},
|
||||
) {
|
||||
super(message);
|
||||
this.name = "BskError";
|
||||
this.code = options.code;
|
||||
this.hint = options.hint;
|
||||
this.exitCode = options.exitCode;
|
||||
this.timedOut = options.timedOut ?? false;
|
||||
}
|
||||
}
|
||||
|
||||
export interface BskRunResult {
|
||||
/** Process exit code, or null when killed by a signal / never started. */
|
||||
code: number | null;
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
timedOut: boolean;
|
||||
aborted: boolean;
|
||||
}
|
||||
|
||||
export interface BskRunOptions {
|
||||
signal?: AbortSignal;
|
||||
timeoutMs?: number;
|
||||
/** Opaque routing tag (e.g. a session id) enabling per-tag kills. */
|
||||
tag?: string;
|
||||
}
|
||||
|
||||
/** Minimal spawn signature so tests can substitute a fake child process. */
|
||||
export type SpawnImpl = (command: string, args: string[]) => ChildProcess;
|
||||
|
||||
export interface BskRunner {
|
||||
/** Run `bsk <args...> --json` and collect its output. */
|
||||
run(args: string[], options?: BskRunOptions): Promise<BskRunResult>;
|
||||
/** Kill every in-flight child (used when the plugin unloads). */
|
||||
killAll(): void;
|
||||
/** Kill only the in-flight children carrying this tag; returns how many were killed. */
|
||||
killFor(tag: string): number;
|
||||
}
|
||||
|
||||
const KILL_GRACE_MS = 2000;
|
||||
|
||||
export function createBskRunner(bskPath: string, spawnImpl: SpawnImpl = spawn): BskRunner {
|
||||
const live = new Map<ChildProcess, string | undefined>();
|
||||
|
||||
function killChild(child: ChildProcess): void {
|
||||
if (child.exitCode !== null || child.signalCode !== null) return;
|
||||
child.kill("SIGTERM");
|
||||
const force = setTimeout(() => {
|
||||
if (child.exitCode === null && child.signalCode === null) child.kill("SIGKILL");
|
||||
}, KILL_GRACE_MS);
|
||||
force.unref();
|
||||
}
|
||||
|
||||
return {
|
||||
run(args, options = {}) {
|
||||
return new Promise<BskRunResult>((resolve, reject) => {
|
||||
let child: ChildProcess;
|
||||
try {
|
||||
child = spawnImpl(bskPath, [...args, "--json"]);
|
||||
} catch (error) {
|
||||
reject(error);
|
||||
return;
|
||||
}
|
||||
live.set(child, options.tag);
|
||||
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
let timedOut = false;
|
||||
let aborted = false;
|
||||
child.stdout?.on("data", (chunk: Buffer | string) => {
|
||||
stdout += chunk;
|
||||
});
|
||||
child.stderr?.on("data", (chunk: Buffer | string) => {
|
||||
stderr += chunk;
|
||||
});
|
||||
|
||||
const timeoutMs = options.timeoutMs;
|
||||
const timer =
|
||||
timeoutMs !== undefined && timeoutMs > 0
|
||||
? setTimeout(() => {
|
||||
timedOut = true;
|
||||
killChild(child);
|
||||
}, timeoutMs)
|
||||
: undefined;
|
||||
timer?.unref();
|
||||
|
||||
const onAbort = () => {
|
||||
aborted = true;
|
||||
killChild(child);
|
||||
};
|
||||
if (options.signal?.aborted) {
|
||||
onAbort();
|
||||
} else {
|
||||
options.signal?.addEventListener("abort", onAbort, { once: true });
|
||||
}
|
||||
|
||||
const settle = () => {
|
||||
if (timer !== undefined) clearTimeout(timer);
|
||||
options.signal?.removeEventListener("abort", onAbort);
|
||||
live.delete(child);
|
||||
};
|
||||
|
||||
child.on("error", (error) => {
|
||||
settle();
|
||||
reject(error);
|
||||
});
|
||||
child.on("close", (code) => {
|
||||
settle();
|
||||
resolve({ code, stdout, stderr, timedOut, aborted });
|
||||
});
|
||||
});
|
||||
},
|
||||
killAll() {
|
||||
for (const child of live.keys()) killChild(child);
|
||||
},
|
||||
killFor(tag: string) {
|
||||
let killed = 0;
|
||||
for (const [child, childTag] of live) {
|
||||
if (childTag === tag) {
|
||||
killChild(child);
|
||||
killed += 1;
|
||||
}
|
||||
}
|
||||
return killed;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** True when the spawn failure means the bsk binary itself is missing. */
|
||||
export function isCommandNotFound(error: unknown): boolean {
|
||||
return (
|
||||
typeof error === "object" &&
|
||||
error !== null &&
|
||||
"code" in error &&
|
||||
(error as { code?: unknown }).code === "ENOENT"
|
||||
);
|
||||
}
|
||||
|
||||
/** Install guidance shown when the bsk CLI cannot be spawned. */
|
||||
export function bskInstallMessage(bskPath: string): string {
|
||||
return (
|
||||
`the bsk CLI ("${bskPath}") was not found. BrowserSkill must be installed and on PATH ` +
|
||||
"for browser tools to work — install it from https://github.com/Tencent/BrowserSkill " +
|
||||
"(see the README install script or `cargo install`), then retry."
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Interpret one finished run: throw `BskError` on timeout / non-zero exit
|
||||
* (parsing the CLI's JSON error envelope when present), otherwise parse and
|
||||
* return the stdout JSON payload.
|
||||
*/
|
||||
export function parseBskJson(result: BskRunResult, commandLabel: string): unknown {
|
||||
if (result.timedOut) {
|
||||
throw new BskError(`bsk ${commandLabel} timed out`, { timedOut: true });
|
||||
}
|
||||
const body = result.stdout.trim();
|
||||
// Killed by our own interrupt (SIGTERM from killFor), not by the abort path:
|
||||
// say so instead of doubling the generic label into the message.
|
||||
if (result.code === null && !result.aborted && !result.timedOut) {
|
||||
throw new BskError(`bsk ${commandLabel} was interrupted (process killed)`);
|
||||
}
|
||||
if (result.code !== 0) {
|
||||
let parsed: BskErrorBody | undefined;
|
||||
try {
|
||||
parsed = JSON.parse(body) as BskErrorBody;
|
||||
} catch {
|
||||
parsed = undefined;
|
||||
}
|
||||
const message =
|
||||
parsed?.message ?? (result.stderr.trim() || body || `bsk ${commandLabel} failed`);
|
||||
// Surface the envelope's actionable hint in the model-facing message; a
|
||||
// hint the model cannot see cannot be followed.
|
||||
const withHint = parsed?.hint !== undefined ? `${message} (hint: ${parsed.hint})` : message;
|
||||
throw new BskError(`bsk ${commandLabel} failed: ${withHint}`, {
|
||||
code: parsed?.code,
|
||||
hint: parsed?.hint,
|
||||
exitCode: result.code,
|
||||
});
|
||||
}
|
||||
try {
|
||||
return JSON.parse(body);
|
||||
} catch {
|
||||
throw new BskError(
|
||||
`bsk ${commandLabel} did not produce JSON output: ${body.slice(0, 200) || "(empty)"}`,
|
||||
{ exitCode: result.code },
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
/**
|
||||
* Tracks the bsk sessions this plugin created and the "current" session
|
||||
* pointer used when a tool call omits its optional `session` arg. The pointer
|
||||
* moves to whatever session was most recently started, stopped, or operated
|
||||
* on, so a model can run several browsers side by side without repeating the
|
||||
* id on every call.
|
||||
*
|
||||
* Strict ownership boundary (the bsk daemon may be shared with other agents,
|
||||
* terminals, or dsh instances): the registry ONLY ever holds sessions created
|
||||
* by this plugin's browser_session_start. Tools cannot see or act on any
|
||||
* other session — an explicit `session` argument naming a foreign id is an
|
||||
* error, the list tool shows owned sessions only, and stop/unload cleanup
|
||||
* can never touch a session this plugin did not create.
|
||||
*/
|
||||
|
||||
export interface TrackedSession {
|
||||
sessionId: string;
|
||||
browserInstanceId?: string;
|
||||
startedAtMs: number;
|
||||
/** Always true: only plugin-created sessions enter the registry at all. */
|
||||
owned: boolean;
|
||||
}
|
||||
|
||||
export class SessionRegistry {
|
||||
private readonly sessions = new Map<string, TrackedSession>();
|
||||
private currentId: string | undefined;
|
||||
/**
|
||||
* Slots reserved by in-flight starts. reserveStart/completeStart/abandonStart
|
||||
* run synchronously around the async spawn, so concurrent starts can never
|
||||
* both pass the capacity check (check-and-reserve is atomic on the event loop).
|
||||
*/
|
||||
private pendingStarts = 0;
|
||||
|
||||
constructor(private readonly maxSessions: number) {}
|
||||
|
||||
/**
|
||||
* Reserve a start slot synchronously, BEFORE spawning.
|
||||
* @throws when the configured concurrency cap (tracked + in-flight) is reached.
|
||||
*/
|
||||
reserveStart(): void {
|
||||
if (this.sessions.size + this.pendingStarts >= this.maxSessions) {
|
||||
throw new Error(
|
||||
`session limit reached (${this.maxSessions} concurrent sessions); ` +
|
||||
"stop one with browser_session_stop before starting another",
|
||||
);
|
||||
}
|
||||
this.pendingStarts += 1;
|
||||
}
|
||||
|
||||
/** Give back a reservation after a start that never produced a session. */
|
||||
abandonStart(): void {
|
||||
this.pendingStarts = Math.max(0, this.pendingStarts - 1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Register a freshly started session, consuming its reservation, and make
|
||||
* it current.
|
||||
*/
|
||||
completeStart(session: Omit<TrackedSession, "owned">): void {
|
||||
this.pendingStarts = Math.max(0, this.pendingStarts - 1);
|
||||
// Backstop only: with the reservation protocol above this never fires.
|
||||
if (!this.sessions.has(session.sessionId) && this.sessions.size >= this.maxSessions) {
|
||||
throw new Error(
|
||||
`session limit reached (${this.maxSessions} concurrent sessions); ` +
|
||||
"stop one with browser_session_stop before starting another",
|
||||
);
|
||||
}
|
||||
this.sessions.set(session.sessionId, { ...session, owned: true });
|
||||
this.currentId = session.sessionId;
|
||||
}
|
||||
|
||||
/** Forget a session; falls back to the most recent remaining one. */
|
||||
remove(sessionId: string): void {
|
||||
this.sessions.delete(sessionId);
|
||||
if (this.currentId === sessionId) {
|
||||
const rest = [...this.sessions.values()];
|
||||
this.currentId = rest.length > 0 ? rest[rest.length - 1].sessionId : undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** Mark an owned session as most recently used (recency order refresh). */
|
||||
private touch(sessionId: string): void {
|
||||
const existing = this.sessions.get(sessionId);
|
||||
if (existing === undefined) return;
|
||||
this.sessions.delete(sessionId);
|
||||
this.sessions.set(sessionId, existing);
|
||||
this.currentId = sessionId;
|
||||
}
|
||||
|
||||
/** The current session id, if any. */
|
||||
current(): string | undefined {
|
||||
return this.currentId;
|
||||
}
|
||||
|
||||
/** Owned sessions in least- to most-recently-used order. */
|
||||
list(): TrackedSession[] {
|
||||
return [...this.sessions.values()];
|
||||
}
|
||||
|
||||
/** Ids of owned sessions — the exact set unload cleanup is allowed to stop. */
|
||||
ownedIds(): string[] {
|
||||
return this.list()
|
||||
.filter((session) => session.owned)
|
||||
.map((session) => session.sessionId);
|
||||
}
|
||||
|
||||
/** Whether the session was created by this plugin. */
|
||||
isOwned(sessionId: string): boolean {
|
||||
return this.sessions.get(sessionId)?.owned === true;
|
||||
}
|
||||
|
||||
size(): number {
|
||||
return this.sessions.size;
|
||||
}
|
||||
|
||||
/** Shared not-yours error for foreign or unknown session ids. */
|
||||
private foreignError(sessionId: string, toolName: string): Error {
|
||||
return new Error(
|
||||
`${toolName}: session "${sessionId}" does not belong to this plugin — only sessions ` +
|
||||
"created by browser_session_start are visible and operable here",
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the session a tool call acts on: an explicit `session` argument
|
||||
* must name an owned session (and becomes current); omitted falls back to
|
||||
* the current session. Foreign ids are rejected, never adopted.
|
||||
* @throws on foreign/unknown ids, or when no session exists.
|
||||
*/
|
||||
resolve(explicit: string | undefined, toolName: string): string {
|
||||
if (explicit !== undefined && explicit.trim().length > 0) {
|
||||
if (!this.isOwned(explicit)) throw this.foreignError(explicit, toolName);
|
||||
this.touch(explicit);
|
||||
return explicit;
|
||||
}
|
||||
const current = this.current();
|
||||
if (current === undefined) {
|
||||
throw new Error(
|
||||
`${toolName} needs a session but none is active — start one with browser_session_start`,
|
||||
);
|
||||
}
|
||||
this.touch(current);
|
||||
return current;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the session a STOP call acts on. Same ownership rule as every
|
||||
* other tool; a rejected stop never moves the current pointer.
|
||||
*/
|
||||
resolveForStop(explicit: string | undefined): string {
|
||||
const candidate =
|
||||
explicit !== undefined && explicit.trim().length > 0 ? explicit : this.current();
|
||||
if (candidate === undefined) {
|
||||
throw new Error(
|
||||
"browser_session_stop needs a session but none is active — start one with browser_session_start",
|
||||
);
|
||||
}
|
||||
if (!this.isOwned(candidate)) throw this.foreignError(candidate, "browser_session_stop");
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
/**
|
||||
* Progressive disclosure of the BrowserSkill agent skill through the harness's
|
||||
* official skill seam (`ctx.skills`). The catalog entry (name + description)
|
||||
* is resident; the body is loaded only when the model invokes the `skill`
|
||||
* tool. The markdown is build-time assembled into a static module (dsh
|
||||
* prelude + the canonical CLI skill body verbatim), so registration and every
|
||||
* pre-step catalog snapshot are pure in-memory reads — no disk, no process,
|
||||
* no daemon.
|
||||
*/
|
||||
|
||||
import type { Context } from "@deepseek-ai/cordis";
|
||||
import {
|
||||
BSK_SKILL_DESCRIPTION,
|
||||
BSK_SKILL_MARKDOWN,
|
||||
BSK_SKILL_NAME,
|
||||
} from "./skill-content.generated";
|
||||
|
||||
/** Structural view of the skill registry seam (absent in bare compositions). */
|
||||
interface SkillsLike {
|
||||
register(skill: {
|
||||
name: string;
|
||||
description: string;
|
||||
content: string;
|
||||
source?: string;
|
||||
}): () => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Publish the BSK skill as an embedded runtime skill. Returns the unregistration
|
||||
* disposer. A composition without the `skills` service (or with dsh-tool-skill
|
||||
* retired) leaves the rest of the plugin unaffected — the no-op is silent.
|
||||
*/
|
||||
export function registerBskSkill(ctx: Context): () => void {
|
||||
const skills = ctx.get("skills") as SkillsLike | undefined | null;
|
||||
if (skills == null || typeof skills.register !== "function") {
|
||||
return () => {};
|
||||
}
|
||||
return skills.register({
|
||||
name: BSK_SKILL_NAME,
|
||||
description: BSK_SKILL_DESCRIPTION,
|
||||
content: BSK_SKILL_MARKDOWN,
|
||||
// Prompt-visible origin bucket: packaged with a plugin, not user/project files.
|
||||
source: "bundled",
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,961 @@
|
||||
/**
|
||||
* Model-facing tool definitions. Each tool maps to one `bsk <cmd> --json`
|
||||
* invocation: spawn via the runner, parse the JSON envelope, and return a
|
||||
* canonical value whose `output.render` produces the model-facing text.
|
||||
* UI cards reuse the terminal style: the pending card is the command line,
|
||||
* the completed card carries the output.
|
||||
*/
|
||||
|
||||
import { readFile, unlink } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { Context } from "@deepseek-ai/cordis";
|
||||
import type { ContentBlock } from "@deepseek-ai/dsh-llm";
|
||||
import {
|
||||
defineTool,
|
||||
type ToolDefinition,
|
||||
type ToolResult,
|
||||
type ToolRunContext,
|
||||
} from "@deepseek-ai/dsh-tools";
|
||||
import { trySaveScreenshot } from "./image";
|
||||
import { actionForLabel, type ObservationService } from "./observation";
|
||||
import type { KeyedExecutor } from "./queue";
|
||||
import { type BskRunner, bskInstallMessage, isCommandNotFound, parseBskJson } from "./runner";
|
||||
import type { SessionRegistry } from "./sessions";
|
||||
|
||||
/** Plugin configuration resolved from the Schemastery schema in index.ts. */
|
||||
export interface PluginConfig {
|
||||
bskPath: string;
|
||||
defaultTimeoutMs: number;
|
||||
maxSessions: number;
|
||||
observationEnabled: boolean;
|
||||
thumbnailIntervalMs: number;
|
||||
idleIntervalMs: number;
|
||||
/**
|
||||
* Lazy tool-schema injection: when true (default), the browser_* tools are
|
||||
* registered only after the `browser-skill` skill has been successfully
|
||||
* invoked (skill catalog entry advertises alone until then); when false,
|
||||
* the suite is registered at apply time (legacy always-on behavior).
|
||||
*/
|
||||
lazyTools: boolean;
|
||||
}
|
||||
|
||||
export interface ToolDeps {
|
||||
ctx: Context;
|
||||
runner: BskRunner;
|
||||
registry: SessionRegistry;
|
||||
config: PluginConfig;
|
||||
observation: ObservationService;
|
||||
/** Per-session FIFO: the daemon rejects a second command while one is unfinished. */
|
||||
queue: KeyedExecutor;
|
||||
}
|
||||
|
||||
/** Device presets supported by `bsk emulate --device`. */
|
||||
const DEVICE_PRESETS = [
|
||||
"iphone-14",
|
||||
"iphone-14-pro-max",
|
||||
"iphone-se",
|
||||
"pixel-7",
|
||||
"galaxy-s23",
|
||||
"ipad-mini",
|
||||
"galaxy-tab-s8",
|
||||
] as const;
|
||||
|
||||
const SESSION_PARAM = {
|
||||
type: "string",
|
||||
description:
|
||||
"bsk session id to act on; must be one created by this plugin's browser_session_start. " +
|
||||
"Omit to use the current session (the one most recently started or used).",
|
||||
} as const;
|
||||
|
||||
/**
|
||||
* Run one bsk command and return its parsed JSON payload, or throw.
|
||||
* `observeSession` marks the run as model-facing work on that session: it is
|
||||
* instrumented into the observation service (action begin/end) and tagged so
|
||||
* interrupt() can kill exactly this child. Observation traffic itself never
|
||||
* passes an observeSession.
|
||||
*/
|
||||
async function runBsk(
|
||||
deps: ToolDeps,
|
||||
exec: ToolRunContext,
|
||||
args: string[],
|
||||
label: string,
|
||||
observeSession?: string,
|
||||
): Promise<unknown> {
|
||||
// beginAction must fire only when the task actually starts inside the
|
||||
// per-session queue — marking it while still queued would overwrite the
|
||||
// in-flight action's label (and flash it idle on a queued abort).
|
||||
let began = false;
|
||||
let actionError: string | undefined;
|
||||
try {
|
||||
let result;
|
||||
try {
|
||||
const runOnce = () => {
|
||||
if (observeSession !== undefined) {
|
||||
began = true;
|
||||
deps.observation.beginAction(observeSession, actionForLabel(label));
|
||||
}
|
||||
return deps.runner.run(args, {
|
||||
signal: exec.signal,
|
||||
timeoutMs: deps.config.defaultTimeoutMs,
|
||||
...(observeSession !== undefined ? { tag: observeSession } : {}),
|
||||
});
|
||||
};
|
||||
result =
|
||||
observeSession !== undefined
|
||||
? await deps.queue.run(observeSession, runOnce, exec.signal)
|
||||
: await runOnce();
|
||||
} catch (error) {
|
||||
if (isCommandNotFound(error)) {
|
||||
throw new Error(bskInstallMessage(deps.config.bskPath));
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
if (result.aborted) {
|
||||
const error = new Error("tool call aborted");
|
||||
error.name = "AbortError";
|
||||
throw error;
|
||||
}
|
||||
return parseBskJson(result, label);
|
||||
} catch (error) {
|
||||
actionError = error instanceof Error ? error.message.split("\n")[0] : String(error);
|
||||
throw error;
|
||||
} finally {
|
||||
if (observeSession !== undefined && began) {
|
||||
deps.observation.endAction(observeSession, actionError);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Build the command line shown on the pending terminal card (pure). */
|
||||
function cmdline(deps: ToolDeps, args: string[]): string {
|
||||
return [deps.config.bskPath, ...args].join(" ");
|
||||
}
|
||||
|
||||
/** Shared completed-card presenter: terminal card with the rendered text. */
|
||||
function presentTerminalResult(_args: never, result: ToolResult) {
|
||||
// Multi-block results (e.g. screenshot text + image) still project their
|
||||
// text block onto the terminal card.
|
||||
const block = result.content.find((b) => b.type === "text");
|
||||
if (block === undefined || block.type !== "text") return undefined;
|
||||
if (result.isError) return undefined;
|
||||
return { card: "terminal" as const, output: block.text, exitCode: 0 };
|
||||
}
|
||||
|
||||
function abortError(): Error {
|
||||
const error = new Error("tool call aborted");
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
/** Register the full browser tool suite; returns the combined unregister disposer. */
|
||||
export function registerTools(deps: ToolDeps): () => void {
|
||||
const { ctx, registry } = deps;
|
||||
// Track every registration without touching the cordis context itself
|
||||
// (contexts are fiber-owned; the lazy path registers from a listener fiber).
|
||||
const disposers: (() => void)[] = [];
|
||||
const register = (definition: ToolDefinition): (() => void) => {
|
||||
const dispose = ctx.tools.register(definition);
|
||||
// Test doubles occasionally return the registry map instead of a disposer.
|
||||
if (typeof dispose === "function") disposers.push(dispose);
|
||||
return dispose;
|
||||
};
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_session_start",
|
||||
description:
|
||||
"Start a new browser session: opens an Agent Window in the connected browser and returns " +
|
||||
"its session id. The new session becomes the current session for subsequent browser_* calls. " +
|
||||
"Optionally navigate to an initial URL and/or apply a mobile device emulation preset.",
|
||||
parameters: {
|
||||
url: {
|
||||
type: "string",
|
||||
description: "Initial URL to navigate to after the session starts.",
|
||||
},
|
||||
width: {
|
||||
type: "integer",
|
||||
description: "Agent Window outer width in CSS pixels (100..=7680). Requires height.",
|
||||
},
|
||||
height: {
|
||||
type: "integer",
|
||||
description: "Agent Window outer height in CSS pixels (100..=7680). Requires width.",
|
||||
},
|
||||
noFocus: {
|
||||
type: "boolean",
|
||||
description: "Open the Agent Window in the background without stealing focus.",
|
||||
},
|
||||
browser: {
|
||||
type: "string",
|
||||
description:
|
||||
"Target browser instance id (only needed when multiple browsers are connected).",
|
||||
},
|
||||
device: {
|
||||
type: "string",
|
||||
enum: DEVICE_PRESETS,
|
||||
description: "Mobile device emulation preset applied to the tab after start.",
|
||||
},
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
sessionId: { type: "string", required: true },
|
||||
browserInstanceId: { type: "string", required: true },
|
||||
url: { type: "string" },
|
||||
device: { type: "string" },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text:
|
||||
`started browser session ${value.sessionId}` +
|
||||
(value.url !== undefined ? ` and navigated to ${value.url}` : "") +
|
||||
(value.device !== undefined ? ` (device: ${value.device})` : ""),
|
||||
},
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
if (args.url !== undefined && args.url.trim().length === 0) {
|
||||
throw new Error("url must be a non-empty string");
|
||||
}
|
||||
if ((args.width === undefined) !== (args.height === undefined)) {
|
||||
throw new Error("width and height must be given together");
|
||||
}
|
||||
// Reserve the slot BEFORE spawning: check-and-reserve is synchronous,
|
||||
// so concurrent starts can never both pass the cap and leak a session.
|
||||
registry.reserveStart();
|
||||
const startArgs = ["session", "start"];
|
||||
if (args.width !== undefined && args.height !== undefined) {
|
||||
startArgs.push("--width", String(args.width), "--height", String(args.height));
|
||||
}
|
||||
if (args.noFocus === true) startArgs.push("--no-focus");
|
||||
if (args.browser !== undefined) startArgs.push("--browser", args.browser);
|
||||
let reply: { session_id: string; browser_instance_id: string };
|
||||
try {
|
||||
reply = (await runBsk(deps, exec, startArgs, "session start")) as typeof reply;
|
||||
} catch (error) {
|
||||
registry.abandonStart();
|
||||
throw error;
|
||||
}
|
||||
registry.completeStart({
|
||||
sessionId: reply.session_id,
|
||||
browserInstanceId: reply.browser_instance_id,
|
||||
startedAtMs: Date.now(),
|
||||
});
|
||||
deps.observation.addSession(reply.session_id, args.url);
|
||||
try {
|
||||
if (args.device !== undefined) {
|
||||
await runBsk(
|
||||
deps,
|
||||
exec,
|
||||
["emulate", "--session", reply.session_id, "--device", args.device],
|
||||
"emulate",
|
||||
reply.session_id,
|
||||
);
|
||||
}
|
||||
if (args.url !== undefined) {
|
||||
await runBsk(
|
||||
deps,
|
||||
exec,
|
||||
["navigate", "--session", reply.session_id, args.url],
|
||||
"navigate",
|
||||
reply.session_id,
|
||||
);
|
||||
}
|
||||
} catch (error) {
|
||||
// A half-initialized session must not leak: stop it before surfacing.
|
||||
// The stop funnels through the session's queue (an in-flight command
|
||||
// — e.g. an observation capture — would otherwise make the daemon
|
||||
// refuse it), with one retry for good measure.
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
try {
|
||||
const stopped = await deps.queue.run(reply.session_id, () =>
|
||||
deps.runner.run(["session", "stop", reply.session_id], { timeoutMs: 30_000 }),
|
||||
);
|
||||
if (stopped.code === 0) break;
|
||||
} catch {
|
||||
// retry once, then give up quietly (registry/observation are
|
||||
// cleaned below either way; the daemon reaps orphans).
|
||||
}
|
||||
}
|
||||
registry.remove(reply.session_id);
|
||||
deps.observation.removeSession(reply.session_id);
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
sessionId: reply.session_id,
|
||||
browserInstanceId: reply.browser_instance_id,
|
||||
...(args.url !== undefined ? { url: args.url } : {}),
|
||||
...(args.device !== undefined ? { device: args.device } : {}),
|
||||
};
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: cmdline(deps, [
|
||||
"session",
|
||||
"start",
|
||||
...(args.device !== undefined ? ["+ emulate", args.device] : []),
|
||||
...(args.url !== undefined ? ["+ navigate", args.url] : []),
|
||||
]),
|
||||
description: "Start a browser session",
|
||||
}),
|
||||
presentResult: presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_session_stop",
|
||||
description:
|
||||
"Stop a browser session and close its Agent Window. Stops the given session, or the " +
|
||||
"current session when `session` is omitted. Only sessions created by browser_session_start " +
|
||||
"can be stopped — sessions owned by other programs sharing the bsk daemon are refused.",
|
||||
parameters: { session: SESSION_PARAM },
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: { stopped: { type: "string", required: true } },
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{ type: "text", text: `stopped browser session ${value.stopped}` },
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolveForStop(args.session);
|
||||
await runBsk(deps, exec, ["session", "stop", sessionId], "session stop", sessionId);
|
||||
registry.remove(sessionId);
|
||||
deps.observation.removeSession(sessionId);
|
||||
return { stopped: sessionId };
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: cmdline(deps, ["session", "stop", args.session ?? "(current session)"]),
|
||||
description: "Stop a browser session",
|
||||
}),
|
||||
presentResult: presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_session_list",
|
||||
description:
|
||||
"List the browser sessions created by this plugin. Sessions owned by other programs on the " +
|
||||
"shared bsk daemon are not visible here. The session marked `current` is the one browser_* " +
|
||||
"tools act on when no explicit `session` is passed.",
|
||||
parameters: {},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
sessions: {
|
||||
type: "array",
|
||||
required: true,
|
||||
items: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
sessionId: { type: "string", required: true },
|
||||
browserInstanceId: { type: "string", required: true },
|
||||
current: { type: "boolean", required: true },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text:
|
||||
value.sessions.length === 0
|
||||
? "no active browser sessions"
|
||||
: value.sessions
|
||||
.map(
|
||||
(s) =>
|
||||
`${s.sessionId} (browser ${s.browserInstanceId})${s.current ? " [current]" : ""}`,
|
||||
)
|
||||
.join("\n"),
|
||||
},
|
||||
],
|
||||
},
|
||||
isConcurrencySafe: () => true,
|
||||
// Registry-only by design: no daemon call, so foreign sessions on a
|
||||
// shared daemon can never even be SEEN through this tool.
|
||||
async execute() {
|
||||
const current = registry.current();
|
||||
return {
|
||||
sessions: registry.list().map((entry) => ({
|
||||
sessionId: entry.sessionId,
|
||||
browserInstanceId: entry.browserInstanceId ?? "",
|
||||
current: entry.sessionId === current,
|
||||
})),
|
||||
};
|
||||
},
|
||||
presentCall: () => ({
|
||||
card: "terminal",
|
||||
title: cmdline(deps, ["session", "list"]),
|
||||
description: "List browser sessions",
|
||||
}),
|
||||
presentResult: presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_navigate",
|
||||
description:
|
||||
"Navigate the session's active tab to a URL and wait for a page lifecycle phase " +
|
||||
"(default: load). Returns the final URL after redirects.",
|
||||
parameters: {
|
||||
url: { type: "string", required: true, description: "Destination URL." },
|
||||
session: SESSION_PARAM,
|
||||
waitUntil: {
|
||||
type: "string",
|
||||
enum: ["load", "domcontentloaded", "networkidle", "commit"],
|
||||
description: "Lifecycle phase to wait for (default: load).",
|
||||
},
|
||||
timeoutMs: {
|
||||
type: "integer",
|
||||
description: "Navigation wait timeout in milliseconds (default 30000).",
|
||||
},
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
url: { type: "string", required: true },
|
||||
finalUrl: { type: "string" },
|
||||
reached: { type: "string", required: true },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text:
|
||||
`[session ${value.session}] navigated to ${value.finalUrl ?? value.url} (reached: ${value.reached})` +
|
||||
(value.reached === "timeout"
|
||||
? " — the wait timed out before the requested phase"
|
||||
: ""),
|
||||
},
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
if (args.url.trim().length === 0) throw new Error("url must be a non-empty string");
|
||||
const sessionId = registry.resolve(args.session, "browser_navigate");
|
||||
const cmdArgs = ["navigate", "--session", sessionId];
|
||||
if (args.waitUntil !== undefined) cmdArgs.push("--wait-until", args.waitUntil);
|
||||
if (args.timeoutMs !== undefined) cmdArgs.push("--timeout", `${args.timeoutMs}ms`);
|
||||
cmdArgs.push(args.url);
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "navigate", sessionId)) as {
|
||||
tab_id: number;
|
||||
url: string;
|
||||
final_url?: string;
|
||||
reached: string;
|
||||
};
|
||||
deps.observation.setUrl(sessionId, reply.final_url ?? reply.url);
|
||||
return {
|
||||
session: sessionId,
|
||||
tabId: reply.tab_id,
|
||||
url: reply.url,
|
||||
...(reply.final_url !== undefined ? { finalUrl: reply.final_url } : {}),
|
||||
reached: reply.reached,
|
||||
};
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: cmdline(deps, ["navigate", "--session", args.session ?? "(current)", args.url]),
|
||||
description: "Navigate to a URL",
|
||||
}),
|
||||
presentResult: presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
const registerObservationTool = (kind: "snapshot" | "observe") => {
|
||||
const isSnapshot = kind === "snapshot";
|
||||
const name = isSnapshot ? "browser_snapshot" : "browser_observe";
|
||||
const description = isSnapshot
|
||||
? "Capture an indented aria-tree snapshot of the session's active tab. Interactive elements " +
|
||||
"carry @eN refs for browser_click / browser_fill. Prefer browser_observe for a richer semantic view."
|
||||
: "Produce a semantic VOM observation of the session's active tab (roles, states, perception " +
|
||||
"probes) with @eN refs for browser_click / browser_fill. Read-only: never submits input.";
|
||||
register(
|
||||
defineTool({
|
||||
name,
|
||||
description,
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
maxDepth: { type: "integer", description: "Cap on tree depth before truncating." },
|
||||
maxTokens: {
|
||||
type: "integer",
|
||||
description: "Soft cap on rendered tokens (~4 chars/token).",
|
||||
},
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
text: { type: "string", required: true },
|
||||
refCount: { type: "integer", required: true },
|
||||
truncated: { type: "boolean", required: true },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text:
|
||||
value.text.length > 0
|
||||
? value.text + (value.truncated ? "\n(truncated — re-run with looser caps)" : "")
|
||||
: "(empty observation — page may still be loading)",
|
||||
},
|
||||
],
|
||||
},
|
||||
isConcurrencySafe: () => true,
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, name);
|
||||
const cmdArgs = [kind, "--session", sessionId];
|
||||
if (args.maxDepth !== undefined) cmdArgs.push("--max-depth", String(args.maxDepth));
|
||||
if (args.maxTokens !== undefined) cmdArgs.push("--max-tokens", String(args.maxTokens));
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, kind, sessionId)) as {
|
||||
text: string;
|
||||
ref_count: number;
|
||||
tab_id: number;
|
||||
truncated?: boolean;
|
||||
};
|
||||
return {
|
||||
session: sessionId,
|
||||
tabId: reply.tab_id,
|
||||
text: reply.text,
|
||||
refCount: reply.ref_count,
|
||||
truncated: reply.truncated ?? false,
|
||||
};
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: cmdline(deps, [kind, "--session", args.session ?? "(current)"]),
|
||||
description: isSnapshot ? "Capture an aria snapshot" : "Observe the page semantically",
|
||||
}),
|
||||
presentResult: presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
};
|
||||
registerObservationTool("snapshot");
|
||||
registerObservationTool("observe");
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_click",
|
||||
description:
|
||||
"Click an element in the session's active tab. Target is a snapshot ref (@e3) from the " +
|
||||
"last browser_snapshot/browser_observe, or a CSS selector.",
|
||||
parameters: {
|
||||
target: {
|
||||
type: "string",
|
||||
required: true,
|
||||
description: "Snapshot ref (@e3 / e3) or CSS selector of the element to click.",
|
||||
},
|
||||
session: SESSION_PARAM,
|
||||
button: {
|
||||
type: "string",
|
||||
enum: ["left", "middle", "right"],
|
||||
description: "Mouse button (default: left).",
|
||||
},
|
||||
clickCount: {
|
||||
type: "integer",
|
||||
description: "Number of consecutive presses (double-click = 2).",
|
||||
},
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
x: { type: "number", required: true },
|
||||
y: { type: "number", required: true },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text: `[session ${value.session}] clicked at (${value.x}, ${value.y}) on tab ${value.tabId}`,
|
||||
},
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
if (args.target.trim().length === 0) throw new Error("target must be a non-empty string");
|
||||
const sessionId = registry.resolve(args.session, "browser_click");
|
||||
const cmdArgs = ["click", "--session", sessionId];
|
||||
if (args.button !== undefined) cmdArgs.push("--button", args.button);
|
||||
if (args.clickCount !== undefined) cmdArgs.push("--click-count", String(args.clickCount));
|
||||
cmdArgs.push(args.target);
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "click", sessionId)) as {
|
||||
tab_id: number;
|
||||
x: number;
|
||||
y: number;
|
||||
};
|
||||
return { session: sessionId, tabId: reply.tab_id, x: reply.x, y: reply.y };
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: cmdline(deps, ["click", args.target, "--session", args.session ?? "(current)"]),
|
||||
description: "Click an element",
|
||||
}),
|
||||
presentResult: presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_fill",
|
||||
description:
|
||||
"Fill an input / textarea / contenteditable element, clearing it first by default. " +
|
||||
"Target is a snapshot ref (@e3) or a CSS selector.",
|
||||
parameters: {
|
||||
target: {
|
||||
type: "string",
|
||||
required: true,
|
||||
description: "Snapshot ref (@e3 / e3) or CSS selector of the field.",
|
||||
},
|
||||
value: { type: "string", required: true, description: "Text to type into the element." },
|
||||
session: SESSION_PARAM,
|
||||
noClear: {
|
||||
type: "boolean",
|
||||
description: "Skip the default wipe-the-field-first pass (append instead).",
|
||||
},
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
valueLength: { type: "integer", required: true },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text: `[session ${value.session}] filled field on tab ${value.tabId} (${value.valueLength} chars)`,
|
||||
},
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
if (args.target.trim().length === 0) throw new Error("target must be a non-empty string");
|
||||
const sessionId = registry.resolve(args.session, "browser_fill");
|
||||
const cmdArgs = ["fill", "--session", sessionId, "--value", args.value];
|
||||
if (args.noClear === true) cmdArgs.push("--no-clear");
|
||||
cmdArgs.push(args.target);
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "fill", sessionId)) as {
|
||||
tab_id: number;
|
||||
value_length: number;
|
||||
};
|
||||
return { session: sessionId, tabId: reply.tab_id, valueLength: reply.value_length };
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: cmdline(deps, ["fill", args.target, "--session", args.session ?? "(current)"]),
|
||||
description: `Fill a field with ${args.value.length} chars`,
|
||||
}),
|
||||
presentResult: presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_press",
|
||||
description:
|
||||
"Dispatch a keyboard key or combo (Enter, Escape, ArrowDown, Ctrl+A, …) to the session's " +
|
||||
"active tab, optionally focusing a target element first.",
|
||||
parameters: {
|
||||
key: {
|
||||
type: "string",
|
||||
required: true,
|
||||
description: "Key spec: a CDP key name (Enter, a, ArrowLeft) or a combo (Ctrl+A).",
|
||||
},
|
||||
session: SESSION_PARAM,
|
||||
target: {
|
||||
type: "string",
|
||||
description: "Snapshot ref (@e3) or CSS selector to focus before pressing.",
|
||||
},
|
||||
holdMs: {
|
||||
type: "integer",
|
||||
description: "Hold the key down for N milliseconds between keyDown and keyUp.",
|
||||
},
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
key: { type: "string", required: true },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text: `[session ${value.session}] pressed ${value.key} on tab ${value.tabId}`,
|
||||
},
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
if (args.key.trim().length === 0) throw new Error("key must be a non-empty string");
|
||||
const sessionId = registry.resolve(args.session, "browser_press");
|
||||
const cmdArgs = ["press", "--session", sessionId];
|
||||
if (args.target !== undefined) {
|
||||
if (args.target.trim().length === 0) throw new Error("target must be a non-empty string");
|
||||
// bsk distinguishes ref vs selector via flags; positional detection is CLI-side for
|
||||
// click/fill only, so pass the explicit flag that matches the target's shape.
|
||||
if (/^@?e\d+$/.test(args.target)) {
|
||||
cmdArgs.push("--ref", args.target);
|
||||
} else {
|
||||
cmdArgs.push("--selector", args.target);
|
||||
}
|
||||
}
|
||||
if (args.holdMs !== undefined) cmdArgs.push("--hold-ms", String(args.holdMs));
|
||||
cmdArgs.push(args.key);
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "press", sessionId)) as {
|
||||
tab_id: number;
|
||||
key: string;
|
||||
};
|
||||
return { session: sessionId, tabId: reply.tab_id, key: reply.key };
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: cmdline(deps, ["press", args.key, "--session", args.session ?? "(current)"]),
|
||||
description: "Press a key",
|
||||
}),
|
||||
presentResult: presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_screenshot",
|
||||
description:
|
||||
"Capture a PNG screenshot of the session's active tab, or crop to a snapshot ref element. " +
|
||||
"Returns the image itself when the deployment supports image input, otherwise a file path.",
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
ref: {
|
||||
type: "string",
|
||||
description: "Snapshot ref (@e3) from the last snapshot/observe; crops to that element.",
|
||||
},
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
path: { type: "string", required: true },
|
||||
width: { type: "integer", required: true },
|
||||
height: { type: "integer", required: true },
|
||||
byteSize: { type: "integer", required: true },
|
||||
image: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
attachmentId: { type: "string", required: true },
|
||||
mediaType: { type: "string", required: true, const: "image/png" },
|
||||
bytes: { type: "integer", required: true },
|
||||
width: { type: "integer", required: true },
|
||||
height: { type: "integer", required: true },
|
||||
name: { type: "string" },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
render: (_args, value) => {
|
||||
const text =
|
||||
value.image !== undefined
|
||||
? `[session ${value.session}] screenshot of tab ${value.tabId} (${value.width}x${value.height}px)`
|
||||
: `[session ${value.session}] screenshot saved to ${value.path} (${value.width}x${value.height}px, ${value.byteSize} bytes) — this deployment cannot inline images; read the file to view it`;
|
||||
const blocks: ContentBlock[] = [{ type: "text", text }];
|
||||
if (value.image !== undefined) {
|
||||
blocks.push({
|
||||
type: "image",
|
||||
// Structurally an ImageAttachmentRef; branded without a runtime import.
|
||||
attachment: {
|
||||
attachmentId: value.image.attachmentId,
|
||||
mediaType: value.image.mediaType,
|
||||
bytes: value.image.bytes,
|
||||
width: value.image.width,
|
||||
height: value.image.height,
|
||||
...(value.image.name !== undefined ? { name: value.image.name } : {}),
|
||||
} as never,
|
||||
});
|
||||
}
|
||||
return blocks;
|
||||
},
|
||||
},
|
||||
isConcurrencySafe: () => true,
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, "browser_screenshot");
|
||||
const outPath = join(
|
||||
tmpdir(),
|
||||
`bsk-screenshot-${Date.now()}-${Math.random().toString(36).slice(2, 8)}.png`,
|
||||
);
|
||||
const cmdArgs = ["screenshot", "--session", sessionId, "--out", outPath];
|
||||
if (args.ref !== undefined) cmdArgs.push("--ref", args.ref);
|
||||
// The scratch PNG is only kept when it IS the model-facing artifact
|
||||
// (no attachment store to inline through); once the bytes are saved
|
||||
// into the store — and on every failure/abort path — it is deleted.
|
||||
let keepFile = false;
|
||||
let writtenPath: string | undefined;
|
||||
try {
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "screenshot", sessionId)) as {
|
||||
tab_id: number;
|
||||
width: number;
|
||||
height: number;
|
||||
path: string;
|
||||
byte_size: number;
|
||||
};
|
||||
writtenPath = reply.path;
|
||||
const data = await readFile(reply.path);
|
||||
if (exec.signal.aborted) throw abortError();
|
||||
const ref = await trySaveScreenshot(deps.ctx, exec, data, `screenshot-${sessionId}.png`);
|
||||
keepFile = ref === undefined;
|
||||
return {
|
||||
session: sessionId,
|
||||
tabId: reply.tab_id,
|
||||
path: reply.path,
|
||||
width: reply.width,
|
||||
height: reply.height,
|
||||
byteSize: reply.byte_size,
|
||||
...(ref !== undefined
|
||||
? {
|
||||
image: {
|
||||
attachmentId: String(ref.attachmentId),
|
||||
mediaType: "image/png" as const,
|
||||
bytes: ref.bytes,
|
||||
width: ref.width,
|
||||
height: ref.height,
|
||||
...(ref.name !== undefined ? { name: ref.name } : {}),
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
} finally {
|
||||
if (!keepFile) await unlink(writtenPath ?? outPath).catch(() => {});
|
||||
}
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: cmdline(deps, [
|
||||
"screenshot",
|
||||
"--session",
|
||||
args.session ?? "(current)",
|
||||
...(args.ref !== undefined ? ["--ref", args.ref] : []),
|
||||
]),
|
||||
description: "Capture a screenshot",
|
||||
}),
|
||||
presentResult: presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
|
||||
register(
|
||||
defineTool({
|
||||
name: "browser_emulate",
|
||||
description:
|
||||
"Apply (or clear) mobile device emulation on the session's active tab: viewport metrics, " +
|
||||
"user agent, and touch. Overrides are per-tab and not inherited by new tabs.",
|
||||
parameters: {
|
||||
session: SESSION_PARAM,
|
||||
device: {
|
||||
type: "string",
|
||||
enum: DEVICE_PRESETS,
|
||||
description: "Built-in device preset.",
|
||||
},
|
||||
width: { type: "integer", description: "Viewport width in CSS pixels (requires height)." },
|
||||
height: { type: "integer", description: "Viewport height in CSS pixels (requires width)." },
|
||||
mobile: {
|
||||
type: "boolean",
|
||||
description:
|
||||
"Emulate a mobile viewport. Requires width+height — the bsk daemon refuses " +
|
||||
"--mobile without viewport dimensions, so this flag cannot be used alone.",
|
||||
},
|
||||
off: { type: "boolean", description: "Clear every emulation override on the tab." },
|
||||
},
|
||||
output: {
|
||||
schema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
session: { type: "string", required: true },
|
||||
tabId: { type: "integer", required: true },
|
||||
cleared: { type: "boolean", required: true },
|
||||
},
|
||||
},
|
||||
render: (_args, value) => [
|
||||
{
|
||||
type: "text",
|
||||
text: value.cleared
|
||||
? `[session ${value.session}] cleared emulation on tab ${value.tabId}`
|
||||
: `[session ${value.session}] applied emulation on tab ${value.tabId}`,
|
||||
},
|
||||
],
|
||||
},
|
||||
async execute(args, exec) {
|
||||
const sessionId = registry.resolve(args.session, "browser_emulate");
|
||||
if (args.off === true) {
|
||||
if (args.device !== undefined || args.width !== undefined || args.height !== undefined) {
|
||||
throw new Error("off is mutually exclusive with device/width/height");
|
||||
}
|
||||
} else if (args.device === undefined && args.width === undefined) {
|
||||
throw new Error(
|
||||
"nothing to apply: pass device or width+height (mobile also requires width+height), or off",
|
||||
);
|
||||
}
|
||||
if ((args.width === undefined) !== (args.height === undefined)) {
|
||||
throw new Error("width and height must be given together");
|
||||
}
|
||||
const cmdArgs = ["emulate", "--session", sessionId];
|
||||
if (args.off === true) {
|
||||
cmdArgs.push("--off");
|
||||
} else {
|
||||
if (args.device !== undefined) cmdArgs.push("--device", args.device);
|
||||
if (args.width !== undefined && args.height !== undefined) {
|
||||
cmdArgs.push("--width", String(args.width), "--height", String(args.height));
|
||||
}
|
||||
if (args.mobile === true) cmdArgs.push("--mobile");
|
||||
}
|
||||
const reply = (await runBsk(deps, exec, cmdArgs, "emulate", sessionId)) as {
|
||||
tab_id: number;
|
||||
cleared: boolean;
|
||||
};
|
||||
return { session: sessionId, tabId: reply.tab_id, cleared: reply.cleared };
|
||||
},
|
||||
presentCall: (args) => ({
|
||||
card: "terminal",
|
||||
title: cmdline(deps, [
|
||||
"emulate",
|
||||
"--session",
|
||||
args.session ?? "(current)",
|
||||
...(args.off === true ? ["--off"] : []),
|
||||
...(args.device !== undefined ? ["--device", args.device] : []),
|
||||
]),
|
||||
description: "Emulate a device environment",
|
||||
}),
|
||||
presentResult: presentTerminalResult,
|
||||
}),
|
||||
);
|
||||
return () => {
|
||||
for (const dispose of disposers.splice(0)) dispose();
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,375 @@
|
||||
// @vitest-environment happy-dom
|
||||
// ObservationOverlay: visibility lifecycle, focus view, interrupt interaction,
|
||||
// hover tooltip, drag move/resize from every corner with clamps, collapse capsule, and PiP
|
||||
// upgrade/fallback (mocked documentPictureInPicture). NOTE: use RTL's waitFor
|
||||
// (act-flushing), not vi.waitFor, when asserting store-driven UI updates.
|
||||
|
||||
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { applyResize, ObservationOverlay } from "../../src/client/ObservationOverlay";
|
||||
import { type EventSourceLike, ObservationClientStore } from "../../src/client/observation-store";
|
||||
import type { SessionObservation } from "../../src/observation";
|
||||
|
||||
const BUSY: SessionObservation = { sessionId: "s1", action: "clicking", since: Date.now() - 7000 };
|
||||
|
||||
interface Harness {
|
||||
store: ObservationClientStore;
|
||||
es: () => EventSourceLike;
|
||||
push: (sessions: SessionObservation[]) => void;
|
||||
emitRaw: (event: unknown) => void;
|
||||
fetches: { url: string; init?: { body?: string } }[];
|
||||
loadImage: ReturnType<typeof vi.fn>;
|
||||
}
|
||||
|
||||
function makeHarness(initial: SessionObservation[]): Harness {
|
||||
const fetches: Harness["fetches"] = [];
|
||||
let es: EventSourceLike | undefined;
|
||||
let current = initial;
|
||||
const loadImage = vi.fn(async (id: string) => `blob:${id}`);
|
||||
const store = new ObservationClientStore({
|
||||
fetchFn: async (url: string, init?: { body?: string }) => {
|
||||
fetches.push({ url, init });
|
||||
if (url === "/bsk-observation/state") {
|
||||
return { ok: true, json: async () => ({ sessions: current }) };
|
||||
}
|
||||
return { ok: true, json: async () => ({ interrupted: true }) };
|
||||
},
|
||||
eventSourceFactory: () => {
|
||||
es = { onmessage: null, close: vi.fn() };
|
||||
return es;
|
||||
},
|
||||
loadImage,
|
||||
});
|
||||
return {
|
||||
store,
|
||||
es: () => {
|
||||
if (es === undefined) throw new Error("no EventSource yet");
|
||||
return es;
|
||||
},
|
||||
push: (sessions) => {
|
||||
current = sessions;
|
||||
for (const s of sessions) {
|
||||
es?.onmessage?.({ data: JSON.stringify({ type: "upsert", session: s }) });
|
||||
}
|
||||
if (sessions.length === 0) es?.onmessage?.({ data: JSON.stringify({ type: "reset" }) });
|
||||
},
|
||||
emitRaw: (event) => {
|
||||
es?.onmessage?.({ data: JSON.stringify(event) });
|
||||
},
|
||||
fetches,
|
||||
loadImage,
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(cleanup);
|
||||
|
||||
describe("ObservationOverlay", () => {
|
||||
beforeEach(() => {
|
||||
delete (window as unknown as Record<string, unknown>).documentPictureInPicture;
|
||||
});
|
||||
|
||||
it("renders nothing with no sessions, appears when one starts", async () => {
|
||||
const h = makeHarness([]);
|
||||
const { container } = render(<ObservationOverlay store={h.store} />);
|
||||
await waitFor(() => expect(h.es).not.toThrow());
|
||||
expect(container.firstChild).toBeNull();
|
||||
h.push([{ sessionId: "s1", action: "idle", since: Date.now() }]);
|
||||
await screen.findByText(/s1 · idle/);
|
||||
expect(screen.getByTestId("obs-card")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("shows the status row and a placeholder without a thumbnail", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
await screen.findByText(/s1 · clicking · 00:0/);
|
||||
expect(screen.getByText("waiting for page")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("loads and renders the thumbnail through the store", async () => {
|
||||
const h = makeHarness([{ ...BUSY, thumbnailAttachmentId: "att-9" }]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
await waitFor(() => expect(h.loadImage).toHaveBeenCalledWith("att-9"));
|
||||
await screen.findByRole("img", { name: "session s1 view" });
|
||||
});
|
||||
|
||||
it("keeps the current frame on stage while the next thumbnail loads", async () => {
|
||||
let resolveNext: ((url: string) => void) | undefined;
|
||||
const h = makeHarness([{ ...BUSY, thumbnailAttachmentId: "att-1" }]);
|
||||
h.loadImage.mockImplementation(async (id: string) => {
|
||||
if (id === "att-2") {
|
||||
return new Promise<string>((resolve) => {
|
||||
resolveNext = resolve;
|
||||
});
|
||||
}
|
||||
return `blob:${id}`;
|
||||
});
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
const img = await screen.findByRole("img", { name: "session s1 view" });
|
||||
expect(img.getAttribute("src")).toBe("blob:att-1");
|
||||
h.push([{ ...BUSY, thumbnailAttachmentId: "att-2" }]);
|
||||
await waitFor(() => expect(h.loadImage).toHaveBeenCalledWith("att-2"));
|
||||
// Same <img> node, same src — no placeholder flash, no remount/fade.
|
||||
expect(screen.queryByText("waiting for page")).toBeNull();
|
||||
expect(screen.getByRole("img", { name: "session s1 view" })).toBe(img);
|
||||
expect(img.getAttribute("src")).toBe("blob:att-1");
|
||||
resolveNext?.("blob:att-2");
|
||||
await waitFor(() => expect(img.getAttribute("src")).toBe("blob:att-2"));
|
||||
expect(screen.getByRole("img", { name: "session s1 view" })).toBe(img);
|
||||
});
|
||||
|
||||
it("interrupt: disabled while idle, active call posts and shows progress", async () => {
|
||||
const h = makeHarness([{ sessionId: "s1", action: "idle", since: Date.now() }]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
const button = await screen.findByRole("button", { name: /Interrupt the current/ });
|
||||
expect(button).toHaveProperty("disabled", true);
|
||||
h.push([BUSY]);
|
||||
await waitFor(() => expect(button).toHaveProperty("disabled", false));
|
||||
fireEvent.click(button);
|
||||
await waitFor(() =>
|
||||
expect(h.fetches.some((f) => f.url === "/bsk-observation/interrupt")).toBe(true),
|
||||
);
|
||||
});
|
||||
|
||||
it("shows a hover tooltip on the interrupt icon and keeps the button icon-only", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
const button = await screen.findByRole("button", { name: /Interrupt the current/ });
|
||||
expect(button.textContent ?? "").not.toMatch(/Interrupt/i);
|
||||
fireEvent.pointerEnter(button.parentElement as HTMLElement);
|
||||
const tip = await screen.findByRole("tooltip");
|
||||
expect(tip.textContent).toBe("Stop the current browser action.");
|
||||
fireEvent.pointerLeave(button.parentElement as HTMLElement);
|
||||
expect(screen.queryByRole("tooltip")).toBeNull();
|
||||
fireEvent.pointerEnter(button.parentElement as HTMLElement);
|
||||
expect(screen.getByRole("tooltip")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("collapses to a capsule and expands back", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
await screen.findByText(/s1 · clicking/);
|
||||
fireEvent.click(screen.getByRole("button", { name: "Collapse" }));
|
||||
const capsule = await screen.findByRole("button", {
|
||||
name: "Expand browser observation overlay",
|
||||
});
|
||||
expect(capsule.textContent).toContain("clicking");
|
||||
fireEvent.click(capsule);
|
||||
await screen.findByText(/s1 · clicking/);
|
||||
});
|
||||
|
||||
it("resizes within min/max clamps", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
const card = await screen.findByTestId("obs-card");
|
||||
card.getBoundingClientRect = () =>
|
||||
({
|
||||
x: 0,
|
||||
y: 0,
|
||||
left: 0,
|
||||
top: 0,
|
||||
width: 320,
|
||||
height: 240,
|
||||
right: 320,
|
||||
bottom: 240,
|
||||
toJSON: () => {},
|
||||
}) as DOMRect;
|
||||
expect(Number.parseFloat(card.style.width)).toBe(320);
|
||||
fireEvent.pointerDown(screen.getByTestId("obs-resize-se"), { clientX: 100, clientY: 100 });
|
||||
fireEvent.pointerMove(document, { clientX: 0, clientY: 0 });
|
||||
fireEvent.pointerUp(document);
|
||||
expect(Number.parseFloat(card.style.width)).toBe(240); // min width clamp
|
||||
fireEvent.pointerDown(screen.getByTestId("obs-resize-se"), { clientX: 100, clientY: 100 });
|
||||
fireEvent.pointerMove(document, { clientX: 5000, clientY: 5000 });
|
||||
fireEvent.pointerUp(document);
|
||||
expect(Number.parseFloat(card.style.width)).toBe(window.innerWidth * 0.8);
|
||||
});
|
||||
|
||||
it("exposes an invisible resize hit target on every corner", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
await screen.findByTestId("obs-card");
|
||||
for (const corner of ["nw", "ne", "sw", "se"]) {
|
||||
expect(screen.getByTestId(`obs-resize-${corner}`)).toBeTruthy();
|
||||
}
|
||||
});
|
||||
|
||||
it("moves the card with header drags, clamped to the viewport", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
const card = await screen.findByTestId("obs-card");
|
||||
fireEvent.pointerDown(screen.getByTestId("obs-header"), { clientX: 50, clientY: 50 });
|
||||
fireEvent.pointerMove(document, { clientX: -500, clientY: -500 });
|
||||
fireEvent.pointerUp(document);
|
||||
expect(card.style.left).toBe("0px");
|
||||
expect(card.style.top).toBe("0px");
|
||||
});
|
||||
|
||||
it("hides Pop out when Document PiP is unsupported", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
await screen.findByText(/s1 · clicking/);
|
||||
expect(screen.queryByRole("button", { name: /Pop out/ })).toBeNull();
|
||||
});
|
||||
|
||||
it("pops out with the card's current size and falls back on pagehide", async () => {
|
||||
const h = makeHarness([BUSY]);
|
||||
const pipDoc = document.implementation.createHTMLDocument("pip");
|
||||
const listeners = new Map<string, (() => void)[]>();
|
||||
const requestWindow = vi.fn(async (options?: { width?: number; height?: number }) => {
|
||||
void options;
|
||||
return {
|
||||
document: pipDoc,
|
||||
addEventListener: (name: string, fn: () => void) => {
|
||||
listeners.set(name, [...(listeners.get(name) ?? []), fn]);
|
||||
},
|
||||
} as unknown as Window;
|
||||
});
|
||||
(window as unknown as Record<string, unknown>).documentPictureInPicture = { requestWindow };
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
const popout = await screen.findByRole("button", { name: /Pop out/ });
|
||||
fireEvent.click(popout);
|
||||
await waitFor(() => expect(requestWindow).toHaveBeenCalledWith({ width: 320, height: 240 }));
|
||||
// Content now renders inside the PiP document.
|
||||
await waitFor(() => expect(pipDoc.body.textContent).toContain("s1"));
|
||||
// Closing the PiP returns to the in-page card with state preserved.
|
||||
for (const fn of listeners.get("pagehide") ?? []) fn();
|
||||
await screen.findByTestId("obs-card");
|
||||
});
|
||||
|
||||
it("renders the strip for two sessions and pins focus on click", async () => {
|
||||
const older: SessionObservation = {
|
||||
sessionId: "s1",
|
||||
action: "idle",
|
||||
since: Date.now() - 60000,
|
||||
};
|
||||
const newer: SessionObservation = { sessionId: "s2", action: "clicking", since: Date.now() };
|
||||
const h = makeHarness([older, newer]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
// Auto-follows the most recently active session.
|
||||
await screen.findByText(/s2 · clicking/);
|
||||
const strip = screen.getByTestId("obs-strip");
|
||||
expect(strip.textContent).toContain("s1");
|
||||
expect(strip.textContent).toContain("s2");
|
||||
// Pin s1: focus moves and stays even when s2 gets newer activity.
|
||||
fireEvent.click(screen.getByRole("button", { name: "Pin session s1" }));
|
||||
await screen.findByText(/s1 · idle/);
|
||||
h.push([{ ...newer, since: Date.now() + 5000, action: "filling" }]);
|
||||
await waitFor(() => expect(screen.queryByText(/s2 · filling/)).toBeNull());
|
||||
expect(screen.getByText(/s1 · idle/)).toBeTruthy();
|
||||
// Unpin: auto-follow resumes.
|
||||
fireEvent.click(screen.getByRole("button", { name: "Unpin session s1" }));
|
||||
await screen.findByText(/s2 · filling/);
|
||||
});
|
||||
|
||||
it("flags an errored strip item without stealing focus", async () => {
|
||||
const s1: SessionObservation = { sessionId: "s1", action: "idle", since: Date.now() - 30000 };
|
||||
const s2: SessionObservation = { sessionId: "s2", action: "idle", since: Date.now() - 10000 };
|
||||
const h = makeHarness([s1, s2]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
await screen.findByText(/s2 · idle/);
|
||||
// s2's latest action failed: red edge on its strip item, focus stays on s2 already…
|
||||
// now push an even-more-recent ERROR for s1 — focus must NOT jump to it.
|
||||
h.push([
|
||||
{ ...s2, since: Date.now() - 5000 },
|
||||
{ ...s1, since: Date.now(), lastError: "click failed" },
|
||||
]);
|
||||
await waitFor(() =>
|
||||
expect(
|
||||
screen.getByRole("button", { name: "Pin session s1" }).closest("[data-state]"),
|
||||
).toHaveProperty("dataset", expect.objectContaining({ state: "error" })),
|
||||
);
|
||||
expect(screen.getByText(/s2 · idle/)).toBeTruthy();
|
||||
});
|
||||
|
||||
it("shows browser-unavailable with a kept frame and a greyed interrupt", async () => {
|
||||
const h = makeHarness([{ ...BUSY, thumbnailAttachmentId: "att-1" }]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
await waitFor(() => expect(h.loadImage).toHaveBeenCalledWith("att-1"));
|
||||
await screen.findByRole("img", { name: "session s1 view" });
|
||||
h.emitRaw({ type: "availability", available: false });
|
||||
await screen.findByText("browser unavailable");
|
||||
// Last frame stays on stage; interrupt is greyed.
|
||||
expect(screen.getByRole("img", { name: "session s1 view" })).toBeTruthy();
|
||||
expect(screen.getByRole("button", { name: /Interrupt the current/ })).toHaveProperty(
|
||||
"disabled",
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it("greys out a dead session in the strip and skips it for focus", async () => {
|
||||
const s1: SessionObservation = { sessionId: "s1", action: "idle", since: Date.now() - 20000 };
|
||||
const s2: SessionObservation = { sessionId: "s2", action: "idle", since: Date.now() - 10000 };
|
||||
const h = makeHarness([s1, s2]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
await screen.findByText(/s2 · idle/);
|
||||
h.push([{ ...s1, dead: true, since: Date.now() }, s2]);
|
||||
await waitFor(() =>
|
||||
expect(
|
||||
screen.getByRole("button", { name: "Pin session s1" }).closest("[data-state]"),
|
||||
).toHaveProperty("dataset", expect.objectContaining({ state: "dead" })),
|
||||
);
|
||||
// Dead sessions never take the stage.
|
||||
expect(screen.getByText(/s2 · idle/)).toBeTruthy();
|
||||
});
|
||||
|
||||
it("interrupts a strip session directly on hover without focusing it", async () => {
|
||||
const s1: SessionObservation = {
|
||||
sessionId: "s1",
|
||||
action: "clicking",
|
||||
since: Date.now() - 30000,
|
||||
};
|
||||
const s2: SessionObservation = { sessionId: "s2", action: "idle", since: Date.now() };
|
||||
const h = makeHarness([s1, s2]);
|
||||
render(<ObservationOverlay store={h.store} />);
|
||||
await screen.findByText(/s2 · idle/);
|
||||
const item = screen.getByRole("button", { name: "Pin session s1" }).closest("div");
|
||||
if (item === null) throw new Error("no strip item");
|
||||
fireEvent.pointerEnter(item);
|
||||
const mini = await screen.findByRole("button", { name: "Interrupt session s1" });
|
||||
fireEvent.click(mini);
|
||||
await waitFor(() =>
|
||||
expect(
|
||||
h.fetches.some(
|
||||
(f) => f.url === "/bsk-observation/interrupt" && f.init?.body === '{"sessionId":"s1"}',
|
||||
),
|
||||
).toBe(true),
|
||||
);
|
||||
// Focus did not move to s1.
|
||||
expect(screen.getByText(/s2 · idle/)).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
describe("applyResize", () => {
|
||||
const base = { x: 200, y: 100, w: 320, h: 240 };
|
||||
const viewport = { w: 1000, h: 800 };
|
||||
|
||||
it("grows the planted opposite corner from each handle", () => {
|
||||
expect(applyResize(base, "se", 20, 10, viewport)).toEqual({
|
||||
pos: { x: 200, y: 100 },
|
||||
size: { w: 340, h: 250 },
|
||||
});
|
||||
expect(applyResize(base, "nw", -20, -10, viewport)).toEqual({
|
||||
pos: { x: 180, y: 90 },
|
||||
size: { w: 340, h: 250 },
|
||||
});
|
||||
expect(applyResize(base, "ne", 20, -10, viewport)).toEqual({
|
||||
pos: { x: 200, y: 90 },
|
||||
size: { w: 340, h: 250 },
|
||||
});
|
||||
expect(applyResize(base, "sw", -20, 10, viewport)).toEqual({
|
||||
pos: { x: 180, y: 100 },
|
||||
size: { w: 340, h: 250 },
|
||||
});
|
||||
});
|
||||
|
||||
it("clamps to the minimum size without drifting the planted corner", () => {
|
||||
expect(applyResize(base, "se", -200, -200, viewport)).toEqual({
|
||||
pos: { x: 200, y: 100 },
|
||||
size: { w: 240, h: 180 },
|
||||
});
|
||||
expect(applyResize(base, "nw", 200, 200, viewport)).toEqual({
|
||||
pos: { x: 280, y: 160 },
|
||||
size: { w: 240, h: 180 },
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,246 @@
|
||||
// ObservationClientStore: initial fetch + SSE application, thumbnail loading
|
||||
// lifecycle, interrupt wire shape. All I/O faked.
|
||||
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { type EventSourceLike, ObservationClientStore } from "../../src/client/observation-store";
|
||||
import type { ObservationEvent, SessionObservation } from "../../src/observation";
|
||||
|
||||
const OBS_IDLE: SessionObservation = { sessionId: "s1", action: "idle", since: 1000 };
|
||||
const OBS_BUSY: SessionObservation = { sessionId: "s1", action: "clicking", since: 2000 };
|
||||
|
||||
function harness(state: SessionObservation[] = []) {
|
||||
const fetches: {
|
||||
url: string;
|
||||
init?: { method?: string; body?: string; headers?: Record<string, string> };
|
||||
}[] = [];
|
||||
const esInstances: EventSourceLike[] = [];
|
||||
const fetchFn = vi.fn(
|
||||
async (
|
||||
url: string,
|
||||
init?: { method?: string; body?: string; headers?: Record<string, string> },
|
||||
) => {
|
||||
fetches.push({ url, init });
|
||||
if (url === "/bsk-observation/state") {
|
||||
return { ok: true, json: async () => ({ sessions: state }) };
|
||||
}
|
||||
return { ok: true, json: async () => ({ interrupted: true }) };
|
||||
},
|
||||
);
|
||||
const eventSourceFactory = (url: string) => {
|
||||
const es: EventSourceLike = { onmessage: null, close: vi.fn() };
|
||||
esInstances.push(es);
|
||||
return es;
|
||||
};
|
||||
const loadImage = vi.fn(async (id: string) => `blob:url-${id}`);
|
||||
const store = new ObservationClientStore({ fetchFn, eventSourceFactory, loadImage });
|
||||
return { store, fetches, esInstances, loadImage };
|
||||
}
|
||||
|
||||
function emit(es: EventSourceLike, event: ObservationEvent): void {
|
||||
es.onmessage?.({ data: JSON.stringify(event) });
|
||||
}
|
||||
|
||||
describe("ObservationClientStore", () => {
|
||||
it("loads the initial state and applies SSE increments", async () => {
|
||||
const { store, esInstances } = harness([OBS_IDLE]);
|
||||
const seen: number[] = [];
|
||||
store.subscribe(() => seen.push(store.getSnapshot().sessions.length));
|
||||
store.start();
|
||||
await vi.waitFor(() => expect(store.getSnapshot().sessions).toHaveLength(1));
|
||||
expect(esInstances).toHaveLength(1);
|
||||
emit(esInstances[0], { type: "upsert", session: OBS_BUSY });
|
||||
expect(store.getSnapshot().sessions[0].action).toBe("clicking");
|
||||
emit(esInstances[0], {
|
||||
type: "upsert",
|
||||
session: { sessionId: "s2", action: "idle", since: 3 },
|
||||
});
|
||||
expect(store.getSnapshot().sessions).toHaveLength(2);
|
||||
emit(esInstances[0], {
|
||||
type: "remove",
|
||||
session: { sessionId: "s2", action: "idle", since: 0 },
|
||||
});
|
||||
expect(store.getSnapshot().sessions).toHaveLength(1);
|
||||
emit(esInstances[0], { type: "reset" });
|
||||
expect(store.getSnapshot().sessions).toHaveLength(0);
|
||||
expect(seen.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("ignores malformed SSE frames", () => {
|
||||
const { store, esInstances } = harness([]);
|
||||
store.start();
|
||||
esInstances[0].onmessage?.({ data: "not json" });
|
||||
expect(store.getSnapshot().sessions).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("loads each thumbnail once and reports ready", async () => {
|
||||
const { store, loadImage } = harness();
|
||||
store.ensureThumbnail("att-1");
|
||||
store.ensureThumbnail("att-1");
|
||||
expect(loadImage).toHaveBeenCalledTimes(1);
|
||||
await vi.waitFor(() =>
|
||||
expect(store.getSnapshot().thumbnails["att-1"]).toEqual({
|
||||
status: "ready",
|
||||
url: "blob:url-att-1",
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("marks failed thumbnail loads as error", async () => {
|
||||
const { store, loadImage } = harness();
|
||||
loadImage.mockRejectedValueOnce(new Error("nope"));
|
||||
store.ensureThumbnail("att-bad");
|
||||
await vi.waitFor(() => expect(store.getSnapshot().thumbnails["att-bad"].status).toBe("error"));
|
||||
});
|
||||
|
||||
it("posts interrupt with and without a session id", async () => {
|
||||
const { store, fetches } = harness();
|
||||
expect(await store.interrupt()).toBe(true);
|
||||
expect(await store.interrupt("s1")).toBe(true);
|
||||
const calls = fetches.filter((f) => f.url === "/bsk-observation/interrupt");
|
||||
expect(calls[0].init?.headers?.["content-type"]).toBe("application/json");
|
||||
expect(JSON.parse(calls[0].init?.body ?? "")).toEqual({});
|
||||
expect(calls[1].init?.headers?.["content-type"]).toBe("application/json");
|
||||
expect(JSON.parse(calls[1].init?.body ?? "")).toEqual({ sessionId: "s1" });
|
||||
});
|
||||
|
||||
it("returns false when interrupt fails", async () => {
|
||||
const failing = new ObservationClientStore({
|
||||
fetchFn: async () => ({ ok: false, json: async () => ({}) }),
|
||||
eventSourceFactory: () => ({ onmessage: null, close: () => {} }),
|
||||
loadImage: async () => "blob:x",
|
||||
});
|
||||
expect(await failing.interrupt("s1")).toBe(false);
|
||||
});
|
||||
|
||||
it("stop closes the stream and clears state", async () => {
|
||||
const { store, esInstances } = harness([OBS_IDLE]);
|
||||
store.start();
|
||||
await vi.waitFor(() => expect(store.getSnapshot().sessions).toHaveLength(1));
|
||||
store.stop();
|
||||
expect(esInstances[0].close).toHaveBeenCalled();
|
||||
expect(store.getSnapshot().sessions).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("thumbnail blob URL lifecycle", () => {
|
||||
function withRevokeSpy() {
|
||||
const revoke = vi.fn();
|
||||
(URL as unknown as { revokeObjectURL: unknown }).revokeObjectURL = revoke;
|
||||
return revoke;
|
||||
}
|
||||
|
||||
it("holds the last ready frame until the replacement decodes", async () => {
|
||||
const revoke = withRevokeSpy();
|
||||
const { store, esInstances } = harness([{ ...OBS_IDLE, thumbnailAttachmentId: "a1" }]);
|
||||
store.start();
|
||||
await vi.waitFor(() => expect(store.getSnapshot().sessions).toHaveLength(1));
|
||||
store.ensureThumbnail("a1");
|
||||
await vi.waitFor(() =>
|
||||
expect(store.getSnapshot().thumbnails["a1"]).toEqual({ status: "ready", url: "blob:url-a1" }),
|
||||
);
|
||||
emit(esInstances[0], {
|
||||
type: "upsert",
|
||||
session: { ...OBS_IDLE, thumbnailAttachmentId: "a2" },
|
||||
});
|
||||
// Old blob stays painted so the overlay does not flash a placeholder.
|
||||
expect(store.getSnapshot().thumbnails["a1"]).toEqual({
|
||||
status: "ready",
|
||||
url: "blob:url-a1",
|
||||
});
|
||||
expect(store.getSnapshot().displayFrames.s1).toEqual({
|
||||
status: "ready",
|
||||
url: "blob:url-a1",
|
||||
});
|
||||
expect(revoke).not.toHaveBeenCalled();
|
||||
store.ensureThumbnail("a2");
|
||||
await vi.waitFor(() =>
|
||||
expect(store.getSnapshot().displayFrames.s1).toEqual({
|
||||
status: "ready",
|
||||
url: "blob:url-a2",
|
||||
}),
|
||||
);
|
||||
expect(store.getSnapshot().thumbnails["a1"]).toBeUndefined();
|
||||
expect(revoke).toHaveBeenCalledWith("blob:url-a1");
|
||||
store.stop();
|
||||
});
|
||||
|
||||
it("keeps the last good frame in displayFrames when the next load fails", async () => {
|
||||
const { store, esInstances, loadImage } = harness([
|
||||
{ ...OBS_IDLE, thumbnailAttachmentId: "a1" },
|
||||
]);
|
||||
store.start();
|
||||
await vi.waitFor(() => expect(store.getSnapshot().sessions).toHaveLength(1));
|
||||
store.ensureThumbnail("a1");
|
||||
await vi.waitFor(() => expect(store.getSnapshot().thumbnails["a1"].status).toBe("ready"));
|
||||
loadImage.mockRejectedValueOnce(new Error("nope"));
|
||||
emit(esInstances[0], {
|
||||
type: "upsert",
|
||||
session: { ...OBS_IDLE, thumbnailAttachmentId: "a2" },
|
||||
});
|
||||
store.ensureThumbnail("a2");
|
||||
await vi.waitFor(() => expect(store.getSnapshot().thumbnails["a2"]?.status).toBe("error"));
|
||||
expect(store.getSnapshot().displayFrames.s1).toEqual({
|
||||
status: "error",
|
||||
url: "blob:url-a1",
|
||||
});
|
||||
store.stop();
|
||||
});
|
||||
|
||||
it("remove and reset revoke the session's tracked frame", async () => {
|
||||
const revoke = withRevokeSpy();
|
||||
const { store, esInstances } = harness([{ ...OBS_IDLE, thumbnailAttachmentId: "a1" }]);
|
||||
store.start();
|
||||
await vi.waitFor(() => expect(store.getSnapshot().sessions).toHaveLength(1));
|
||||
store.ensureThumbnail("a1");
|
||||
await vi.waitFor(() => expect(store.getSnapshot().thumbnails["a1"].status).toBe("ready"));
|
||||
emit(esInstances[0], {
|
||||
type: "remove",
|
||||
session: { sessionId: "s1", action: "idle", since: 0 },
|
||||
});
|
||||
expect(revoke).toHaveBeenCalledWith("blob:url-a1");
|
||||
expect(store.getSnapshot().thumbnails["a1"]).toBeUndefined();
|
||||
store.stop();
|
||||
});
|
||||
|
||||
it("a load that finishes after replacement never resurrects the old frame", async () => {
|
||||
const revoke = withRevokeSpy();
|
||||
const { store, esInstances, loadImage } = harness([
|
||||
{ ...OBS_IDLE, thumbnailAttachmentId: "a1" },
|
||||
]);
|
||||
let resolveA1!: (url: string) => void;
|
||||
loadImage.mockImplementation((id: string) =>
|
||||
id === "a1"
|
||||
? new Promise<string>((resolve) => {
|
||||
resolveA1 = resolve;
|
||||
})
|
||||
: Promise.resolve(`blob:url-${id}`),
|
||||
);
|
||||
store.start();
|
||||
await vi.waitFor(() => expect(store.getSnapshot().sessions).toHaveLength(1));
|
||||
store.ensureThumbnail("a1");
|
||||
// the frame is replaced while a1 is still loading
|
||||
emit(esInstances[0], {
|
||||
type: "upsert",
|
||||
session: { ...OBS_IDLE, thumbnailAttachmentId: "a2" },
|
||||
});
|
||||
resolveA1("blob:url-a1-late");
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(store.getSnapshot().thumbnails["a1"]).toBeUndefined();
|
||||
expect(revoke).toHaveBeenCalledWith("blob:url-a1-late");
|
||||
store.stop();
|
||||
});
|
||||
});
|
||||
|
||||
describe("SSE (re)open resync", () => {
|
||||
it("refetches the full state when the stream (re)opens", async () => {
|
||||
const { store, fetches, esInstances } = harness([OBS_IDLE]);
|
||||
store.start();
|
||||
await vi.waitFor(() => expect(store.getSnapshot().sessions).toHaveLength(1));
|
||||
const before = fetches.filter((f) => f.url === "/bsk-observation/state").length;
|
||||
esInstances[0].onopen?.();
|
||||
await vi.waitFor(() =>
|
||||
expect(fetches.filter((f) => f.url === "/bsk-observation/state").length).toBe(before + 1),
|
||||
);
|
||||
store.stop();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,154 @@
|
||||
// @vitest-environment happy-dom
|
||||
// browser_screenshot keyed toolview: view-model derivation from frozen blocks,
|
||||
// image/path-only/running/error rendering, the registration key, and the
|
||||
// session-bound attachment loader.
|
||||
|
||||
import type { RunningToolCall, ToolResultNode } from "@deepseek-ai/dsh-client-runtime/client";
|
||||
import { cleanup, render, screen, waitFor } from "@testing-library/react";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { apply } from "../../src/client/index";
|
||||
import {
|
||||
ScreenshotToolView,
|
||||
type ScreenshotToolViewProps,
|
||||
viewModelOf,
|
||||
} from "../../src/client/ScreenshotToolView";
|
||||
|
||||
const ATTACHMENT = {
|
||||
attachmentId: "sha256:abc123",
|
||||
mediaType: "image/png",
|
||||
bytes: 8,
|
||||
width: 800,
|
||||
height: 457,
|
||||
name: "screenshot-s1.png",
|
||||
} as never;
|
||||
|
||||
function runningBlock(argsRaw: string): RunningToolCall {
|
||||
return {
|
||||
callId: "c1",
|
||||
name: "browser_screenshot",
|
||||
argsRaw,
|
||||
turn: 1,
|
||||
step: 1,
|
||||
time: 0,
|
||||
callView: null,
|
||||
subCalls: [],
|
||||
} as never;
|
||||
}
|
||||
|
||||
function settledBlock(argsRaw: string, content: unknown[], isError = false): ToolResultNode {
|
||||
return {
|
||||
kind: "tool-result",
|
||||
seq: 1,
|
||||
time: 0,
|
||||
callId: "c1",
|
||||
call: { name: "browser_screenshot", argsRaw },
|
||||
callTime: 0,
|
||||
content,
|
||||
isError,
|
||||
callView: null,
|
||||
resultView: null,
|
||||
subCalls: [],
|
||||
} as never;
|
||||
}
|
||||
|
||||
const IMAGE_TEXT = "[session s1] screenshot of tab 7 (800x457px)";
|
||||
const PATH_TEXT = "[session s1] screenshot saved to /tmp/shot.png (800x457px, 8 bytes)";
|
||||
|
||||
afterEach(cleanup);
|
||||
|
||||
describe("viewModelOf", () => {
|
||||
it("derives the running model from the call frame", () => {
|
||||
const model = viewModelOf(runningBlock('{"session":"s1"}'));
|
||||
expect(model.state).toBe("running");
|
||||
expect(model.command).toBe("bsk screenshot --session s1");
|
||||
expect(model.image).toBeNull();
|
||||
});
|
||||
|
||||
it("marks settled results carrying an image block", () => {
|
||||
const block = settledBlock("{}", [
|
||||
{ type: "text", text: IMAGE_TEXT },
|
||||
{ type: "image", attachment: ATTACHMENT },
|
||||
]);
|
||||
const model = viewModelOf(block);
|
||||
expect(model.state).toBe("ok");
|
||||
expect(model.image).toBe(ATTACHMENT);
|
||||
expect(model.output).toBe(IMAGE_TEXT);
|
||||
expect(model.command).toBe("bsk screenshot --session (current)");
|
||||
});
|
||||
|
||||
it("keeps the path-only form image-free", () => {
|
||||
const model = viewModelOf(settledBlock("{}", [{ type: "text", text: PATH_TEXT }]));
|
||||
expect(model.state).toBe("ok");
|
||||
expect(model.image).toBeNull();
|
||||
expect(model.summary).toBe(PATH_TEXT);
|
||||
});
|
||||
|
||||
it("marks error results", () => {
|
||||
const model = viewModelOf(settledBlock("{}", [{ type: "text", text: "Error: boom" }], true));
|
||||
expect(model.state).toBe("error");
|
||||
});
|
||||
});
|
||||
|
||||
describe("ScreenshotToolView", () => {
|
||||
const baseProps = { callId: "c1", toolName: "browser_screenshot", openFile: () => {} };
|
||||
const renderView = (
|
||||
block: ScreenshotToolViewProps["block"],
|
||||
loadImage: ScreenshotToolViewProps["loadImage"],
|
||||
) => {
|
||||
const props = { ...baseProps, block, loadImage } as unknown as ScreenshotToolViewProps;
|
||||
return render(<ScreenshotToolView {...props} />);
|
||||
};
|
||||
|
||||
it("renders the screenshot image through the loader", async () => {
|
||||
const block = settledBlock("{}", [
|
||||
{ type: "text", text: IMAGE_TEXT },
|
||||
{ type: "image", attachment: ATTACHMENT },
|
||||
]);
|
||||
const loadImage = vi.fn(async () => "blob:mock-url");
|
||||
renderView(block, loadImage);
|
||||
// Collapsed row shows the summary; expand to reach the image.
|
||||
expect(screen.getByText(IMAGE_TEXT)).toBeTruthy();
|
||||
const toggle = screen.getByRole("button", { name: /screenshot/i });
|
||||
toggle.click();
|
||||
await waitFor(() => expect(loadImage).toHaveBeenCalledWith(ATTACHMENT));
|
||||
await screen.findByRole("img", { name: "screenshot-s1.png" });
|
||||
});
|
||||
|
||||
it("renders the path-only form without touching the loader", async () => {
|
||||
const block = settledBlock("{}", [{ type: "text", text: PATH_TEXT }]);
|
||||
const loadImage = vi.fn(async () => "blob:unused");
|
||||
renderView(block, loadImage);
|
||||
screen.getByRole("button", { name: /screenshot/i }).click();
|
||||
await screen.findByText(PATH_TEXT, { exact: false });
|
||||
expect(loadImage).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("renders the running form without output", () => {
|
||||
const loadImage = vi.fn();
|
||||
renderView(runningBlock("{}"), loadImage);
|
||||
expect(screen.getByText("bsk screenshot --session (current)")).toBeTruthy();
|
||||
expect(loadImage).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("client plugin registration", () => {
|
||||
it("registers the browser_screenshot toolview and the shell observation overlay", () => {
|
||||
const registrations: { name: string; key?: string; id?: string }[] = [];
|
||||
const sessions = { binding: () => undefined };
|
||||
const ctx = {
|
||||
get: (key: string) => (key === "sessions" ? sessions : undefined),
|
||||
slots: {
|
||||
inject: (_name: string, fn: () => unknown) => fn(),
|
||||
register: (slot: { name: string; key?: string; id?: string }, view: unknown) => {
|
||||
registrations.push(slot);
|
||||
expect(typeof view).toBe("function");
|
||||
},
|
||||
},
|
||||
};
|
||||
apply(ctx as never);
|
||||
expect(registrations).toEqual([
|
||||
{ name: "tool.call.toolview", key: "browser_screenshot" },
|
||||
{ name: "shell.overlay", id: "bsk-observation" },
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,94 @@
|
||||
// Unload-cleanup ownership discipline: dispose must stop exactly the sessions
|
||||
// this plugin created — never referenced/foreign sessions on the shared
|
||||
// daemon — and a stale handle (already stopped elsewhere, daemon restarted)
|
||||
// must not abort the remaining stops.
|
||||
|
||||
import type { ToolDefinition, ToolRunContext } from "@deepseek-ai/dsh-tools";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { apply } from "../src/index";
|
||||
import type { BskRunOptions, BskRunResult } from "../src/runner";
|
||||
|
||||
interface FakeCall {
|
||||
args: string[];
|
||||
}
|
||||
|
||||
function ok(payload: unknown): BskRunResult {
|
||||
return { code: 0, stdout: JSON.stringify(payload), stderr: "", timedOut: false, aborted: false };
|
||||
}
|
||||
|
||||
function fail(message: string): BskRunResult {
|
||||
return {
|
||||
code: 4,
|
||||
stdout: JSON.stringify({ code: "session_not_found", message }),
|
||||
stderr: "",
|
||||
timedOut: false,
|
||||
aborted: false,
|
||||
};
|
||||
}
|
||||
|
||||
function makeExec(): ToolRunContext {
|
||||
return {
|
||||
callId: "t",
|
||||
name: "t",
|
||||
signal: new AbortController().signal,
|
||||
} as unknown as ToolRunContext;
|
||||
}
|
||||
|
||||
describe("dispose cleanup ownership", () => {
|
||||
it("stops only plugin-owned sessions and tolerates stale handles", async () => {
|
||||
const calls: FakeCall[] = [];
|
||||
const startReplies = [
|
||||
{ session_id: "own1", browser_instance_id: "b1" },
|
||||
{ session_id: "own2", browser_instance_id: "b1" },
|
||||
];
|
||||
const runner = {
|
||||
async run(args: string[], _options: BskRunOptions = {}): Promise<BskRunResult> {
|
||||
calls.push({ args });
|
||||
const joined = args.join(" ");
|
||||
if (joined.startsWith("status")) return ok({});
|
||||
if (joined.startsWith("session start")) return ok(startReplies.shift());
|
||||
if (joined.startsWith("snapshot")) {
|
||||
return ok({ text: "x", ref_count: 1, tab_id: 7, truncated: false });
|
||||
}
|
||||
if (joined === "session stop own1") return ok({});
|
||||
if (joined === "session stop own2") {
|
||||
// Stale handle: already stopped by someone else / daemon restarted.
|
||||
return fail("session not found");
|
||||
}
|
||||
return fail(`unexpected command: ${joined}`);
|
||||
},
|
||||
killAll() {},
|
||||
killFor: () => 0,
|
||||
};
|
||||
const tools = new Map<string, ToolDefinition>();
|
||||
const disposers: Array<() => Promise<void>> = [];
|
||||
const ctx = {
|
||||
tools: { register: (def: ToolDefinition) => tools.set(def.name, def) },
|
||||
get: () => undefined,
|
||||
// No webServer in this harness: the inject callback never runs.
|
||||
inject: () => {},
|
||||
effect: (fn: () => () => Promise<void>) => {
|
||||
disposers.push(fn());
|
||||
},
|
||||
};
|
||||
apply(ctx as never, { maxSessions: 5, lazyTools: false }, { runnerFactory: () => runner });
|
||||
|
||||
const start = tools.get("browser_session_start");
|
||||
const snapshot = tools.get("browser_snapshot");
|
||||
if (start === undefined || snapshot === undefined) throw new Error("tools not registered");
|
||||
await start.execute({}, makeExec());
|
||||
await start.execute({}, makeExec());
|
||||
// A foreign session cannot even be referenced any more.
|
||||
await expect(snapshot.execute({ session: "ext9" }, makeExec())).rejects.toThrow(
|
||||
/does not belong to this plugin/,
|
||||
);
|
||||
|
||||
await Promise.all(disposers.map((dispose) => dispose()));
|
||||
|
||||
const stops = calls
|
||||
.map((call) => call.args.join(" "))
|
||||
.filter((joined) => joined.startsWith("session stop"));
|
||||
expect(stops.sort()).toEqual(["session stop own1", "session stop own2"]);
|
||||
expect(stops.some((joined) => joined.includes("ext9"))).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,229 @@
|
||||
// Lazy tool-schema injection: suite hidden before a successful skill
|
||||
// invocation, revealed by the model trigger / user gesture / session history,
|
||||
// idempotent on repeats, and torn down cleanly. Plus the apply-level
|
||||
// lazyTools two-state wiring.
|
||||
|
||||
import type { ToolDefinition, ToolRunContext } from "@deepseek-ai/dsh-tools";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { apply } from "../src/index";
|
||||
import { armLazyTools, hasSuccessfulSkillInvocation } from "../src/lazy-tools";
|
||||
import type { BskRunOptions, BskRunResult } from "../src/runner";
|
||||
|
||||
function fakeEventCtx(sessions?: { list(): { events: unknown[] }[] }) {
|
||||
const listeners = new Map<string, (...args: never[]) => void>();
|
||||
const ctx = {
|
||||
on: (event: string, listener: (...args: never[]) => void) => {
|
||||
listeners.set(event, listener);
|
||||
return () => listeners.delete(event);
|
||||
},
|
||||
get: (key: string) => (key === "sessions" ? sessions : undefined),
|
||||
};
|
||||
return { ctx: ctx as never, listeners };
|
||||
}
|
||||
|
||||
function callListeners(
|
||||
listeners: Map<string, (...args: never[]) => void>,
|
||||
event: string,
|
||||
...args: unknown[]
|
||||
): void {
|
||||
(listeners.get(event) as ((...a: unknown[]) => void) | undefined)?.(...args);
|
||||
}
|
||||
|
||||
describe("armLazyTools", () => {
|
||||
it("keeps the suite hidden until a successful skill invocation reveals it", () => {
|
||||
const { ctx, listeners } = fakeEventCtx();
|
||||
const registerSuite = vi.fn(() => () => {});
|
||||
armLazyTools(ctx, registerSuite);
|
||||
expect(registerSuite).not.toHaveBeenCalled();
|
||||
|
||||
// wrong tool
|
||||
callListeners(listeners, "tools/result", { name: "bash", arguments: {} }, { isError: false });
|
||||
// wrong skill name
|
||||
callListeners(
|
||||
listeners,
|
||||
"tools/result",
|
||||
{ name: "skill", arguments: { name: "other-skill" } },
|
||||
{ isError: false },
|
||||
);
|
||||
// failed invocation
|
||||
callListeners(
|
||||
listeners,
|
||||
"tools/result",
|
||||
{ name: "skill", arguments: { name: "browser-skill" } },
|
||||
{ isError: true },
|
||||
);
|
||||
expect(registerSuite).not.toHaveBeenCalled();
|
||||
|
||||
// the real trigger (arguments may arrive as a raw JSON string)
|
||||
callListeners(
|
||||
listeners,
|
||||
"tools/result",
|
||||
{ name: "skill", arguments: '{"name":"browser-skill"}' },
|
||||
{ isError: false },
|
||||
);
|
||||
expect(registerSuite).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("is idempotent across repeated invocations", () => {
|
||||
const { ctx, listeners } = fakeEventCtx();
|
||||
const registerSuite = vi.fn(() => () => {});
|
||||
armLazyTools(ctx, registerSuite);
|
||||
for (let i = 0; i < 3; i++) {
|
||||
callListeners(
|
||||
listeners,
|
||||
"tools/result",
|
||||
{ name: "skill", arguments: { name: "browser-skill" } },
|
||||
{ isError: false },
|
||||
);
|
||||
}
|
||||
expect(registerSuite).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("reveals on a /browser-skill user gesture (skill-invocation message)", () => {
|
||||
const { ctx, listeners } = fakeEventCtx();
|
||||
const registerSuite = vi.fn(() => () => {});
|
||||
armLazyTools(ctx, registerSuite);
|
||||
callListeners(
|
||||
listeners,
|
||||
"session/event",
|
||||
{ events: [] },
|
||||
{
|
||||
type: "user/message",
|
||||
data: { source: { kind: "skill-invocation", name: "browser-skill" } },
|
||||
},
|
||||
);
|
||||
expect(registerSuite).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("restores from session history: hit at boot, on session entry, and miss", () => {
|
||||
const hitEvents = [
|
||||
{
|
||||
type: "tool/call",
|
||||
data: { callId: "c1", name: "skill", arguments: '{"name":"browser-skill"}' },
|
||||
},
|
||||
{ type: "tool/result", data: { message: { callId: "c1", isError: false } } },
|
||||
];
|
||||
// boot-time live sessions are scanned
|
||||
const boot = fakeEventCtx({ list: () => [{ events: hitEvents }] });
|
||||
const bootRegister = vi.fn(() => () => {});
|
||||
armLazyTools(boot.ctx, bootRegister);
|
||||
expect(bootRegister).toHaveBeenCalledTimes(1);
|
||||
|
||||
// a session entered later with the same proof
|
||||
const { ctx, listeners } = fakeEventCtx({ list: () => [] });
|
||||
const registerSuite = vi.fn(() => () => {});
|
||||
armLazyTools(ctx, registerSuite);
|
||||
callListeners(listeners, "session/created", { events: hitEvents });
|
||||
expect(registerSuite).toHaveBeenCalledTimes(1);
|
||||
|
||||
// miss: failed result proves nothing
|
||||
const miss = fakeEventCtx({
|
||||
list: () => [
|
||||
{
|
||||
events: [
|
||||
{
|
||||
type: "tool/call",
|
||||
data: { callId: "c1", name: "skill", arguments: '{"name":"browser-skill"}' },
|
||||
},
|
||||
{ type: "tool/result", data: { message: { callId: "c1", isError: true } } },
|
||||
],
|
||||
},
|
||||
],
|
||||
});
|
||||
const missRegister = vi.fn(() => () => {});
|
||||
armLazyTools(miss.ctx, missRegister);
|
||||
expect(missRegister).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("disposes listeners and the revealed suite", () => {
|
||||
const { ctx, listeners } = fakeEventCtx();
|
||||
const suiteDispose = vi.fn();
|
||||
const disarm = armLazyTools(ctx, () => suiteDispose);
|
||||
callListeners(
|
||||
listeners,
|
||||
"tools/result",
|
||||
{ name: "skill", arguments: { name: "browser-skill" } },
|
||||
{ isError: false },
|
||||
);
|
||||
disarm();
|
||||
expect(suiteDispose).toHaveBeenCalledTimes(1);
|
||||
expect(listeners.size).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("hasSuccessfulSkillInvocation", () => {
|
||||
it("pairs call and result by callId and honors gestures", () => {
|
||||
expect(hasSuccessfulSkillInvocation([])).toBe(false);
|
||||
expect(
|
||||
hasSuccessfulSkillInvocation([
|
||||
{
|
||||
type: "tool/call",
|
||||
data: { callId: "x", name: "skill", arguments: { name: "browser-skill" } },
|
||||
},
|
||||
{ type: "tool/result", data: { message: { callId: "x", isError: false } } },
|
||||
]),
|
||||
).toBe(true);
|
||||
// result for an unrelated call id does not count
|
||||
expect(
|
||||
hasSuccessfulSkillInvocation([
|
||||
{
|
||||
type: "tool/call",
|
||||
data: { callId: "x", name: "skill", arguments: { name: "browser-skill" } },
|
||||
},
|
||||
{ type: "tool/result", data: { message: { callId: "y", isError: false } } },
|
||||
]),
|
||||
).toBe(false);
|
||||
expect(
|
||||
hasSuccessfulSkillInvocation([
|
||||
{
|
||||
type: "user/message",
|
||||
data: { source: { kind: "skill-invocation", name: "browser-skill" } },
|
||||
},
|
||||
]),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("lazyTools wiring in apply()", () => {
|
||||
function applyHarness(config: Record<string, unknown>) {
|
||||
const tools = new Map<string, ToolDefinition>();
|
||||
const listeners = new Map<string, (...args: never[]) => void>();
|
||||
const ctx = {
|
||||
tools: { register: (def: ToolDefinition) => tools.set(def.name, def) },
|
||||
get: () => undefined,
|
||||
inject: () => {},
|
||||
effect: () => {},
|
||||
on: (event: string, listener: (...args: never[]) => void) => {
|
||||
listeners.set(event, listener);
|
||||
return () => listeners.delete(event);
|
||||
},
|
||||
};
|
||||
const runner = {
|
||||
async run(): Promise<BskRunResult> {
|
||||
return { code: 0, stdout: "{}", stderr: "", timedOut: false, aborted: false };
|
||||
},
|
||||
killAll() {},
|
||||
killFor: () => 0,
|
||||
};
|
||||
apply(ctx as never, config, { runnerFactory: () => runner as never });
|
||||
return { tools, listeners };
|
||||
}
|
||||
|
||||
it("lazyTools: false registers the suite at apply time", () => {
|
||||
const { tools } = applyHarness({ lazyTools: false });
|
||||
expect(tools.has("browser_session_start")).toBe(true);
|
||||
expect(tools.has("browser_emulate")).toBe(true);
|
||||
});
|
||||
|
||||
it("lazyTools: true hides the suite until the skill fires", () => {
|
||||
const { tools, listeners } = applyHarness({ lazyTools: true });
|
||||
expect(tools.has("browser_session_start")).toBe(false);
|
||||
callListeners(
|
||||
listeners,
|
||||
"tools/result",
|
||||
{ name: "skill", arguments: { name: "browser-skill" } },
|
||||
{ isError: false },
|
||||
);
|
||||
expect(tools.has("browser_session_start")).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,654 @@
|
||||
// ObservationService host tests: state machine, cadence/backoff, observation-
|
||||
// traffic isolation, interrupt routing, the owned boundary, and the HTTP/SSE
|
||||
// interface. All bsk runs are faked; the scheduler is a manual timer queue.
|
||||
|
||||
import { existsSync, mkdtempSync, writeFileSync } from "node:fs";
|
||||
import type { IncomingMessage, ServerResponse } from "node:http";
|
||||
import { tmpdir } from "node:os";
|
||||
import { basename, dirname } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
actionForLabel,
|
||||
type ObservationEvent,
|
||||
type ObservationScheduler,
|
||||
ObservationService,
|
||||
} from "../src/observation";
|
||||
import { registerObservationRoutes } from "../src/observation-http";
|
||||
import { KeyedExecutor } from "../src/queue";
|
||||
import type { BskRunner, BskRunOptions, BskRunResult } from "../src/runner";
|
||||
import { SessionRegistry } from "../src/sessions";
|
||||
|
||||
const OPTIONS = { enabled: true, thumbnailIntervalMs: 1500, idleIntervalMs: 8000 };
|
||||
|
||||
/** Manual timer queue + controllable clock. */
|
||||
function fakeScheduler() {
|
||||
let now = 1_000_000;
|
||||
const timers: { handle: object; fn: () => void; ms: number }[] = [];
|
||||
const scheduler: ObservationScheduler & {
|
||||
advance: (ms: number) => void;
|
||||
runNext: () => void;
|
||||
pending: () => number[];
|
||||
} = {
|
||||
now: () => now,
|
||||
setTimeout: (fn, ms) => {
|
||||
const handle = { fn, ms };
|
||||
timers.push({ handle, fn, ms });
|
||||
return handle;
|
||||
},
|
||||
clearTimeout: (h) => {
|
||||
const index = timers.findIndex((t) => t.handle === h);
|
||||
if (index >= 0) timers.splice(index, 1);
|
||||
},
|
||||
advance: (ms) => {
|
||||
now += ms;
|
||||
},
|
||||
runNext: () => {
|
||||
const next = timers.shift();
|
||||
next?.fn();
|
||||
},
|
||||
pending: () => timers.map((t) => t.ms),
|
||||
};
|
||||
return scheduler;
|
||||
}
|
||||
|
||||
interface FakeRunner extends BskRunner {
|
||||
calls: { args: string[]; options: BskRunOptions }[];
|
||||
killed: string[];
|
||||
}
|
||||
|
||||
/** Runner that answers screenshot with a real temp PNG and records kills. */
|
||||
function fakeRunner(
|
||||
opts: {
|
||||
screenshotFails?: boolean;
|
||||
screenshotNotFound?: boolean;
|
||||
killCount?: number;
|
||||
screenshotBytes?: () => Uint8Array;
|
||||
} = {},
|
||||
): FakeRunner {
|
||||
const calls: FakeRunner["calls"] = [];
|
||||
const killed: string[] = [];
|
||||
return {
|
||||
calls,
|
||||
killed,
|
||||
async run(args: string[], options: BskRunOptions = {}): Promise<BskRunResult> {
|
||||
calls.push({ args, options });
|
||||
if (args[0] === "screenshot") {
|
||||
if (opts.screenshotNotFound) {
|
||||
return {
|
||||
code: 4,
|
||||
stdout: JSON.stringify({ code: "session_not_found", message: "no such session" }),
|
||||
stderr: "",
|
||||
timedOut: false,
|
||||
aborted: false,
|
||||
};
|
||||
}
|
||||
if (opts.screenshotFails) {
|
||||
return { code: 1, stdout: "", stderr: "boom", timedOut: false, aborted: false };
|
||||
}
|
||||
const outIndex = args.indexOf("--out") + 1;
|
||||
const out = args[outIndex];
|
||||
writeFileSync(out, opts.screenshotBytes?.() ?? Buffer.from([0x89, 0x50, 0x4e, 0x47]));
|
||||
return {
|
||||
code: 0,
|
||||
stdout: JSON.stringify({
|
||||
tab_id: 7,
|
||||
width: 4,
|
||||
height: 4,
|
||||
format: "png",
|
||||
path: out,
|
||||
byte_size: 4,
|
||||
}),
|
||||
stderr: "",
|
||||
timedOut: false,
|
||||
aborted: false,
|
||||
};
|
||||
}
|
||||
return { code: 0, stdout: "{}", stderr: "", timedOut: false, aborted: false };
|
||||
},
|
||||
killAll() {},
|
||||
killFor(tag: string) {
|
||||
killed.push(tag);
|
||||
return opts.killCount ?? 1;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function fakeCtx() {
|
||||
return { get: () => undefined } as never;
|
||||
}
|
||||
|
||||
function setup(opts: {
|
||||
runner?: FakeRunner;
|
||||
registry?: SessionRegistry;
|
||||
scheduler?: ReturnType<typeof fakeScheduler>;
|
||||
}) {
|
||||
const registry = opts.registry ?? new SessionRegistry(5);
|
||||
const runner = opts.runner ?? fakeRunner();
|
||||
const scheduler = opts.scheduler ?? fakeScheduler();
|
||||
const service = new ObservationService({
|
||||
ctx: fakeCtx(),
|
||||
runner,
|
||||
registry,
|
||||
queue: new KeyedExecutor(),
|
||||
options: OPTIONS,
|
||||
scheduler,
|
||||
});
|
||||
const events: ObservationEvent[] = [];
|
||||
service.subscribe((event) => events.push(event));
|
||||
return { service, registry, runner, scheduler, events };
|
||||
}
|
||||
|
||||
/** Drive the registry through a real start so the session is owned. */
|
||||
function own(registry: SessionRegistry, sessionId: string): void {
|
||||
registry.reserveStart();
|
||||
registry.completeStart({ sessionId, startedAtMs: 1 });
|
||||
}
|
||||
|
||||
/** Poll until the condition holds (capture completion is real async I/O). */
|
||||
async function waitFor(cond: () => boolean, timeoutMs = 10_000): Promise<void> {
|
||||
const start = Date.now();
|
||||
while (!cond()) {
|
||||
if (Date.now() - start > timeoutMs) throw new Error("waitFor timeout");
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
}
|
||||
}
|
||||
|
||||
describe("state machine", () => {
|
||||
it("tracks add/action/end/url/remove and emits upsert/remove/reset events", () => {
|
||||
const { service, events } = setup({});
|
||||
service.addSession("s1");
|
||||
expect(service.getState()).toEqual([{ sessionId: "s1", action: "idle", since: 1_000_000 }]);
|
||||
service.beginAction("s1", "navigating");
|
||||
expect(service.getState()[0].action).toBe("navigating");
|
||||
service.endAction("s1");
|
||||
expect(service.getState()[0].action).toBe("idle");
|
||||
service.setUrl("s1", "https://example.com");
|
||||
expect(service.getState()[0].url).toBe("https://example.com");
|
||||
service.endAction("s1", "click failed: no ref");
|
||||
expect(service.getState()[0].lastError).toBe("click failed: no ref");
|
||||
service.removeSession("s1");
|
||||
expect(service.getState()).toEqual([]);
|
||||
expect(events.map((e) => e.type)).toEqual([
|
||||
"upsert", // add
|
||||
"upsert", // begin
|
||||
"upsert", // end
|
||||
"upsert", // setUrl
|
||||
"upsert", // end with error
|
||||
"remove",
|
||||
]);
|
||||
service.dispose();
|
||||
expect(events[events.length - 1].type).toBe("reset");
|
||||
});
|
||||
|
||||
it("ignores instrumentation for unknown sessions (owned-only by construction)", () => {
|
||||
const { service } = setup({});
|
||||
service.beginAction("foreign", "clicking");
|
||||
service.endAction("foreign");
|
||||
service.setUrl("foreign", "https://x");
|
||||
expect(service.getState()).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("thumbnail cadence", () => {
|
||||
it("captures immediately on add and after action end, then fast-cadences while active", async () => {
|
||||
const scheduler = fakeScheduler();
|
||||
const runner = fakeRunner();
|
||||
const { service } = setup({ runner, scheduler });
|
||||
const captures = () => runner.calls.filter((c) => c.args[0] === "screenshot").length;
|
||||
service.addSession("s1");
|
||||
// add schedules an immediate (0ms) capture.
|
||||
expect(scheduler.pending()).toEqual([0]);
|
||||
scheduler.runNext();
|
||||
await waitFor(() => captures() === 1 && scheduler.pending().length === 1);
|
||||
expect(service.getState()[0].thumbnailAttachmentId).toBe("obs-s1-1");
|
||||
// Just captured with fresh activity: next frame on the fast cadence.
|
||||
expect(scheduler.pending()).toEqual([1500]);
|
||||
scheduler.runNext();
|
||||
await waitFor(() => captures() === 2 && scheduler.pending().length === 1);
|
||||
expect(scheduler.pending()).toEqual([1500]);
|
||||
});
|
||||
|
||||
it("downclocks to the idle cadence when the session has been quiet", async () => {
|
||||
const scheduler = fakeScheduler();
|
||||
const runner = fakeRunner();
|
||||
const { service, scheduler: sch } = setup({ scheduler, runner });
|
||||
const captures = () => runner.calls.filter((c) => c.args[0] === "screenshot").length;
|
||||
service.addSession("s1");
|
||||
sch.runNext();
|
||||
await waitFor(() => captures() === 1 && sch.pending().length === 1);
|
||||
// Fresh activity: fast cadence.
|
||||
expect(sch.pending()).toEqual([1500]);
|
||||
// Go quiet beyond the idle window before the next frame fires.
|
||||
sch.advance(9000);
|
||||
sch.runNext();
|
||||
await waitFor(() => captures() === 2 && sch.pending().length === 1);
|
||||
expect(sch.pending()).toEqual([8000]);
|
||||
});
|
||||
|
||||
it("backs off to the idle cadence after repeated capture failures, keeping the old frame", async () => {
|
||||
const scheduler = fakeScheduler();
|
||||
const runner = fakeRunner({ screenshotFails: true });
|
||||
const { service } = setup({ runner, scheduler });
|
||||
const captures = () => runner.calls.filter((c) => c.args[0] === "screenshot").length;
|
||||
service.addSession("s1");
|
||||
for (let i = 1; i <= 4; i++) {
|
||||
scheduler.runNext();
|
||||
await waitFor(() => captures() === i && scheduler.pending().length === 1);
|
||||
}
|
||||
// 4 attempts (1 immediate + 3 fast), then backoff to the idle cadence.
|
||||
expect(captures()).toBe(4);
|
||||
expect(scheduler.pending()).toEqual([8000]);
|
||||
expect(service.getState()[0].thumbnailAttachmentId).toBeUndefined();
|
||||
});
|
||||
|
||||
it("captures frames without an attachment store", async () => {
|
||||
const scheduler = fakeScheduler();
|
||||
const runner = fakeRunner();
|
||||
const { service } = setup({ runner, scheduler });
|
||||
service.addSession("s1");
|
||||
scheduler.runNext();
|
||||
await waitFor(() => service.getState()[0]?.thumbnailAttachmentId === "obs-s1-1");
|
||||
expect((await service.readThumbnail("obs-s1-1"))?.mediaType).toBe("image/png");
|
||||
});
|
||||
});
|
||||
|
||||
describe("observation traffic isolation", () => {
|
||||
it("captures never emit action changes nor touch the registry", async () => {
|
||||
const scheduler = fakeScheduler();
|
||||
const registry = new SessionRegistry(5);
|
||||
own(registry, "s1");
|
||||
const { service, events } = setup({ registry, scheduler });
|
||||
service.addSession("s1");
|
||||
events.length = 0;
|
||||
scheduler.runNext();
|
||||
await waitFor(() => events.filter((e) => e.type === "upsert").length === 1);
|
||||
// The capture upsert carries the thumbnail but the action stays idle…
|
||||
const captureEvents = events.filter((e) => e.type === "upsert");
|
||||
expect(captureEvents).toHaveLength(1);
|
||||
const captureEvent = captureEvents[0];
|
||||
if (captureEvent.type !== "upsert") throw new Error("unreachable");
|
||||
expect(captureEvent.session?.action).toBe("idle");
|
||||
expect(captureEvent.session?.thumbnailAttachmentId).toBe("obs-s1-1");
|
||||
// …and the registry's current pointer never moved through observation.
|
||||
expect(registry.current()).toBe("s1");
|
||||
expect(registry.list()[0].startedAtMs).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("interrupt routing", () => {
|
||||
it("interrupts the current session by default", () => {
|
||||
const registry = new SessionRegistry(5);
|
||||
own(registry, "s1");
|
||||
const runner = fakeRunner();
|
||||
const { service } = setup({ registry, runner });
|
||||
expect(service.interrupt()).toBe(true);
|
||||
expect(runner.killed).toEqual(["s1"]);
|
||||
});
|
||||
|
||||
it("interrupts the specified owned session", () => {
|
||||
const registry = new SessionRegistry(5);
|
||||
own(registry, "s1");
|
||||
own(registry, "s2");
|
||||
const runner = fakeRunner();
|
||||
const { service } = setup({ registry, runner });
|
||||
expect(service.interrupt("s1")).toBe(true);
|
||||
expect(runner.killed).toEqual(["s1"]);
|
||||
});
|
||||
|
||||
it("returns false with no current session and for foreign ids without killing", () => {
|
||||
const registry = new SessionRegistry(5);
|
||||
own(registry, "s1");
|
||||
const runner = fakeRunner();
|
||||
const { service } = setup({ registry, runner });
|
||||
expect(service.interrupt("foreign")).toBe(false);
|
||||
registry.remove("s1");
|
||||
expect(service.interrupt()).toBe(false);
|
||||
expect(runner.killed).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("HTTP/SSE interface", () => {
|
||||
interface RecordedRoute {
|
||||
path: string;
|
||||
handler: (req: IncomingMessage, res: ServerResponse) => void | Promise<void>;
|
||||
}
|
||||
|
||||
function routeHarness() {
|
||||
const routes = new Map<string, RecordedRoute>();
|
||||
const webServer = {
|
||||
register(route: RecordedRoute & { kind: string }) {
|
||||
routes.set(route.path, route);
|
||||
return () => routes.delete(route.path);
|
||||
},
|
||||
};
|
||||
return { routes, webServer };
|
||||
}
|
||||
|
||||
function fakeRes() {
|
||||
const chunks: string[] = [];
|
||||
const closeHandlers: (() => void)[] = [];
|
||||
const res = {
|
||||
status: 0,
|
||||
writeHead(status: number) {
|
||||
res.status = status;
|
||||
return res;
|
||||
},
|
||||
write(chunk: string) {
|
||||
chunks.push(chunk);
|
||||
},
|
||||
end(chunk?: string) {
|
||||
if (chunk !== undefined) chunks.push(chunk);
|
||||
},
|
||||
on(event: string, fn: () => void) {
|
||||
if (event === "close") closeHandlers.push(fn);
|
||||
},
|
||||
close: () => closeHandlers.forEach((fn) => fn()),
|
||||
body: () => chunks.join(""),
|
||||
};
|
||||
return {
|
||||
res: res as unknown as ServerResponse & {
|
||||
status: number;
|
||||
body: () => string;
|
||||
close: () => void;
|
||||
},
|
||||
chunks,
|
||||
};
|
||||
}
|
||||
|
||||
/** Loopback GET/POST request stubs the fence accepts. */
|
||||
function fakeReq(overrides: Record<string, unknown> = {}): IncomingMessage {
|
||||
return {
|
||||
method: "GET",
|
||||
headers: { host: "127.0.0.1:3999" },
|
||||
on: (event: string, fn: (chunk?: string) => void) => {
|
||||
if (event === "data") fn(JSON.stringify({ sessionId: "s1" }));
|
||||
if (event === "end") fn();
|
||||
},
|
||||
...overrides,
|
||||
} as unknown as IncomingMessage;
|
||||
}
|
||||
|
||||
it("serves state, streams events, and routes interrupt", async () => {
|
||||
const registry = new SessionRegistry(5);
|
||||
own(registry, "s1");
|
||||
const runner = fakeRunner();
|
||||
const { service } = setup({ registry, runner });
|
||||
service.addSession("s1");
|
||||
|
||||
const { routes, webServer } = routeHarness();
|
||||
const ctx = { get: (key: string) => (key === "webServer" ? webServer : undefined) } as never;
|
||||
const dispose = registerObservationRoutes(ctx, service);
|
||||
|
||||
// state
|
||||
const stateRoute = routes.get("/bsk-observation/state");
|
||||
expect(stateRoute).toBeDefined();
|
||||
const stateRes = fakeRes();
|
||||
await stateRoute?.handler(fakeReq(), stateRes.res as never);
|
||||
expect(stateRes.res.status).toBe(200);
|
||||
expect(JSON.parse(stateRes.res.body()).sessions[0].sessionId).toBe("s1");
|
||||
|
||||
// events (SSE): one write per service event
|
||||
const eventsRoute = routes.get("/bsk-observation/events");
|
||||
const eventsRes = fakeRes();
|
||||
await eventsRoute?.handler(fakeReq(), eventsRes.res as never);
|
||||
service.beginAction("s1", "clicking");
|
||||
expect(eventsRes.res.body()).toContain('"action":"clicking"');
|
||||
eventsRes.res.close();
|
||||
|
||||
// interrupt (POST {sessionId})
|
||||
const interruptRoute = routes.get("/bsk-observation/interrupt");
|
||||
const interruptRes = fakeRes();
|
||||
const postReq = fakeReq({
|
||||
method: "POST",
|
||||
headers: { host: "127.0.0.1:3999", "content-type": "application/json" },
|
||||
});
|
||||
await interruptRoute?.handler(postReq, interruptRes.res as never);
|
||||
expect(JSON.parse(interruptRes.res.body())).toEqual({ interrupted: true });
|
||||
|
||||
dispose();
|
||||
expect(routes.size).toBe(0);
|
||||
service.dispose();
|
||||
});
|
||||
|
||||
it("fences off non-loopback, cross-site, and simple-request traffic", async () => {
|
||||
const registry = new SessionRegistry(5);
|
||||
own(registry, "s1");
|
||||
const { service } = setup({ registry });
|
||||
service.addSession("s1");
|
||||
const { routes, webServer } = routeHarness();
|
||||
const ctx = { get: (key: string) => (key === "webServer" ? webServer : undefined) } as never;
|
||||
const dispose = registerObservationRoutes(ctx, service);
|
||||
const stateRoute = routes.get("/bsk-observation/state");
|
||||
const interruptRoute = routes.get("/bsk-observation/interrupt");
|
||||
|
||||
const getState = async (headers: Record<string, string>) => {
|
||||
const res = fakeRes();
|
||||
await stateRoute?.handler(fakeReq({ headers }), res.res as never);
|
||||
return res.res.status;
|
||||
};
|
||||
|
||||
// Host must be a loopback authority.
|
||||
expect(await getState({ host: "192.168.1.20:3999" })).toBe(403);
|
||||
expect(await getState({ host: "attacker.example" })).toBe(403);
|
||||
expect(await getState({ host: "localhost:3999" })).toBe(200);
|
||||
expect(await getState({ host: "[::1]:3999" })).toBe(200);
|
||||
// A present Origin must match the Host (DNS-rebinding reads die here).
|
||||
expect(await getState({ host: "127.0.0.1:3999", origin: "http://evil.example" })).toBe(403);
|
||||
expect(await getState({ host: "127.0.0.1:3999", origin: "http://127.0.0.1:3999" })).toBe(200);
|
||||
// Explicitly cross-site fetches are refused even without an Origin.
|
||||
expect(await getState({ host: "127.0.0.1:3999", "sec-fetch-site": "cross-site" })).toBe(403);
|
||||
expect(await getState({ host: "127.0.0.1:3999", "sec-fetch-site": "same-origin" })).toBe(200);
|
||||
|
||||
// POST must be application/json — a cross-site simple request can never comply.
|
||||
const post = async (contentType: string | undefined) => {
|
||||
const res = fakeRes();
|
||||
const req = fakeReq({
|
||||
method: "POST",
|
||||
headers: {
|
||||
host: "127.0.0.1:3999",
|
||||
...(contentType !== undefined ? { "content-type": contentType } : {}),
|
||||
},
|
||||
});
|
||||
await interruptRoute?.handler(req, res.res as never);
|
||||
return res.res.status;
|
||||
};
|
||||
expect(await post("text/plain")).toBe(403);
|
||||
expect(await post("application/x-www-form-urlencoded")).toBe(403);
|
||||
expect(await post(undefined)).toBe(403);
|
||||
expect(await post("application/json; charset=utf-8")).toBe(200);
|
||||
|
||||
dispose();
|
||||
service.dispose();
|
||||
});
|
||||
|
||||
it("registers nothing when no webServer is mounted", () => {
|
||||
const { service } = setup({});
|
||||
const ctx = { get: () => undefined } as never;
|
||||
const dispose = registerObservationRoutes(ctx, service);
|
||||
expect(typeof dispose).toBe("function");
|
||||
dispose();
|
||||
service.dispose();
|
||||
});
|
||||
});
|
||||
|
||||
describe("availability and dead sessions", () => {
|
||||
it("flips availability off after repeated global failures and back on success", async () => {
|
||||
const scheduler = fakeScheduler();
|
||||
const runner = fakeRunner({ screenshotFails: true });
|
||||
const { service, events } = setup({ runner, scheduler });
|
||||
service.addSession("s1");
|
||||
for (let i = 1; i <= 3; i++) {
|
||||
scheduler.runNext();
|
||||
// the frame's scratch-file unlink lands before the next timer is queued
|
||||
await waitFor(
|
||||
() =>
|
||||
runner.calls.filter((c) => c.args[0] === "screenshot").length === i &&
|
||||
scheduler.pending().length === 1,
|
||||
);
|
||||
}
|
||||
expect(service.isAvailable()).toBe(false);
|
||||
expect(events.some((e) => e.type === "availability" && e.available === false)).toBe(true);
|
||||
service.dispose();
|
||||
});
|
||||
|
||||
it("marks a session dead on session_not_found and stops asking for frames", async () => {
|
||||
const scheduler = fakeScheduler();
|
||||
const runner = fakeRunner({ screenshotNotFound: true });
|
||||
const { service, events } = setup({ runner, scheduler });
|
||||
service.addSession("s1");
|
||||
scheduler.runNext();
|
||||
await waitFor(() => events.some((e) => e.type === "upsert" && e.session?.dead === true));
|
||||
expect(service.getState()[0].dead).toBe(true);
|
||||
expect(service.getState()[0].action).toBe("idle");
|
||||
// No further captures are scheduled for a dead session.
|
||||
expect(scheduler.pending()).toEqual([]);
|
||||
// Instrumentation is dropped for dead sessions.
|
||||
service.beginAction("s1", "clicking");
|
||||
expect(service.getState()[0].action).toBe("idle");
|
||||
// A dead session leaves cleanly.
|
||||
service.removeSession("s1");
|
||||
expect(service.getState()).toEqual([]);
|
||||
});
|
||||
|
||||
it("clears lastError on the next action and keeps it on failure", () => {
|
||||
const { service } = setup({});
|
||||
service.addSession("s1");
|
||||
service.endAction("s1", "click failed");
|
||||
expect(service.getState()[0].lastError).toBe("click failed");
|
||||
service.beginAction("s1", "clicking");
|
||||
expect(service.getState()[0].lastError).toBeUndefined();
|
||||
service.endAction("s1");
|
||||
expect(service.getState()[0].lastError).toBeUndefined();
|
||||
service.endAction("s1", "fill failed");
|
||||
expect(service.getState()[0].lastError).toBe("fill failed");
|
||||
service.dispose();
|
||||
});
|
||||
});
|
||||
|
||||
describe("actionForLabel", () => {
|
||||
it("maps tool labels onto observation verbs", () => {
|
||||
expect(actionForLabel("navigate")).toBe("navigating");
|
||||
expect(actionForLabel("screenshot")).toBe("capturing");
|
||||
expect(actionForLabel("session start")).toBe("starting");
|
||||
});
|
||||
});
|
||||
|
||||
describe("scratch files and thumbnail references", () => {
|
||||
it("reuses one scratch path per session and deletes it after reading", async () => {
|
||||
const scheduler = fakeScheduler();
|
||||
const runner = fakeRunner();
|
||||
const { service } = setup({ runner, scheduler });
|
||||
service.addSession("s1");
|
||||
// first frame (scheduled with delay 0)
|
||||
scheduler.runNext();
|
||||
await waitFor(
|
||||
() =>
|
||||
service.getState()[0]?.thumbnailAttachmentId !== undefined &&
|
||||
scheduler.pending().length === 1,
|
||||
);
|
||||
const firstOut = runner.calls.filter((c) => c.args[0] === "screenshot").at(-1)?.args;
|
||||
const pathOf = (args: string[]) => args[args.indexOf("--out") + 1];
|
||||
const firstPath = pathOf(firstOut ?? []);
|
||||
expect(dirname(firstPath)).toBe(tmpdir());
|
||||
expect(basename(firstPath)).toMatch(
|
||||
/^bsk-obs-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}-[0-9a-f]{16}\.png$/,
|
||||
);
|
||||
expect(existsSync(firstPath)).toBe(false);
|
||||
// second frame: same path, deleted again
|
||||
service.endAction("s1");
|
||||
scheduler.runNext();
|
||||
await waitFor(
|
||||
() =>
|
||||
runner.calls.filter((c) => c.args[0] === "screenshot").length >= 2 &&
|
||||
scheduler.pending().length === 1,
|
||||
);
|
||||
const secondOut = runner.calls.filter((c) => c.args[0] === "screenshot").at(-1)?.args;
|
||||
const secondPath = pathOf(secondOut ?? []);
|
||||
expect(secondPath).toBe(firstPath);
|
||||
await waitFor(() => !existsSync(secondPath));
|
||||
service.dispose();
|
||||
});
|
||||
|
||||
it("isolates scratch paths between service instances", async () => {
|
||||
const first = setup({ scheduler: fakeScheduler(), runner: fakeRunner() });
|
||||
const second = setup({ scheduler: fakeScheduler(), runner: fakeRunner() });
|
||||
first.service.addSession("s1");
|
||||
second.service.addSession("s1");
|
||||
first.scheduler.runNext();
|
||||
second.scheduler.runNext();
|
||||
await waitFor(
|
||||
() =>
|
||||
first.service.getState()[0]?.thumbnailAttachmentId !== undefined &&
|
||||
second.service.getState()[0]?.thumbnailAttachmentId !== undefined,
|
||||
);
|
||||
const pathOf = (runner: FakeRunner) => {
|
||||
const args = runner.calls.find((call) => call.args[0] === "screenshot")?.args ?? [];
|
||||
return args[args.indexOf("--out") + 1];
|
||||
};
|
||||
expect(pathOf(first.runner)).not.toBe(pathOf(second.runner));
|
||||
first.service.dispose();
|
||||
second.service.dispose();
|
||||
});
|
||||
|
||||
it("keeps the last two distinct frames and evicts the third", async () => {
|
||||
let n = 0;
|
||||
const runner = fakeRunner({
|
||||
screenshotBytes: () => Uint8Array.from([0x89, 0x50, 0x4e, 0x47, n++]),
|
||||
});
|
||||
const scheduler = fakeScheduler();
|
||||
const { service } = setup({ scheduler, runner });
|
||||
service.addSession("s1");
|
||||
scheduler.runNext();
|
||||
await waitFor(
|
||||
() =>
|
||||
service.getState()[0]?.thumbnailAttachmentId === "obs-s1-1" &&
|
||||
scheduler.pending().length === 1,
|
||||
);
|
||||
service.endAction("s1");
|
||||
scheduler.runNext();
|
||||
await waitFor(
|
||||
() =>
|
||||
service.getState()[0]?.thumbnailAttachmentId === "obs-s1-2" &&
|
||||
scheduler.pending().length === 1,
|
||||
);
|
||||
expect((await service.readThumbnail("obs-s1-1"))?.mediaType).toBe("image/png");
|
||||
expect((await service.readThumbnail("obs-s1-2"))?.mediaType).toBe("image/png");
|
||||
service.endAction("s1");
|
||||
scheduler.runNext();
|
||||
await waitFor(
|
||||
() =>
|
||||
service.getState()[0]?.thumbnailAttachmentId === "obs-s1-3" &&
|
||||
scheduler.pending().length === 1,
|
||||
);
|
||||
expect(await service.readThumbnail("obs-s1-1")).toBeUndefined();
|
||||
expect((await service.readThumbnail("obs-s1-2"))?.mediaType).toBe("image/png");
|
||||
expect((await service.readThumbnail("obs-s1-3"))?.mediaType).toBe("image/png");
|
||||
service.removeSession("s1");
|
||||
expect(await service.readThumbnail("obs-s1-2")).toBeUndefined();
|
||||
expect(await service.readThumbnail("obs-s1-3")).toBeUndefined();
|
||||
service.dispose();
|
||||
});
|
||||
|
||||
it("reuses the current frame id when the PNG bytes are unchanged", async () => {
|
||||
const scheduler = fakeScheduler();
|
||||
const runner = fakeRunner();
|
||||
const { service } = setup({ scheduler, runner });
|
||||
service.addSession("s1");
|
||||
scheduler.runNext();
|
||||
await waitFor(
|
||||
() =>
|
||||
service.getState()[0]?.thumbnailAttachmentId === "obs-s1-1" &&
|
||||
scheduler.pending().length === 1,
|
||||
);
|
||||
service.endAction("s1");
|
||||
scheduler.runNext();
|
||||
await waitFor(
|
||||
() =>
|
||||
runner.calls.filter((c) => c.args[0] === "screenshot").length >= 2 &&
|
||||
scheduler.pending().length === 1,
|
||||
);
|
||||
expect(service.getState()[0]?.thumbnailAttachmentId).toBe("obs-s1-1");
|
||||
expect((await service.readThumbnail("obs-s1-1"))?.mediaType).toBe("image/png");
|
||||
service.dispose();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,85 @@
|
||||
// KeyedExecutor: per-key FIFO guarantees, including the abort-while-queued
|
||||
// race that used to release the tail early (letting a later task overlap the
|
||||
// one still running on the same session).
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { KeyedExecutor } from "../src/queue";
|
||||
|
||||
describe("KeyedExecutor", () => {
|
||||
it("runs same-key tasks in FIFO order", async () => {
|
||||
const queue = new KeyedExecutor();
|
||||
const order: string[] = [];
|
||||
const gate = new Promise<void>((resolve) => setTimeout(resolve, 5));
|
||||
await Promise.all([
|
||||
queue.run("k", async () => {
|
||||
await gate;
|
||||
order.push("a");
|
||||
}),
|
||||
queue.run("k", async () => {
|
||||
order.push("b");
|
||||
}),
|
||||
queue.run("k", async () => {
|
||||
order.push("c");
|
||||
}),
|
||||
]);
|
||||
expect(order).toEqual(["a", "b", "c"]);
|
||||
});
|
||||
|
||||
it("a task aborted while queued does not let the next one overlap the runner", async () => {
|
||||
const queue = new KeyedExecutor();
|
||||
let releaseA!: () => void;
|
||||
const aRuns = new Promise<void>((resolve) => {
|
||||
releaseA = resolve;
|
||||
});
|
||||
const events: string[] = [];
|
||||
|
||||
// A holds the session until releaseA.
|
||||
const a = queue.run("s1", async () => {
|
||||
events.push("a:start");
|
||||
await aRuns;
|
||||
events.push("a:end");
|
||||
});
|
||||
// B is queued behind A and aborted before A finishes.
|
||||
const abortB = new AbortController();
|
||||
const b = queue.run(
|
||||
"s1",
|
||||
async () => {
|
||||
events.push("b:start");
|
||||
},
|
||||
abortB.signal,
|
||||
);
|
||||
// C is queued behind B.
|
||||
const c = queue.run("s1", async () => {
|
||||
events.push("c:start");
|
||||
});
|
||||
|
||||
abortB.abort();
|
||||
await expect(b).rejects.toMatchObject({ name: "AbortError" });
|
||||
// Give C every chance to (wrongly) start while A is still in flight.
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
expect(events).toEqual(["a:start"]);
|
||||
|
||||
releaseA();
|
||||
await a;
|
||||
await c;
|
||||
expect(events).toEqual(["a:start", "a:end", "c:start"]);
|
||||
});
|
||||
|
||||
it("a failed task never strands the queue", async () => {
|
||||
const queue = new KeyedExecutor();
|
||||
await expect(
|
||||
queue.run("k", async () => {
|
||||
throw new Error("boom");
|
||||
}),
|
||||
).rejects.toThrow("boom");
|
||||
await expect(queue.run("k", async () => "ok")).resolves.toBe("ok");
|
||||
});
|
||||
|
||||
it("drops the tail entry once the queue drains", async () => {
|
||||
const queue = new KeyedExecutor();
|
||||
const internals = queue as unknown as { tails: Map<string, Promise<void>> };
|
||||
await queue.run("k", async () => {});
|
||||
await new Promise((resolve) => setImmediate(resolve));
|
||||
expect(internals.tails.size).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,142 @@
|
||||
import type { ChildProcess } from "node:child_process";
|
||||
import { EventEmitter } from "node:events";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { type BskRunResult, createBskRunner, isCommandNotFound, parseBskJson } from "../src/runner";
|
||||
|
||||
/** Minimal fake ChildProcess driven by the test. */
|
||||
class FakeChild extends EventEmitter {
|
||||
stdout = new EventEmitter();
|
||||
stderr = new EventEmitter();
|
||||
exitCode: number | null = null;
|
||||
signalCode: string | null = null;
|
||||
killedWith: string[] = [];
|
||||
|
||||
kill(signal: string): boolean {
|
||||
if (this.exitCode !== null || this.signalCode !== null) return false;
|
||||
this.killedWith.push(signal);
|
||||
this.signalCode = signal;
|
||||
queueMicrotask(() => this.emit("close", null));
|
||||
return true;
|
||||
}
|
||||
|
||||
finish(code: number, stdout = "", stderr = ""): void {
|
||||
if (stdout) this.stdout.emit("data", stdout);
|
||||
if (stderr) this.stderr.emit("data", stderr);
|
||||
this.exitCode = code;
|
||||
this.emit("close", code);
|
||||
}
|
||||
}
|
||||
|
||||
function fakeSpawn(children: FakeChild[]) {
|
||||
return () => {
|
||||
const child = children.shift();
|
||||
if (child === undefined) throw new Error("no fake child queued");
|
||||
return child as unknown as ChildProcess;
|
||||
};
|
||||
}
|
||||
|
||||
describe("createBskRunner", () => {
|
||||
it("appends --json and collects stdout", async () => {
|
||||
const child = new FakeChild();
|
||||
const seen: string[][] = [];
|
||||
const runner = createBskRunner("bsk", (cmd: string, args: string[]) => {
|
||||
seen.push([cmd, ...args]);
|
||||
return child as unknown as ChildProcess;
|
||||
});
|
||||
const promise = runner.run(["session", "list"]);
|
||||
child.finish(0, '{"ok":true}');
|
||||
const result = await promise;
|
||||
expect(seen).toEqual([["bsk", "session", "list", "--json"]]);
|
||||
expect(result).toMatchObject({
|
||||
code: 0,
|
||||
stdout: '{"ok":true}',
|
||||
aborted: false,
|
||||
timedOut: false,
|
||||
});
|
||||
});
|
||||
|
||||
it("kills the child when the abort signal fires", async () => {
|
||||
const child = new FakeChild();
|
||||
const runner = createBskRunner("bsk", fakeSpawn([child]));
|
||||
const controller = new AbortController();
|
||||
const promise = runner.run(["snapshot"], { signal: controller.signal });
|
||||
controller.abort();
|
||||
const result = await promise;
|
||||
expect(result.aborted).toBe(true);
|
||||
expect(child.killedWith).toContain("SIGTERM");
|
||||
});
|
||||
|
||||
it("kills the child on timeout", async () => {
|
||||
const child = new FakeChild();
|
||||
const runner = createBskRunner("bsk", fakeSpawn([child]));
|
||||
const result = await runner.run(["snapshot"], { timeoutMs: 5 });
|
||||
expect(result.timedOut).toBe(true);
|
||||
expect(child.killedWith.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("killAll terminates in-flight children", async () => {
|
||||
const child = new FakeChild();
|
||||
const runner = createBskRunner("bsk", fakeSpawn([child]));
|
||||
const promise = runner.run(["session", "list"]);
|
||||
runner.killAll();
|
||||
const result = await promise;
|
||||
expect(child.killedWith).toContain("SIGTERM");
|
||||
expect(result.code).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseBskJson", () => {
|
||||
const base: BskRunResult = { code: 0, stdout: "", stderr: "", timedOut: false, aborted: false };
|
||||
|
||||
it("parses stdout JSON on success", () => {
|
||||
expect(parseBskJson({ ...base, stdout: '{"a":1}' }, "status")).toEqual({ a: 1 });
|
||||
});
|
||||
|
||||
it("maps the JSON error envelope on non-zero exit", () => {
|
||||
const body = JSON.stringify({
|
||||
code: "no_browser",
|
||||
message: "no browser connected",
|
||||
hint: "open Chrome",
|
||||
});
|
||||
try {
|
||||
parseBskJson({ ...base, code: 3, stdout: body }, "session start");
|
||||
expect.unreachable();
|
||||
} catch (error) {
|
||||
expect(error).toMatchObject({
|
||||
name: "BskError",
|
||||
code: "no_browser",
|
||||
hint: "open Chrome",
|
||||
exitCode: 3,
|
||||
});
|
||||
expect((error as Error).message).toContain("no browser connected");
|
||||
expect((error as Error).message).toContain("hint: open Chrome");
|
||||
}
|
||||
});
|
||||
|
||||
it("falls back to stderr when the error body is not JSON", () => {
|
||||
expect(() => parseBskJson({ ...base, code: 1, stderr: "boom" }, "x")).toThrow(/boom/);
|
||||
});
|
||||
|
||||
it("reports killed-by-interrupt children (null exit code) as interrupted", () => {
|
||||
expect(() => parseBskJson({ ...base, code: null }, "navigate")).toThrow(/interrupted/);
|
||||
});
|
||||
|
||||
it("reports timeouts distinctly", () => {
|
||||
expect(() => parseBskJson({ ...base, timedOut: true }, "x")).toThrow(/timed out/);
|
||||
});
|
||||
|
||||
it("rejects non-JSON success output", () => {
|
||||
expect(() => parseBskJson({ ...base, stdout: "not json" }, "x")).toThrow(
|
||||
/did not produce JSON/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("isCommandNotFound", () => {
|
||||
it("detects ENOENT spawn failures", () => {
|
||||
expect(
|
||||
isCommandNotFound(Object.assign(new Error("spawn bsk ENOENT"), { code: "ENOENT" })),
|
||||
).toBe(true);
|
||||
expect(isCommandNotFound(new Error("other"))).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,108 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { SessionRegistry } from "../src/sessions";
|
||||
|
||||
/** Drive one full successful start through the reservation protocol. */
|
||||
function start(registry: SessionRegistry, sessionId: string): void {
|
||||
registry.reserveStart();
|
||||
registry.completeStart({ sessionId, startedAtMs: Date.now() });
|
||||
}
|
||||
|
||||
describe("SessionRegistry", () => {
|
||||
it("tracks the current session across start and remove", () => {
|
||||
const registry = new SessionRegistry(5);
|
||||
expect(registry.current()).toBeUndefined();
|
||||
start(registry, "s1");
|
||||
expect(registry.current()).toBe("s1");
|
||||
start(registry, "s2");
|
||||
expect(registry.current()).toBe("s2");
|
||||
registry.remove("s2");
|
||||
expect(registry.current()).toBe("s1");
|
||||
registry.remove("s1");
|
||||
expect(registry.current()).toBeUndefined();
|
||||
});
|
||||
|
||||
it("enforces the concurrency cap including in-flight starts", () => {
|
||||
const registry = new SessionRegistry(2);
|
||||
start(registry, "s1");
|
||||
start(registry, "s2");
|
||||
expect(() => registry.reserveStart()).toThrow(/session limit/);
|
||||
registry.remove("s1");
|
||||
registry.reserveStart();
|
||||
registry.completeStart({ sessionId: "s3", startedAtMs: Date.now() });
|
||||
expect(registry.size()).toBe(2);
|
||||
});
|
||||
|
||||
it("counts reserved (not yet spawned) starts against the cap — the race case", () => {
|
||||
const registry = new SessionRegistry(2);
|
||||
registry.reserveStart();
|
||||
registry.reserveStart();
|
||||
// Both slots are in flight; a third concurrent start must reject even
|
||||
// though no session has been registered yet.
|
||||
expect(() => registry.reserveStart()).toThrow(/session limit/);
|
||||
registry.abandonStart();
|
||||
registry.reserveStart();
|
||||
registry.completeStart({ sessionId: "s1", startedAtMs: Date.now() });
|
||||
expect(registry.ownedIds()).toEqual(["s1"]);
|
||||
});
|
||||
|
||||
it("resolve accepts an explicit owned session and makes it current", () => {
|
||||
const registry = new SessionRegistry(5);
|
||||
start(registry, "s1");
|
||||
start(registry, "s2");
|
||||
expect(registry.resolve("s1", "tool")).toBe("s1");
|
||||
expect(registry.current()).toBe("s1");
|
||||
});
|
||||
|
||||
it("resolve rejects foreign ids and never adopts them", () => {
|
||||
const registry = new SessionRegistry(5);
|
||||
start(registry, "s1");
|
||||
expect(() => registry.resolve("foreign", "tool")).toThrow(/does not belong to this plugin/);
|
||||
expect(registry.isOwned("foreign")).toBe(false);
|
||||
expect(registry.ownedIds()).toEqual(["s1"]);
|
||||
// A rejected resolve must not move the current pointer.
|
||||
expect(registry.current()).toBe("s1");
|
||||
});
|
||||
|
||||
it("resolve falls back to the current session", () => {
|
||||
const registry = new SessionRegistry(5);
|
||||
start(registry, "s1");
|
||||
expect(registry.resolve(undefined, "tool")).toBe("s1");
|
||||
});
|
||||
|
||||
it("resolve throws actionable guidance when no session exists", () => {
|
||||
const registry = new SessionRegistry(5);
|
||||
expect(() => registry.resolve(undefined, "browser_click")).toThrow(/browser_session_start/);
|
||||
});
|
||||
|
||||
it("recency order survives remove of the current session", () => {
|
||||
const registry = new SessionRegistry(5);
|
||||
start(registry, "s1");
|
||||
start(registry, "s2");
|
||||
registry.resolve("s1", "tool");
|
||||
registry.remove("s1");
|
||||
expect(registry.current()).toBe("s2");
|
||||
});
|
||||
|
||||
it("resolveForStop accepts owned sessions and refuses foreign ones", () => {
|
||||
const registry = new SessionRegistry(5);
|
||||
start(registry, "s1");
|
||||
expect(registry.resolveForStop(undefined)).toBe("s1");
|
||||
expect(registry.resolveForStop("s1")).toBe("s1");
|
||||
expect(() => registry.resolveForStop("foreign")).toThrow(/does not belong to this plugin/);
|
||||
expect(() => registry.resolveForStop("never-seen")).toThrow(/does not belong to this plugin/);
|
||||
});
|
||||
|
||||
it("a refused stop does not move the current pointer", () => {
|
||||
const registry = new SessionRegistry(5);
|
||||
start(registry, "s1");
|
||||
expect(() => registry.resolveForStop("foreign")).toThrow();
|
||||
expect(registry.current()).toBe("s1");
|
||||
});
|
||||
|
||||
it("ownedIds always equals every registered session (foreign ids never enter)", () => {
|
||||
const registry = new SessionRegistry(5);
|
||||
start(registry, "own1");
|
||||
start(registry, "own2");
|
||||
expect(registry.ownedIds().sort()).toEqual(["own1", "own2"]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,74 @@
|
||||
// BSK skill injection: registration wiring, catalog content, progressive-load
|
||||
// weight, and silent degradation without the skill seam.
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { registerBskSkill } from "../src/skill";
|
||||
|
||||
function fakeCtx(skills?: unknown) {
|
||||
return { get: (key: string) => (key === "skills" ? skills : undefined) } as never;
|
||||
}
|
||||
|
||||
describe("registerBskSkill", () => {
|
||||
it("registers the catalog entry with the assembled body", () => {
|
||||
const registrations: { skill: Record<string, unknown>; disposer: () => void }[] = [];
|
||||
const skills = {
|
||||
register(skill: Record<string, unknown>) {
|
||||
const disposer = () => {
|
||||
skill.__disposed = true;
|
||||
};
|
||||
registrations.push({ skill, disposer });
|
||||
return disposer;
|
||||
},
|
||||
};
|
||||
const unregister = registerBskSkill(fakeCtx(skills));
|
||||
expect(registrations).toHaveLength(1);
|
||||
const skill = registrations[0].skill;
|
||||
expect(skill.name).toBe("browser-skill");
|
||||
expect(typeof skill.description).toBe("string");
|
||||
expect(String(skill.description)).toContain("browser_*");
|
||||
// body = dsh prelude + canonical CLI skill body + dsh postlude
|
||||
const content = String(skill.content);
|
||||
expect(content).toContain("browser_observe");
|
||||
expect(String(skill.description)).toContain("never invoke bsk through bash or shell");
|
||||
expect(content).toContain("Always call those injected `browser_*` tools");
|
||||
expect(content).toContain("Never use `bash`, `shell`, `exec`");
|
||||
expect(content).toContain("## DSH routing reminder");
|
||||
expect(content.trimEnd()).toMatch(
|
||||
/always begin with[\s\S]*`browser_session_start`[\s\S]*`browser_session_stop`\.$/,
|
||||
);
|
||||
expect(content).toContain("Owned sessions only");
|
||||
expect(content).toContain("Mandatory workflow");
|
||||
expect(content).toContain("Refs invalidate after navigation");
|
||||
expect(content.length).toBeGreaterThan(10_000);
|
||||
expect(skill.source).toBe("bundled");
|
||||
// the original YAML frontmatter must not leak into the body
|
||||
expect(content.includes("name: browser-skill\ndescription:")).toBe(false);
|
||||
// disposer passthrough
|
||||
unregister();
|
||||
expect(skill.__disposed).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps one in-memory copy: repeated reads share the same content", () => {
|
||||
let captured: Record<string, unknown> | undefined;
|
||||
const skills = {
|
||||
register(skill: Record<string, unknown>) {
|
||||
captured = skill;
|
||||
return () => {};
|
||||
},
|
||||
};
|
||||
registerBskSkill(fakeCtx(skills));
|
||||
// Pre-step catalog snapshots re-read the registration: the body must be
|
||||
// the identical in-memory string every time (no reload, no copy).
|
||||
expect(captured?.content).toBe(captured?.content);
|
||||
expect(typeof captured?.content).toBe("string");
|
||||
});
|
||||
|
||||
it("degrades silently when the skills seam is absent or foreign", () => {
|
||||
expect(() => registerBskSkill(fakeCtx())).not.toThrow();
|
||||
expect(() => registerBskSkill(fakeCtx(null))).not.toThrow();
|
||||
expect(() => registerBskSkill(fakeCtx({}))).not.toThrow();
|
||||
const disposer = registerBskSkill(fakeCtx());
|
||||
expect(typeof disposer).toBe("function");
|
||||
expect(() => disposer()).not.toThrow();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,821 @@
|
||||
import { existsSync, mkdtempSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { ToolDefinition, ToolRunContext } from "@deepseek-ai/dsh-tools";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { ObservationService } from "../src/observation";
|
||||
import { KeyedExecutor } from "../src/queue";
|
||||
import type { BskRunner, BskRunOptions, BskRunResult } from "../src/runner";
|
||||
import { SessionRegistry } from "../src/sessions";
|
||||
import { type PluginConfig, registerTools } from "../src/tools";
|
||||
|
||||
const CONFIG: PluginConfig = {
|
||||
bskPath: "bsk",
|
||||
defaultTimeoutMs: 120_000,
|
||||
maxSessions: 5,
|
||||
observationEnabled: false,
|
||||
thumbnailIntervalMs: 1500,
|
||||
idleIntervalMs: 8000,
|
||||
lazyTools: false,
|
||||
};
|
||||
|
||||
/** An observation service with the feature off: instrumentation becomes a no-op. */
|
||||
function disabledObservation(deps: { ctx: unknown; runner: BskRunner; registry: SessionRegistry }) {
|
||||
return new ObservationService({
|
||||
ctx: deps.ctx as never,
|
||||
runner: deps.runner,
|
||||
registry: deps.registry,
|
||||
queue: new KeyedExecutor(),
|
||||
options: { enabled: false, thumbnailIntervalMs: 1500, idleIntervalMs: 8000 },
|
||||
});
|
||||
}
|
||||
|
||||
interface FakeCall {
|
||||
args: string[];
|
||||
options: BskRunOptions;
|
||||
}
|
||||
|
||||
/** Mark a canned answer as a per-call sequence (each call shifts one reply). */
|
||||
function seq(items: unknown[]): { __seq: unknown[] } {
|
||||
return { __seq: items };
|
||||
}
|
||||
|
||||
/** A runner that answers canned JSON per command prefix and records calls. */
|
||||
function fakeRunner(responses: Record<string, unknown>) {
|
||||
const calls: FakeCall[] = [];
|
||||
return {
|
||||
calls,
|
||||
runner: {
|
||||
async run(args: string[], options: BskRunOptions = {}): Promise<BskRunResult> {
|
||||
calls.push({ args, options });
|
||||
if (options.signal?.aborted) {
|
||||
return { code: null, stdout: "", stderr: "", timedOut: false, aborted: true };
|
||||
}
|
||||
for (const [prefix, response] of Object.entries(responses)) {
|
||||
if (args.join(" ").startsWith(prefix)) {
|
||||
if (response instanceof Error) throw response;
|
||||
const reply =
|
||||
typeof response === "object" && response !== null && "__seq" in response
|
||||
? (response as { __seq: unknown[] }).__seq.shift()
|
||||
: response;
|
||||
return {
|
||||
code: 0,
|
||||
stdout: JSON.stringify(reply),
|
||||
stderr: "",
|
||||
timedOut: false,
|
||||
aborted: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
return {
|
||||
code: 2,
|
||||
stdout: JSON.stringify({
|
||||
code: "unexpected",
|
||||
message: `no canned response for: ${args.join(" ")}`,
|
||||
}),
|
||||
stderr: "",
|
||||
timedOut: false,
|
||||
aborted: false,
|
||||
};
|
||||
},
|
||||
killAll() {},
|
||||
killFor: () => 0,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function makeCtx(services: Record<string, unknown> = {}) {
|
||||
const tools = new Map<string, ToolDefinition>();
|
||||
const ctx = {
|
||||
tools: { register: (def: ToolDefinition) => tools.set(def.name, def) },
|
||||
get: (key: string) => services[key],
|
||||
};
|
||||
return { ctx, tools };
|
||||
}
|
||||
|
||||
function makeExec(overrides: Record<string, unknown> = {}): ToolRunContext {
|
||||
return {
|
||||
callId: "test-call",
|
||||
name: "test",
|
||||
signal: new AbortController().signal,
|
||||
...overrides,
|
||||
} as unknown as ToolRunContext;
|
||||
}
|
||||
|
||||
const SNAPSHOT_REPLY = {
|
||||
text: '- button "OK" [ref=@e1]',
|
||||
ref_count: 1,
|
||||
tab_id: 7,
|
||||
truncated: false,
|
||||
};
|
||||
|
||||
function setup(
|
||||
responses: Record<string, unknown>,
|
||||
services: Record<string, unknown> = {},
|
||||
maxSessions = 5,
|
||||
) {
|
||||
const { ctx, tools } = makeCtx(services);
|
||||
const { runner, calls } = fakeRunner(responses);
|
||||
const registry = new SessionRegistry(maxSessions);
|
||||
const config = { ...CONFIG, maxSessions };
|
||||
registerTools({
|
||||
ctx: ctx as never,
|
||||
runner: runner as BskRunner,
|
||||
registry,
|
||||
config,
|
||||
observation: disabledObservation({ ctx, runner: runner as BskRunner, registry }),
|
||||
queue: new KeyedExecutor(),
|
||||
});
|
||||
return { tools, calls, registry };
|
||||
}
|
||||
|
||||
async function startSession(tools: Map<string, ToolDefinition>, sessionId = "s1") {
|
||||
const tool = tools.get("browser_session_start");
|
||||
if (tool === undefined) throw new Error("browser_session_start not registered");
|
||||
return tool.execute({}, makeExec()) as Promise<{ sessionId: string }>;
|
||||
}
|
||||
|
||||
const START_REPLY = (id: string) => ({ session_id: id, browser_instance_id: "chrome-1" });
|
||||
|
||||
describe("tool registration", () => {
|
||||
it("registers the full browser tool suite", () => {
|
||||
const { tools } = setup({});
|
||||
expect([...tools.keys()].sort()).toEqual([
|
||||
"browser_click",
|
||||
"browser_emulate",
|
||||
"browser_fill",
|
||||
"browser_navigate",
|
||||
"browser_observe",
|
||||
"browser_press",
|
||||
"browser_screenshot",
|
||||
"browser_session_list",
|
||||
"browser_session_start",
|
||||
"browser_session_stop",
|
||||
"browser_snapshot",
|
||||
]);
|
||||
});
|
||||
|
||||
it("validates model args through defineTool (missing required param)", async () => {
|
||||
const { tools } = setup({});
|
||||
const navigate = tools.get("browser_navigate");
|
||||
await expect(navigate?.execute({}, makeExec())).rejects.toThrow(/invalid arguments/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("browser_session_start", () => {
|
||||
it("maps the start reply and tracks the session as current", async () => {
|
||||
const { tools, registry } = setup({ "session start": START_REPLY("s1") });
|
||||
const value = await startSession(tools);
|
||||
expect(value.sessionId).toBe("s1");
|
||||
expect(registry.current()).toBe("s1");
|
||||
});
|
||||
|
||||
it("passes window size and opens the initial URL and device preset", async () => {
|
||||
const { tools, calls } = setup({
|
||||
"session start": START_REPLY("s1"),
|
||||
emulate: { tab_id: 7, cleared: false },
|
||||
navigate: { tab_id: 7, url: "https://example.com", reached: "load" },
|
||||
});
|
||||
const tool = tools.get("browser_session_start");
|
||||
const value = (await tool?.execute(
|
||||
{ width: 1280, height: 800, url: "https://example.com", device: "iphone-14", noFocus: true },
|
||||
makeExec(),
|
||||
)) as { sessionId: string; url: string; device: string };
|
||||
expect(value).toMatchObject({
|
||||
sessionId: "s1",
|
||||
url: "https://example.com",
|
||||
device: "iphone-14",
|
||||
});
|
||||
expect(calls[0].args).toEqual([
|
||||
"session",
|
||||
"start",
|
||||
"--width",
|
||||
"1280",
|
||||
"--height",
|
||||
"800",
|
||||
"--no-focus",
|
||||
]);
|
||||
expect(calls[1].args).toEqual(["emulate", "--session", "s1", "--device", "iphone-14"]);
|
||||
expect(calls[2].args).toEqual(["navigate", "--session", "s1", "https://example.com"]);
|
||||
});
|
||||
|
||||
it("requires width and height together", async () => {
|
||||
const { tools } = setup({ "session start": START_REPLY("s1") });
|
||||
const tool = tools.get("browser_session_start");
|
||||
await expect(tool?.execute({ width: 1280 }, makeExec())).rejects.toThrow(/together/);
|
||||
});
|
||||
|
||||
it("cleans up a half-initialized session when the initial navigate fails", async () => {
|
||||
const { tools, calls, registry } = setup({
|
||||
"session start": START_REPLY("s1"),
|
||||
"session stop": {},
|
||||
});
|
||||
// navigate has no canned response -> non-zero exit -> error.
|
||||
const tool = tools.get("browser_session_start");
|
||||
await expect(tool?.execute({ url: "https://example.com" }, makeExec())).rejects.toThrow(
|
||||
/navigate/,
|
||||
);
|
||||
expect(calls.some((c) => c.args.join(" ") === "session stop s1")).toBe(true);
|
||||
expect(registry.current()).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("multi-session behavior", () => {
|
||||
const responses = {
|
||||
"session start": START_REPLY("s1"),
|
||||
snapshot: SNAPSHOT_REPLY,
|
||||
};
|
||||
|
||||
it("operation tools default to the current session and echo it back", async () => {
|
||||
const { tools, calls } = setup(responses);
|
||||
await startSession(tools);
|
||||
const snapshot = tools.get("browser_snapshot");
|
||||
const value = (await snapshot?.execute({}, makeExec())) as {
|
||||
session: string;
|
||||
refCount: number;
|
||||
};
|
||||
expect(value).toMatchObject({ session: "s1", refCount: 1 });
|
||||
expect(calls[1].args).toEqual(["snapshot", "--session", "s1"]);
|
||||
});
|
||||
|
||||
it("an explicit owned session wins and becomes the new current session", async () => {
|
||||
const { tools, calls, registry } = setup({
|
||||
"session start": seq([START_REPLY("s1"), START_REPLY("s2")]),
|
||||
snapshot: SNAPSHOT_REPLY,
|
||||
});
|
||||
await startSession(tools, "s1");
|
||||
await startSession(tools, "s2");
|
||||
const snapshot = tools.get("browser_snapshot");
|
||||
const value = (await snapshot?.execute({ session: "s1" }, makeExec())) as {
|
||||
session: string;
|
||||
};
|
||||
expect(value.session).toBe("s1");
|
||||
expect(calls[2].args).toEqual(["snapshot", "--session", "s1"]);
|
||||
expect(registry.current()).toBe("s1");
|
||||
});
|
||||
|
||||
it("rejects an explicit foreign session without touching the daemon", async () => {
|
||||
const { tools, calls } = setup(responses);
|
||||
await startSession(tools);
|
||||
const callsBefore = calls.length;
|
||||
const snapshot = tools.get("browser_snapshot");
|
||||
await expect(snapshot?.execute({ session: "other" }, makeExec())).rejects.toThrow(
|
||||
/does not belong to this plugin/,
|
||||
);
|
||||
expect(calls.length).toBe(callsBefore);
|
||||
});
|
||||
|
||||
it("fails with guidance when no session exists", async () => {
|
||||
const { tools } = setup(responses);
|
||||
const snapshot = tools.get("browser_snapshot");
|
||||
await expect(snapshot?.execute({}, makeExec())).rejects.toThrow(/browser_session_start/);
|
||||
});
|
||||
|
||||
it("enforces the configured session cap before spawning", async () => {
|
||||
const { tools, calls } = setup(
|
||||
{ "session start": seq([START_REPLY("s1"), START_REPLY("s2")]) },
|
||||
{},
|
||||
1,
|
||||
);
|
||||
await startSession(tools);
|
||||
const start = tools.get("browser_session_start");
|
||||
await expect(start?.execute({}, makeExec())).rejects.toThrow(/session limit/);
|
||||
// A rejected start must not spawn another bsk session (no leaked session).
|
||||
expect(calls.filter((c) => c.args.join(" ").startsWith("session start"))).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("shares the cap atomically across concurrent starts (reservation race)", async () => {
|
||||
const { tools, calls } = setup(
|
||||
{ "session start": seq([START_REPLY("s1"), START_REPLY("s2")]) },
|
||||
{},
|
||||
1,
|
||||
);
|
||||
const start = tools.get("browser_session_start");
|
||||
if (start === undefined) throw new Error("not registered");
|
||||
// Two truly concurrent starts against a cap of 1: exactly one may spawn.
|
||||
const outcomes = await Promise.allSettled([
|
||||
start.execute({}, makeExec()),
|
||||
start.execute({}, makeExec()),
|
||||
]);
|
||||
const fulfilled = outcomes.filter((o) => o.status === "fulfilled");
|
||||
const rejected = outcomes.filter((o) => o.status === "rejected");
|
||||
expect(fulfilled).toHaveLength(1);
|
||||
expect(rejected).toHaveLength(1);
|
||||
expect(String((rejected[0] as PromiseRejectedResult).reason)).toMatch(/session limit/);
|
||||
expect(calls.filter((c) => c.args.join(" ").startsWith("session start"))).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("refuses to stop a session the plugin did not create", async () => {
|
||||
const { tools, calls } = setup(responses);
|
||||
await startSession(tools);
|
||||
const stop = tools.get("browser_session_stop");
|
||||
await expect(stop?.execute({ session: "ext7" }, makeExec())).rejects.toThrow(
|
||||
/does not belong to this plugin/,
|
||||
);
|
||||
// No stop command may reach the daemon for a foreign session.
|
||||
expect(calls.some((c) => c.args.join(" ").startsWith("session stop"))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("browser_session_stop / list", () => {
|
||||
it("stops the current session and clears the registry", async () => {
|
||||
const { tools, calls, registry } = setup({
|
||||
"session start": START_REPLY("s1"),
|
||||
"session stop": {},
|
||||
});
|
||||
await startSession(tools);
|
||||
const stop = tools.get("browser_session_stop");
|
||||
const value = (await stop?.execute({}, makeExec())) as { stopped: string };
|
||||
expect(value.stopped).toBe("s1");
|
||||
expect(calls[1].args).toEqual(["session", "stop", "s1"]);
|
||||
expect(registry.current()).toBeUndefined();
|
||||
});
|
||||
|
||||
it("lists only plugin-owned sessions with the current marker (no daemon view)", async () => {
|
||||
const { tools, calls } = setup({
|
||||
"session start": seq([START_REPLY("s1"), START_REPLY("s2")]),
|
||||
});
|
||||
await startSession(tools);
|
||||
await startSession(tools);
|
||||
const list = tools.get("browser_session_list");
|
||||
const value = (await list?.execute({}, makeExec())) as {
|
||||
sessions: { sessionId: string; current: boolean }[];
|
||||
};
|
||||
expect(value.sessions).toEqual([
|
||||
{ sessionId: "s1", browserInstanceId: "chrome-1", current: false },
|
||||
{ sessionId: "s2", browserInstanceId: "chrome-1", current: true },
|
||||
]);
|
||||
// Registry-only: listing must not call the daemon at all.
|
||||
expect(calls.some((c) => c.args.join(" ").startsWith("session list"))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("interaction tools", () => {
|
||||
const responses = {
|
||||
"session start": START_REPLY("s1"),
|
||||
click: { tab_id: 7, used_ref: "@e1", x: 10, y: 20 },
|
||||
fill: { tab_id: 7, used_ref: "@e2", value_length: 5 },
|
||||
press: { tab_id: 7, key: "Enter", code: "Enter", modifiers: [] },
|
||||
navigate: { tab_id: 7, url: "https://a.test", final_url: "https://b.test", reached: "load" },
|
||||
emulate: { tab_id: 7, cleared: false },
|
||||
};
|
||||
|
||||
it("browser_click maps target, button, and click count", async () => {
|
||||
const { tools, calls } = setup(responses);
|
||||
await startSession(tools);
|
||||
const click = tools.get("browser_click");
|
||||
const value = (await click?.execute(
|
||||
{ target: "@e1", button: "right", clickCount: 2 },
|
||||
makeExec(),
|
||||
)) as { session: string; x: number; y: number };
|
||||
expect(value).toMatchObject({ session: "s1", x: 10, y: 20 });
|
||||
expect(calls[1].args).toEqual([
|
||||
"click",
|
||||
"--session",
|
||||
"s1",
|
||||
"--button",
|
||||
"right",
|
||||
"--click-count",
|
||||
"2",
|
||||
"@e1",
|
||||
]);
|
||||
});
|
||||
|
||||
it("browser_fill passes value and --no-clear", async () => {
|
||||
const { tools, calls } = setup(responses);
|
||||
await startSession(tools);
|
||||
const fill = tools.get("browser_fill");
|
||||
await fill?.execute({ target: "e2", value: "hello", noClear: true }, makeExec());
|
||||
expect(calls[1].args).toEqual([
|
||||
"fill",
|
||||
"--session",
|
||||
"s1",
|
||||
"--value",
|
||||
"hello",
|
||||
"--no-clear",
|
||||
"e2",
|
||||
]);
|
||||
});
|
||||
|
||||
it("browser_press picks --ref for snapshot refs and --selector otherwise", async () => {
|
||||
const { tools, calls } = setup(responses);
|
||||
await startSession(tools);
|
||||
const press = tools.get("browser_press");
|
||||
await press?.execute({ key: "Enter", target: "@e3" }, makeExec());
|
||||
expect(calls[1].args).toEqual(["press", "--session", "s1", "--ref", "@e3", "Enter"]);
|
||||
await press?.execute({ key: "Ctrl+A", target: "#field" }, makeExec());
|
||||
expect(calls[2].args).toEqual(["press", "--session", "s1", "--selector", "#field", "Ctrl+A"]);
|
||||
});
|
||||
|
||||
it("browser_navigate converts waitUntil and timeoutMs", async () => {
|
||||
const { tools, calls } = setup(responses);
|
||||
await startSession(tools);
|
||||
const navigate = tools.get("browser_navigate");
|
||||
const value = (await navigate?.execute(
|
||||
{ url: "https://a.test", waitUntil: "networkidle", timeoutMs: 1500 },
|
||||
makeExec(),
|
||||
)) as { finalUrl: string; reached: string };
|
||||
expect(value).toMatchObject({ finalUrl: "https://b.test", reached: "load" });
|
||||
expect(calls[1].args).toEqual([
|
||||
"navigate",
|
||||
"--session",
|
||||
"s1",
|
||||
"--wait-until",
|
||||
"networkidle",
|
||||
"--timeout",
|
||||
"1500ms",
|
||||
"https://a.test",
|
||||
]);
|
||||
});
|
||||
|
||||
it("browser_emulate rejects off combined with overrides", async () => {
|
||||
const { tools } = setup(responses);
|
||||
await startSession(tools);
|
||||
const emulate = tools.get("browser_emulate");
|
||||
await expect(emulate?.execute({ off: true, device: "iphone-14" }, makeExec())).rejects.toThrow(
|
||||
/mutually exclusive/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("browser_screenshot", () => {
|
||||
function pngFile(): string {
|
||||
const dir = mkdtempSync(join(tmpdir(), "bsk-test-"));
|
||||
const path = join(dir, "shot.png");
|
||||
// Minimal PNG header bytes are enough: the plugin never decodes the image.
|
||||
writeFileSync(path, Buffer.from([0x89, 0x50, 0x4e, 0x47, 1, 2, 3, 4]));
|
||||
return path;
|
||||
}
|
||||
|
||||
it("returns the PNG path when no attachment store is mounted", async () => {
|
||||
const path = pngFile();
|
||||
const { tools, calls } = setup({
|
||||
"session start": START_REPLY("s1"),
|
||||
screenshot: { tab_id: 7, width: 800, height: 600, format: "png", path, byte_size: 8 },
|
||||
});
|
||||
await startSession(tools);
|
||||
const screenshot = tools.get("browser_screenshot");
|
||||
const value = (await screenshot?.execute({ ref: "@e1" }, makeExec())) as {
|
||||
path: string;
|
||||
image?: unknown;
|
||||
};
|
||||
expect(value.path).toBe(path);
|
||||
expect(value.image).toBeUndefined();
|
||||
expect(calls[1].args.slice(0, 3)).toEqual(["screenshot", "--session", "s1"]);
|
||||
expect(calls[1].args).toContain("--ref");
|
||||
const rendered = screenshot?.output.render({ session: "s1" }, value as never);
|
||||
expect(rendered?.every((block) => block.type === "text")).toBe(true);
|
||||
});
|
||||
|
||||
it("commits the image through the attachment store on an image-capable route", async () => {
|
||||
const path = pngFile();
|
||||
const services = {
|
||||
attachments: {
|
||||
imageLimits: {
|
||||
mediaTypes: ["image/png"],
|
||||
maxImageBytes: 10_000_000,
|
||||
maxMessageImageBytes: 10_000_000,
|
||||
},
|
||||
saveImage: async (input: { data: Uint8Array; mediaType: string; name?: string }) => ({
|
||||
attachmentId: "att-1",
|
||||
mediaType: input.mediaType,
|
||||
bytes: input.data.byteLength,
|
||||
width: 800,
|
||||
height: 600,
|
||||
name: input.name,
|
||||
}),
|
||||
},
|
||||
llm: {
|
||||
resolveModelInfo: async () => ({ inputModalities: ["text", "image"] }),
|
||||
},
|
||||
};
|
||||
const { tools } = setup(
|
||||
{
|
||||
"session start": START_REPLY("s1"),
|
||||
screenshot: { tab_id: 7, width: 800, height: 600, format: "png", path, byte_size: 8 },
|
||||
},
|
||||
services,
|
||||
);
|
||||
await startSession(tools);
|
||||
const exec = makeExec({
|
||||
agent: {
|
||||
session: { requestHeader: () => ({ config: { provider: "deepseek", model: "vl" } }) },
|
||||
options: {},
|
||||
},
|
||||
});
|
||||
const screenshot = tools.get("browser_screenshot");
|
||||
const value = (await screenshot?.execute({}, exec)) as {
|
||||
image?: { attachmentId: string; mediaType: string };
|
||||
};
|
||||
expect(value.image).toMatchObject({ attachmentId: "att-1", mediaType: "image/png" });
|
||||
const rendered = screenshot?.output.render({}, value as never) as { type: string }[];
|
||||
expect(rendered.map((block) => block.type)).toEqual(["text", "image"]);
|
||||
});
|
||||
|
||||
it("stays path-only when the model route is text-only", async () => {
|
||||
const path = pngFile();
|
||||
const services = {
|
||||
attachments: {
|
||||
imageLimits: { mediaTypes: ["image/png"], maxImageBytes: 1e7, maxMessageImageBytes: 1e7 },
|
||||
saveImage: async () => {
|
||||
throw new Error("must not be called");
|
||||
},
|
||||
},
|
||||
llm: { resolveModelInfo: async () => ({ inputModalities: ["text"] }) },
|
||||
};
|
||||
const { tools } = setup(
|
||||
{
|
||||
"session start": START_REPLY("s1"),
|
||||
screenshot: { tab_id: 7, width: 1, height: 1, format: "png", path, byte_size: 8 },
|
||||
},
|
||||
services,
|
||||
);
|
||||
await startSession(tools);
|
||||
const exec = makeExec({
|
||||
agent: {
|
||||
session: { requestHeader: () => ({ config: { provider: "p", model: "m" } }) },
|
||||
options: {},
|
||||
},
|
||||
});
|
||||
const screenshot = tools.get("browser_screenshot");
|
||||
const value = (await screenshot?.execute({}, exec)) as { image?: unknown; path: string };
|
||||
expect(value.image).toBeUndefined();
|
||||
expect(value.path).toBe(path);
|
||||
});
|
||||
});
|
||||
|
||||
describe("error and cancellation semantics", () => {
|
||||
it("surfaces the bsk JSON error envelope with code and hint", async () => {
|
||||
const { tools } = setup({
|
||||
"session start": START_REPLY("s1"),
|
||||
});
|
||||
await startSession(tools);
|
||||
const snapshot = tools.get("browser_snapshot");
|
||||
await expect(snapshot?.execute({}, makeExec())).rejects.toThrow(/no canned response/);
|
||||
});
|
||||
|
||||
it("turns an aborted child into an AbortError", async () => {
|
||||
const { tools } = setup({ "session start": START_REPLY("s1") });
|
||||
await startSession(tools);
|
||||
const snapshot = tools.get("browser_snapshot");
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
const exec = makeExec({ signal: controller.signal });
|
||||
await expect(snapshot?.execute({}, exec)).rejects.toMatchObject({ name: "AbortError" });
|
||||
});
|
||||
|
||||
it("maps a missing bsk binary onto install guidance", async () => {
|
||||
const { ctx, tools } = makeCtx();
|
||||
const registry = new SessionRegistry(5);
|
||||
const runner = {
|
||||
async run(): Promise<BskRunResult> {
|
||||
throw Object.assign(new Error("spawn bsk ENOENT"), { code: "ENOENT" });
|
||||
},
|
||||
killAll() {},
|
||||
killFor: () => 0,
|
||||
};
|
||||
registerTools({
|
||||
ctx: ctx as never,
|
||||
runner,
|
||||
registry,
|
||||
config: CONFIG,
|
||||
observation: disabledObservation({ ctx, runner, registry }),
|
||||
queue: new KeyedExecutor(),
|
||||
});
|
||||
const start = tools.get("browser_session_start");
|
||||
await expect(start?.execute({}, makeExec())).rejects.toThrow(/BrowserSkill must be installed/);
|
||||
});
|
||||
});
|
||||
|
||||
describe("presentation", () => {
|
||||
it("presents calls as terminal cards with the bsk command line", () => {
|
||||
const { tools } = setup({});
|
||||
const click = tools.get("browser_click");
|
||||
const view = click?.presentCall?.({ target: "@e1" }) as {
|
||||
card: string;
|
||||
title: string;
|
||||
};
|
||||
expect(view.card).toBe("terminal");
|
||||
expect(view.title).toContain("bsk click @e1");
|
||||
});
|
||||
|
||||
it("presents results as terminal cards carrying the rendered text", () => {
|
||||
const { tools } = setup({});
|
||||
const click = tools.get("browser_click");
|
||||
const view = click?.presentResult?.({ target: "@e1" }, {
|
||||
content: [{ type: "text", text: "clicked" }],
|
||||
isError: false,
|
||||
} as never) as { card: string; output: string; exitCode: number };
|
||||
expect(view).toMatchObject({ card: "terminal", output: "clicked", exitCode: 0 });
|
||||
});
|
||||
|
||||
it("projects the text block of multi-block results (screenshot text + image)", () => {
|
||||
const { tools } = setup({});
|
||||
const screenshot = tools.get("browser_screenshot");
|
||||
const view = screenshot?.presentResult?.({}, {
|
||||
content: [
|
||||
{ type: "text", text: "[session s1] screenshot of tab 7" },
|
||||
{ type: "image", attachment: { attachmentId: "a1" } },
|
||||
],
|
||||
isError: false,
|
||||
} as never) as { card: string; output: string };
|
||||
expect(view).toMatchObject({ card: "terminal", output: "[session s1] screenshot of tab 7" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("screenshot scratch file lifecycle", () => {
|
||||
/** Runner that honors --out like the real bsk (writes the PNG there). */
|
||||
function outWritingRunner() {
|
||||
const calls: FakeCall[] = [];
|
||||
return {
|
||||
calls,
|
||||
runner: {
|
||||
async run(args: string[], options: BskRunOptions = {}): Promise<BskRunResult> {
|
||||
calls.push({ args, options });
|
||||
if (args[0] === "session") {
|
||||
return {
|
||||
code: 0,
|
||||
stdout: JSON.stringify({ session_id: "s1", browser_instance_id: "chrome-1" }),
|
||||
stderr: "",
|
||||
timedOut: false,
|
||||
aborted: false,
|
||||
};
|
||||
}
|
||||
if (args[0] === "screenshot") {
|
||||
const out = args[args.indexOf("--out") + 1];
|
||||
writeFileSync(out, Buffer.from([0x89, 0x50, 0x4e, 0x47]));
|
||||
return {
|
||||
code: 0,
|
||||
stdout: JSON.stringify({
|
||||
tab_id: 7,
|
||||
width: 4,
|
||||
height: 4,
|
||||
format: "png",
|
||||
path: out,
|
||||
byte_size: 4,
|
||||
}),
|
||||
stderr: "",
|
||||
timedOut: false,
|
||||
aborted: false,
|
||||
};
|
||||
}
|
||||
return { code: 2, stdout: "{}", stderr: "", timedOut: false, aborted: false };
|
||||
},
|
||||
killAll() {},
|
||||
killFor: () => 0,
|
||||
} as BskRunner,
|
||||
};
|
||||
}
|
||||
|
||||
function setupWithRunner(runner: BskRunner, services: Record<string, unknown> = {}) {
|
||||
const { ctx, tools } = makeCtx(services);
|
||||
const registry = new SessionRegistry(5);
|
||||
registerTools({
|
||||
ctx: ctx as never,
|
||||
runner,
|
||||
registry,
|
||||
config: CONFIG,
|
||||
observation: disabledObservation({ ctx, runner, registry }),
|
||||
queue: new KeyedExecutor(),
|
||||
});
|
||||
return { tools, registry };
|
||||
}
|
||||
|
||||
const IMAGE_SERVICES = {
|
||||
attachments: {
|
||||
imageLimits: { mediaTypes: ["image/png"], maxImageBytes: 1e7, maxMessageImageBytes: 1e7 },
|
||||
saveImage: async (input: { data: Uint8Array; mediaType: string }) => ({
|
||||
attachmentId: "att-1",
|
||||
mediaType: input.mediaType,
|
||||
bytes: input.data.byteLength,
|
||||
width: 4,
|
||||
height: 4,
|
||||
}),
|
||||
},
|
||||
llm: { resolveModelInfo: async () => ({ inputModalities: ["text", "image"] }) },
|
||||
};
|
||||
|
||||
it("deletes the scratch PNG once the bytes are in the attachment store", async () => {
|
||||
const { runner, calls } = outWritingRunner();
|
||||
const { tools } = setupWithRunner(runner, IMAGE_SERVICES);
|
||||
await startSession(tools);
|
||||
const exec = makeExec({
|
||||
agent: {
|
||||
session: { requestHeader: () => ({ config: { provider: "p", model: "vl" } }) },
|
||||
options: {},
|
||||
},
|
||||
});
|
||||
const screenshot = tools.get("browser_screenshot");
|
||||
const value = (await screenshot?.execute({}, exec)) as { image?: unknown; path: string };
|
||||
expect(value.image).toBeDefined();
|
||||
const out = calls.find((c) => c.args[0] === "screenshot")?.args.at(-1);
|
||||
expect(out).toBeDefined();
|
||||
expect(existsSync(out ?? "")).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps the PNG only when it is the model-facing artifact (no store)", async () => {
|
||||
const { runner, calls } = outWritingRunner();
|
||||
const { tools } = setupWithRunner(runner);
|
||||
await startSession(tools);
|
||||
const screenshot = tools.get("browser_screenshot");
|
||||
const value = (await screenshot?.execute({}, makeExec())) as { image?: unknown; path: string };
|
||||
expect(value.image).toBeUndefined();
|
||||
const out = calls.find((c) => c.args[0] === "screenshot")?.args.at(-1);
|
||||
expect(existsSync(out ?? "")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("observation action instrumentation timing", () => {
|
||||
it("a queued call never overwrites the in-flight action label", async () => {
|
||||
const { ctx, tools } = makeCtx();
|
||||
const registry = new SessionRegistry(5);
|
||||
let releaseNavigate!: () => void;
|
||||
const navigateGate = new Promise<void>((resolve) => {
|
||||
releaseNavigate = resolve;
|
||||
});
|
||||
let releaseSnapshot!: () => void;
|
||||
const snapshotGate = new Promise<void>((resolve) => {
|
||||
releaseSnapshot = resolve;
|
||||
});
|
||||
const runner: BskRunner = {
|
||||
async run(args: string[]): Promise<BskRunResult> {
|
||||
if (args[0] === "session") {
|
||||
return {
|
||||
code: 0,
|
||||
stdout: JSON.stringify({ session_id: "s1", browser_instance_id: "chrome-1" }),
|
||||
stderr: "",
|
||||
timedOut: false,
|
||||
aborted: false,
|
||||
};
|
||||
}
|
||||
if (args[0] === "navigate") {
|
||||
await navigateGate;
|
||||
return {
|
||||
code: 0,
|
||||
stdout: JSON.stringify({ tab_id: 7, url: "http://x" }),
|
||||
stderr: "",
|
||||
timedOut: false,
|
||||
aborted: false,
|
||||
};
|
||||
}
|
||||
if (args[0] === "snapshot") {
|
||||
await snapshotGate;
|
||||
return {
|
||||
code: 0,
|
||||
stdout: JSON.stringify({ text: "ok", ref_count: 0, tab_id: 7, truncated: false }),
|
||||
stderr: "",
|
||||
timedOut: false,
|
||||
aborted: false,
|
||||
};
|
||||
}
|
||||
// Observation captures share the session queue; fail them fast so they
|
||||
// cannot stall the action-label assertions.
|
||||
if (args[0] === "screenshot") {
|
||||
return { code: 1, stdout: "", stderr: "skip", timedOut: false, aborted: false };
|
||||
}
|
||||
return {
|
||||
code: 0,
|
||||
stdout: JSON.stringify({ text: "ok", ref_count: 0, tab_id: 7, truncated: false }),
|
||||
stderr: "",
|
||||
timedOut: false,
|
||||
aborted: false,
|
||||
};
|
||||
},
|
||||
killAll() {},
|
||||
killFor: () => 0,
|
||||
};
|
||||
const queue = new KeyedExecutor();
|
||||
const observation = new ObservationService({
|
||||
ctx: ctx as never,
|
||||
runner,
|
||||
registry,
|
||||
queue,
|
||||
options: { enabled: true, thumbnailIntervalMs: 1500, idleIntervalMs: 8000 },
|
||||
});
|
||||
registerTools({ ctx: ctx as never, runner, registry, config: CONFIG, observation, queue });
|
||||
|
||||
await startSession(tools);
|
||||
const navigate = tools.get("browser_navigate");
|
||||
const snapshot = tools.get("browser_snapshot");
|
||||
const a = navigate?.execute({ url: "http://x" }, makeExec()) as Promise<unknown>;
|
||||
// A starts immediately (empty queue) and blocks on the gate.
|
||||
await vi.waitFor(() =>
|
||||
expect(observation.getState().find((s) => s.sessionId === "s1")?.action).toBe("navigating"),
|
||||
);
|
||||
// B queues behind A: the label must stay "navigating" until A settles.
|
||||
const b = snapshot?.execute({}, makeExec()) as Promise<unknown>;
|
||||
await new Promise((resolve) => setTimeout(resolve, 30));
|
||||
expect(observation.getState().find((s) => s.sessionId === "s1")?.action).toBe("navigating");
|
||||
releaseNavigate();
|
||||
await a;
|
||||
await vi.waitFor(() =>
|
||||
expect(observation.getState().find((s) => s.sessionId === "s1")?.action).toBe("snapshotting"),
|
||||
);
|
||||
releaseSnapshot();
|
||||
await b;
|
||||
await vi.waitFor(() =>
|
||||
expect(observation.getState().find((s) => s.sessionId === "s1")?.action).toBe("idle"),
|
||||
);
|
||||
observation.dispose();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"target": "ES2022",
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"jsx": "react-jsx",
|
||||
"lib": ["ES2022", "DOM", "DOM.Iterable"],
|
||||
"noEmit": true,
|
||||
"esModuleInterop": true,
|
||||
"forceConsistentCasingInFileNames": true,
|
||||
"resolveJsonModule": true,
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"types": ["node"]
|
||||
},
|
||||
"include": ["src/**/*", "tests/**/*", "tsdown.config.ts", "vitest.config.ts"]
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { basename, dirname, resolve as resolvePath } from "node:path";
|
||||
import { transform } from "lightningcss";
|
||||
import { defineConfig, type UserConfig } from "tsdown";
|
||||
import pkg from "./package.json" with { type: "json" };
|
||||
|
||||
/**
|
||||
* Two build faces of the dual-face package:
|
||||
* - host: the Cordis tool plugin (lib/index.mjs, ESM, dsh packages external);
|
||||
* - client: the Web UI bundle (lib/client.cjs, CJS in a type:module package) in dsh's closure-factory shape —
|
||||
* the artifact registers itself through `window.__ModuleLoader__.load`, with
|
||||
* platform modules resolved from the loader's frozen module table and every
|
||||
* other dependency inlined. Mirrors deepseek-harness `packages/client/tsdown.client.ts`.
|
||||
*/
|
||||
|
||||
/** Module specifiers the dsh web shell shares into its frozen module table. */
|
||||
const CLIENT_EXTERNALS: readonly string[] = [
|
||||
"react",
|
||||
"react/jsx-runtime",
|
||||
"react-dom",
|
||||
"react-dom/client",
|
||||
"@deepseek-ai/cordis",
|
||||
"@deepseek-ai/dsh-client-ui-slots",
|
||||
"@deepseek-ai/dsh-client-web-react",
|
||||
"@deepseek-ai/dsh-client-ui-primitives",
|
||||
"@deepseek-ai/dsh-client-ui-attachment",
|
||||
"@deepseek-ai/dsh-client-schema-form",
|
||||
"@deepseek-ai/dsh-client-runtime/client",
|
||||
];
|
||||
|
||||
/** Wire/type layers with no shared runtime identity: safe to inline. */
|
||||
const INLINE_SAFE = /^@deepseek-ai\/dsh-(host-apiproxy|session|llm|tools|brand|attachment)(\/|$)/;
|
||||
const VENDORED_LIBRARY = /^@deepseek-ai\/(cosmokit|schemastery)(\/|$)/;
|
||||
|
||||
const CSS_VIRTUAL_PREFIX = "\0bsk-css:";
|
||||
const CSS_VIRTUAL_SUFFIX = ".mjs";
|
||||
// The web shell resolves client bundles from its module table by package name
|
||||
// (dsh's own bundles register the same way), so this must never drift from
|
||||
// package.json.
|
||||
const CLIENT_ID = pkg.name;
|
||||
|
||||
const client: UserConfig = {
|
||||
name: `${CLIENT_ID}/client`,
|
||||
entry: { client: "src/client/index.ts" },
|
||||
outDir: "lib",
|
||||
format: "cjs",
|
||||
platform: "browser",
|
||||
dts: false,
|
||||
clean: false,
|
||||
external: [...CLIENT_EXTERNALS],
|
||||
// Anything not in the loader module table must inline; an unanswerable
|
||||
// require() is a guaranteed runtime throw.
|
||||
noExternal: (id: string) => (CLIENT_EXTERNALS.includes(id) ? undefined : true),
|
||||
define: {
|
||||
"process.env.NODE_ENV": JSON.stringify(process.env.NODE_ENV ?? "production"),
|
||||
"import.meta.env.MODE": JSON.stringify(process.env.NODE_ENV ?? "production"),
|
||||
"import.meta.env": JSON.stringify({ MODE: process.env.NODE_ENV ?? "production" }),
|
||||
},
|
||||
plugins: [
|
||||
{
|
||||
// Bundle purity gate: platform modules stay external, wire layers
|
||||
// inline, every other @deepseek-ai value import is a build error.
|
||||
name: "bsk-client-bundle-purity",
|
||||
resolveId(source: string) {
|
||||
if (!source.startsWith("@deepseek-ai/")) return null;
|
||||
if (CLIENT_EXTERNALS.includes(source)) return null;
|
||||
if (VENDORED_LIBRARY.test(source)) return null;
|
||||
if (INLINE_SAFE.test(source)) return null;
|
||||
throw new Error(
|
||||
`client bundle purity: "${source}" is not a platform module or an inline-safe wire layer — ` +
|
||||
"collaborate through cordis services (type-only imports are erased and never reach this gate)",
|
||||
);
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "bsk-css-modules-inline",
|
||||
resolveId(source: string, importer: string | undefined) {
|
||||
// `.module.css` gets hashed class maps; `.nomodule.css` is injected
|
||||
// verbatim (minified) — used for the scope-prefixed BSK utility sheet
|
||||
// and tokens, whose selectors must survive untouched.
|
||||
if (!source.endsWith(".module.css") && !source.endsWith(".nomodule.css")) return null;
|
||||
const abs = importer !== undefined ? resolvePath(dirname(importer), source) : source;
|
||||
return CSS_VIRTUAL_PREFIX + abs + CSS_VIRTUAL_SUFFIX;
|
||||
},
|
||||
async load(virtualId: string) {
|
||||
if (!virtualId.startsWith(CSS_VIRTUAL_PREFIX)) return null;
|
||||
const fileId = virtualId.slice(CSS_VIRTUAL_PREFIX.length, -CSS_VIRTUAL_SUFFIX.length);
|
||||
this.addWatchFile(fileId);
|
||||
const source = await readFile(fileId);
|
||||
const isModule = fileId.endsWith(".module.css");
|
||||
const { code, exports: cssExports } = transform({
|
||||
filename: fileId,
|
||||
code: source,
|
||||
...(isModule ? { cssModules: { pattern: "[hash]_[local]" } } : {}),
|
||||
minify: true,
|
||||
});
|
||||
const classMap: Record<string, string> = {};
|
||||
for (const [local, exp] of Object.entries(cssExports ?? {})) classMap[local] = exp.name;
|
||||
return [
|
||||
`const css = ${JSON.stringify(code.toString())};`,
|
||||
`const tagId = ${JSON.stringify(`${CLIENT_ID}/${basename(fileId)}`)};`,
|
||||
"if (typeof document !== 'undefined' && document.querySelector('style[data-plugin-css=' + JSON.stringify(tagId) + ']') === null) {",
|
||||
" const tag = document.createElement('style');",
|
||||
` tag.dataset.plugin = ${JSON.stringify(CLIENT_ID)};`,
|
||||
" tag.dataset.pluginCss = tagId;",
|
||||
" tag.textContent = css;",
|
||||
" document.head.appendChild(tag);",
|
||||
"}",
|
||||
`export default ${JSON.stringify(classMap)};`,
|
||||
].join("\n");
|
||||
},
|
||||
},
|
||||
],
|
||||
outputOptions: {
|
||||
entryFileNames: "client.cjs",
|
||||
banner: `window.__ModuleLoader__.load({ id: ${JSON.stringify(CLIENT_ID)}, factory: (require) => {`,
|
||||
footer: "return module.exports; } });",
|
||||
intro: "var module = { exports: {} }; var exports = module.exports;",
|
||||
},
|
||||
};
|
||||
|
||||
export default defineConfig([
|
||||
{
|
||||
entry: ["src/index.ts"],
|
||||
format: ["esm"],
|
||||
dts: true,
|
||||
outDir: "lib",
|
||||
// dsh host packages are provided by the profile the plugin is installed into.
|
||||
external: [/^@deepseek-ai\//],
|
||||
},
|
||||
client,
|
||||
]);
|
||||
@@ -0,0 +1,32 @@
|
||||
import { createRequire } from "node:module";
|
||||
import path from "node:path";
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
// @browser-skill/ui ships raw source and peers on react ^19, while the dsh
|
||||
// shell — and therefore the production client bundle, where react is
|
||||
// external — runs react 18. Pin every react import in tests to this
|
||||
// package's react 18 copy, or elements created inside ui components come out
|
||||
// as react-19 "transitional" elements that the react-18 renderer rejects.
|
||||
const reactDir = path.dirname(require.resolve("react/package.json"));
|
||||
const reactDomDir = path.dirname(require.resolve("react-dom/package.json"));
|
||||
|
||||
export default defineConfig({
|
||||
resolve: {
|
||||
alias: {
|
||||
react: reactDir,
|
||||
"react-dom": reactDomDir,
|
||||
},
|
||||
},
|
||||
test: {
|
||||
environment: "node",
|
||||
include: ["tests/**/*.test.ts", "tests/**/*.test.tsx"],
|
||||
server: {
|
||||
deps: {
|
||||
// dsh client packages ship ESM with CSS imports; run them through the
|
||||
// transform pipeline (where CSS is stubbed) instead of Node's loader.
|
||||
inline: [/@deepseek-ai\//],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
@@ -24,7 +24,90 @@ function scene(nodes: VomNode[]): VomScene {
|
||||
};
|
||||
}
|
||||
|
||||
function coreRefs(refs: ReturnType<typeof renderVom>["refs"]) {
|
||||
return refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }));
|
||||
}
|
||||
|
||||
describe("renderVom single-layer page", () => {
|
||||
it("does not derive handle context across frame scopes", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({
|
||||
id: 1,
|
||||
role: "RootWebArea",
|
||||
frameId: "main",
|
||||
contextScopeId: "main",
|
||||
}),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
role: "StaticText",
|
||||
name: "创建于",
|
||||
frameId: "main",
|
||||
contextScopeId: "main",
|
||||
}),
|
||||
node({
|
||||
id: 3,
|
||||
parentId: 1,
|
||||
role: "Iframe",
|
||||
frameId: "main",
|
||||
contextScopeId: "main",
|
||||
}),
|
||||
node({
|
||||
id: 4,
|
||||
parentId: 3,
|
||||
role: "button",
|
||||
name: "表格视图",
|
||||
frameId: "child",
|
||||
contextScopeId: "child",
|
||||
}),
|
||||
node({
|
||||
id: 5,
|
||||
parentId: 1,
|
||||
role: "button",
|
||||
name: "表格视图",
|
||||
frameId: "main",
|
||||
contextScopeId: "main",
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toContain('@e1 button "表格视图"');
|
||||
expect(out.text).not.toContain('@e1 button "表格视图" [ctx: 创建于]');
|
||||
});
|
||||
|
||||
it("preserves handle context within a frame scope", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", contextScopeId: "main" }),
|
||||
node({ id: 2, parentId: 1, role: "Iframe", contextScopeId: "main" }),
|
||||
node({
|
||||
id: 3,
|
||||
parentId: 2,
|
||||
role: "StaticText",
|
||||
name: "工具栏",
|
||||
contextScopeId: "child",
|
||||
}),
|
||||
node({
|
||||
id: 4,
|
||||
parentId: 2,
|
||||
role: "button",
|
||||
name: "更多",
|
||||
contextScopeId: "child",
|
||||
}),
|
||||
node({
|
||||
id: 5,
|
||||
parentId: 1,
|
||||
role: "button",
|
||||
name: "更多",
|
||||
contextScopeId: "main",
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toContain('@e1 button "更多" [ctx: 工具栏]');
|
||||
});
|
||||
|
||||
it("always emits a complete @vom single-layer document when there is no blocker", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
@@ -51,7 +134,7 @@ describe("renderVom single-layer page", () => {
|
||||
' @e2 button "加入购物车"',
|
||||
].join("\n"),
|
||||
);
|
||||
expect(out.refs).toEqual([
|
||||
expect(coreRefs(out.refs)).toEqual([
|
||||
{ ref: "e1", backendNodeId: 3 },
|
||||
{ ref: "e2", backendNodeId: 6 },
|
||||
]);
|
||||
@@ -72,7 +155,7 @@ describe("renderVom single-layer page", () => {
|
||||
expect(out.text).toContain(' heading "Title"');
|
||||
expect(out.text).toContain(' img "Logo"');
|
||||
expect(out.text).toContain(' @e1 button "Continue"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||
});
|
||||
|
||||
it("assigns refs to listbox controls", () => {
|
||||
@@ -84,7 +167,7 @@ describe("renderVom single-layer page", () => {
|
||||
);
|
||||
|
||||
expect(out.text).toContain(' @e1 listbox "Choices"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
it("annotates external links regardless of role casing", () => {
|
||||
@@ -121,7 +204,7 @@ describe("renderVom single-layer page", () => {
|
||||
expect(out.text).toContain(" alertdialog");
|
||||
expect(out.text).toContain(" section");
|
||||
expect(out.text).toContain(' @e1 button "Close"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 5 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 5 }]);
|
||||
});
|
||||
|
||||
it("skips generic nodes without adding an extra indentation level", () => {
|
||||
@@ -157,7 +240,7 @@ describe("renderVom single-layer page", () => {
|
||||
expect(out.text).not.toContain("hunter2");
|
||||
});
|
||||
|
||||
it("does not imply a sensitive textbox has a value when the value is missing", () => {
|
||||
it("renders a sensitive filled state without retaining its clear text value", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Login" }),
|
||||
@@ -168,6 +251,26 @@ describe("renderVom single-layer page", () => {
|
||||
name: "密码",
|
||||
tag: "input",
|
||||
sensitive: true,
|
||||
inputState: "filled",
|
||||
}),
|
||||
]),
|
||||
);
|
||||
|
||||
expect(out.text).toContain('@e1 textbox "密码" [filled] ="•••"');
|
||||
});
|
||||
|
||||
it("does not imply an empty sensitive textbox has a value", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Login" }),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
role: "textbox",
|
||||
name: "密码",
|
||||
tag: "input",
|
||||
sensitive: true,
|
||||
inputState: "empty",
|
||||
}),
|
||||
]),
|
||||
);
|
||||
@@ -192,7 +295,7 @@ describe("renderVom single-layer page", () => {
|
||||
expect(out.text).toContain(" section");
|
||||
expect(out.text).not.toContain("Too deep");
|
||||
expect(out.text).toContain(' @e1 button "Later"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||
});
|
||||
|
||||
it("truncates when maxTokens is exceeded while keeping refs in sync with rendered text", () => {
|
||||
@@ -227,7 +330,7 @@ describe("renderVom single-layer page", () => {
|
||||
);
|
||||
|
||||
expect(out.text).toContain('@e1 button "Open settings"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
it("keeps the deepest custom control when wrapper and child both look clickable", () => {
|
||||
@@ -257,7 +360,7 @@ describe("renderVom single-layer page", () => {
|
||||
|
||||
expect(out.text).not.toContain('@e1 button "Card"');
|
||||
expect(out.text).toContain('@e1 button "Details"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
||||
});
|
||||
|
||||
it("does not recover clickable wrappers around native controls", () => {
|
||||
@@ -280,7 +383,7 @@ describe("renderVom single-layer page", () => {
|
||||
|
||||
expect(out.text).not.toContain('@e1 button "Checkout"');
|
||||
expect(out.text).toContain('@e1 button "Pay"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 3 }]);
|
||||
});
|
||||
|
||||
it("adds context to duplicate weak action labels", () => {
|
||||
@@ -343,7 +446,7 @@ describe("renderVom single-layer page", () => {
|
||||
|
||||
expect(out.text).toContain("@layers 1 focus=L1");
|
||||
expect(out.text).toContain('@e1 button "Background"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
it("can filter refs blocked by active positioned regions without changing layer output", () => {
|
||||
@@ -386,7 +489,103 @@ describe("renderVom single-layer page", () => {
|
||||
expect(out.text).not.toContain("active-region");
|
||||
expect(out.text).not.toContain("Background");
|
||||
expect(out.text).toContain('@e1 button "Foreground"');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 4 }]);
|
||||
});
|
||||
|
||||
it("keeps paint-order comparisons inside their frame document", () => {
|
||||
const out = renderVom(
|
||||
{
|
||||
...scene([
|
||||
node({ id: 1, role: "RootWebArea", frameId: "main" }),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
role: "button",
|
||||
name: "Top-level action",
|
||||
tag: "button",
|
||||
rect: { x: 120, y: 120, w: 120, h: 40 },
|
||||
paintOrder: 2,
|
||||
frameId: "main",
|
||||
}),
|
||||
node({
|
||||
id: 3,
|
||||
parentId: 1,
|
||||
role: "Iframe",
|
||||
tag: "iframe",
|
||||
rect: { x: 0, y: 0, w: 1000, h: 700 },
|
||||
paintOrder: 1,
|
||||
frameId: "main",
|
||||
}),
|
||||
node({
|
||||
id: 4,
|
||||
parentId: 3,
|
||||
role: "generic",
|
||||
rect: { x: 0, y: 0, w: 1000, h: 700 },
|
||||
paintOrder: 100,
|
||||
position: "fixed",
|
||||
frameId: "child",
|
||||
}),
|
||||
node({
|
||||
id: 5,
|
||||
parentId: 4,
|
||||
role: "button",
|
||||
name: "Child action",
|
||||
tag: "button",
|
||||
rect: { x: 120, y: 120, w: 120, h: 40 },
|
||||
paintOrder: 101,
|
||||
frameId: "child",
|
||||
}),
|
||||
]),
|
||||
rootFrameId: "main",
|
||||
},
|
||||
{ activeRegionPolicy: true },
|
||||
);
|
||||
|
||||
expect(out.text).toContain("@layers 1 focus=L1");
|
||||
expect(out.text).toContain('button "Top-level action"');
|
||||
expect(out.text).toContain('button "Child action"');
|
||||
});
|
||||
|
||||
it("uses the iframe owner when a parent-frame region covers child content", () => {
|
||||
const out = renderVom(
|
||||
{
|
||||
...scene([
|
||||
node({ id: 1, role: "RootWebArea", frameId: "main" }),
|
||||
node({
|
||||
id: 2,
|
||||
parentId: 1,
|
||||
role: "Iframe",
|
||||
tag: "iframe",
|
||||
rect: { x: 100, y: 100, w: 400, h: 300 },
|
||||
paintOrder: 1,
|
||||
frameId: "main",
|
||||
}),
|
||||
node({
|
||||
id: 3,
|
||||
parentId: 2,
|
||||
role: "button",
|
||||
name: "Covered child action",
|
||||
tag: "button",
|
||||
rect: { x: 150, y: 150, w: 120, h: 40 },
|
||||
paintOrder: 100,
|
||||
frameId: "child",
|
||||
}),
|
||||
node({
|
||||
id: 4,
|
||||
parentId: 1,
|
||||
role: "generic",
|
||||
rect: { x: 120, y: 120, w: 200, h: 120 },
|
||||
paintOrder: 2,
|
||||
position: "fixed",
|
||||
frameId: "main",
|
||||
}),
|
||||
]),
|
||||
rootFrameId: "main",
|
||||
},
|
||||
{ activeRegionPolicy: true },
|
||||
);
|
||||
|
||||
expect(out.text).not.toContain("Covered child action");
|
||||
});
|
||||
|
||||
it("renders conditional surface items inline on the trigger", () => {
|
||||
@@ -405,7 +604,7 @@ describe("renderVom single-layer page", () => {
|
||||
});
|
||||
|
||||
expect(out.text).toContain('@e1 button "Products" [hover first: Shoes | Bags | Accessories]');
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
it("injects active scope blocks immediately after their trigger without refs", () => {
|
||||
@@ -431,7 +630,7 @@ describe("renderVom single-layer page", () => {
|
||||
" Bob - great sound",
|
||||
].join("\n"),
|
||||
);
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
it("does not render redundant children inside named ref controls", () => {
|
||||
@@ -445,7 +644,7 @@ describe("renderVom single-layer page", () => {
|
||||
|
||||
expect(out.text).toContain('@e1 button "Save"');
|
||||
expect(out.text).not.toContain("disk icon");
|
||||
expect(out.refs).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
expect(coreRefs(out.refs)).toEqual([{ ref: "e1", backendNodeId: 2 }]);
|
||||
});
|
||||
|
||||
it("does not render redundant children inside native text inputs", () => {
|
||||
@@ -612,7 +811,7 @@ describe("renderVom double-layer page", () => {
|
||||
].join("\n"),
|
||||
);
|
||||
expect(out.text).not.toContain("底层按钮");
|
||||
expect(out.refs).toEqual([
|
||||
expect(coreRefs(out.refs)).toEqual([
|
||||
{ ref: "e1", backendNodeId: 4 },
|
||||
{ ref: "e2", backendNodeId: 5 },
|
||||
]);
|
||||
@@ -664,4 +863,36 @@ describe("renderVom double-layer page", () => {
|
||||
expect(out.text).toContain(' @e2 textbox "密码" [filled] ="•••"');
|
||||
expect(out.refs.map((r) => r.backendNodeId)).toEqual([101, 102]);
|
||||
});
|
||||
|
||||
it("redacts form values and returns semantic metadata for rendered refs", () => {
|
||||
const out = renderVom(
|
||||
scene([
|
||||
node({ id: 1, role: "RootWebArea", name: "Form" }),
|
||||
node({
|
||||
id: 2,
|
||||
backendNodeId: 42,
|
||||
frameId: "child-frame",
|
||||
parentId: 1,
|
||||
role: "textbox",
|
||||
name: "Email",
|
||||
value: "user@example.com",
|
||||
inputState: "filled",
|
||||
tag: "input",
|
||||
}),
|
||||
]),
|
||||
{ redactValues: true },
|
||||
);
|
||||
|
||||
expect(out.text).toContain('[filled] ="•••"');
|
||||
expect(out.text).not.toContain("user@example.com");
|
||||
expect(out.refs[0]).toMatchObject({
|
||||
ref: "e1",
|
||||
backendNodeId: 42,
|
||||
frameId: "child-frame",
|
||||
role: "textbox",
|
||||
name: "Email",
|
||||
line: expect.any(Number),
|
||||
});
|
||||
expect(out.text.split("\n")[out.refs[0]?.line ?? -1]).toContain('@e1 textbox "Email"');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5,9 +5,11 @@ export type {
|
||||
CondSurface,
|
||||
LayerKind,
|
||||
Rect,
|
||||
RenderedRef,
|
||||
Viewport,
|
||||
VomNode,
|
||||
VomOptions,
|
||||
VomRef,
|
||||
VomResult,
|
||||
VomScene,
|
||||
} from "./types";
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user