mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
7
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 “haonan”andCursor
|
f7e08f461b | ||
|
“haonan”
|
61a0e0d218 | ||
|
“haonan”
|
67b7227abf | ||
|
“haonan”
|
02d1d69379 | ||
|
“haonan”
|
025751ef03 | ||
|
“haonan”
|
13a61b2688 | ||
|
“haonan”
|
1508839bf9 |
@@ -1,105 +0,0 @@
|
||||
name: Release dsh plugin
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- dsh-plugin-v*
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: release-dsh-plugin-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
PACKAGE_NAME: "@wxg-prc-cpg/browser-skill-dsh-plugin"
|
||||
PACKAGE_DIR: packages/dsh-plugin-browserskill
|
||||
|
||||
jobs:
|
||||
resolve:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
tag: ${{ steps.version.outputs.tag }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Resolve plugin version
|
||||
id: version
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
VERSION="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').version")"
|
||||
else
|
||||
VERSION="${GITHUB_REF_NAME#dsh-plugin-v}"
|
||||
fi
|
||||
VERSION="${VERSION#v}"
|
||||
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
echo "tag=dsh-plugin-v${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
guard:
|
||||
needs: resolve
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Verify tag matches package.json name and version
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
PKG_NAME="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').name")"
|
||||
PKG_VERSION="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').version")"
|
||||
RELEASE_VERSION="${{ needs.resolve.outputs.version }}"
|
||||
if [ "$PKG_NAME" != "$PACKAGE_NAME" ]; then
|
||||
echo "package.json name (${PKG_NAME}) does not match expected ${PACKAGE_NAME}"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$PKG_VERSION" != "$RELEASE_VERSION" ]; then
|
||||
echo "${PACKAGE_DIR}/package.json version (${PKG_VERSION}) does not match release version (${RELEASE_VERSION})"
|
||||
exit 1
|
||||
fi
|
||||
echo "Version guard passed: ${PKG_NAME}@${RELEASE_VERSION}"
|
||||
|
||||
publish:
|
||||
needs: [resolve, guard]
|
||||
runs-on: ubuntu-latest
|
||||
# Secret NPM_TOKEN is stored on the "npm-publish" GitHub Environment.
|
||||
environment: npm-publish
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: pnpm/action-setup@v6
|
||||
|
||||
- uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 22
|
||||
cache: pnpm
|
||||
registry-url: https://registry.npmjs.org
|
||||
scope: "@wxg-prc-cpg"
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Typecheck plugin
|
||||
run: pnpm --filter "${{ env.PACKAGE_NAME }}" typecheck
|
||||
|
||||
- name: Test plugin
|
||||
run: pnpm --filter "${{ env.PACKAGE_NAME }}" test
|
||||
|
||||
- name: Publish to npm
|
||||
working-directory: ${{ env.PACKAGE_DIR }}
|
||||
env:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${NODE_AUTH_TOKEN:-}" ]; then
|
||||
echo "::error::Environment secret NPM_TOKEN is not configured"
|
||||
exit 1
|
||||
fi
|
||||
pnpm publish --access public --no-git-checks
|
||||
echo "Published ${PACKAGE_NAME}@${{ needs.resolve.outputs.version }}"
|
||||
@@ -17,3 +17,7 @@ dist/
|
||||
# Environment
|
||||
.env.local
|
||||
.env.*.local
|
||||
.pnpm-store/
|
||||
|
||||
# Local recording output
|
||||
/trace/
|
||||
@@ -3,6 +3,3 @@
|
||||
**/.output/**
|
||||
**/target/**
|
||||
**/node_modules/**
|
||||
# Generated client CSS (tailwind compile output); built by
|
||||
# packages/dsh-plugin-browserskill/scripts/build-client-css.mjs.
|
||||
**/dsh-plugin-browserskill/src/client/bsk-ui.nomodule.css
|
||||
|
||||
+5
-9
@@ -34,17 +34,13 @@ where only `extension connected` fails is expected; go to Step 3.
|
||||
## 3. Open the extension install page
|
||||
|
||||
If `extension connected` is `FAIL` (`0 browsers connected`), the user likely
|
||||
has not installed or enabled the browser extension yet. Open the install page
|
||||
matching the user's browser —
|
||||
[Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||
for Chrome and other Chromium browsers,
|
||||
[Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg)
|
||||
for Microsoft Edge — then tell the user:
|
||||
has not installed or enabled the browser extension yet. Open the
|
||||
[Chrome Web Store install page](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||
in the user's browser, then tell the user:
|
||||
|
||||
> Install **BrowserSkill** from the
|
||||
> [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||
> (or [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg)
|
||||
> on Edge), open the popup, and wait until it turns green. Reply when done.
|
||||
> [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi),
|
||||
> open the popup, and wait until it turns green. Reply when done.
|
||||
|
||||
If opening the page fails, give the user the same link. Then run `bsk doctor`
|
||||
once more. All `ok`/`na` → tell the user it's ready.
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
</p>
|
||||
|
||||
**BrowserSkill** connects Cursor, Claude Code, Codex, OpenClaw, CodeBuddy,
|
||||
WorkBuddy, Pi, Hermes Agent, DeepSeek Harness, and other AI agents to your already logged-in
|
||||
WorkBuddy, Pi, Hermes Agent, and other shell-capable AI agents to your already logged-in
|
||||
browser.
|
||||
|
||||
Need the agent to touch a tab you already have open? It must borrow that tab
|
||||
@@ -67,8 +67,7 @@ Set up browser-skill on this machine by following https://raw.githubusercontent.
|
||||
|
||||
<br>
|
||||
|
||||
Install the CLI, then install the extension from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||
or [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg).
|
||||
Install the CLI, then install the extension from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi).
|
||||
|
||||
#### 1. Install the `bsk` CLI
|
||||
|
||||
@@ -92,14 +91,7 @@ bsk --version
|
||||
|
||||
#### 2. Install the browser extension
|
||||
|
||||
Install BrowserSkill from your browser's store:
|
||||
|
||||
| Browser | Store listing |
|
||||
| --- | --- |
|
||||
| Chrome | [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) |
|
||||
| Microsoft Edge | [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) |
|
||||
|
||||
On other Chromium-based browsers, install the Chrome Web Store build.
|
||||
Install BrowserSkill from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi).
|
||||
|
||||
#### 3. Install the skill
|
||||
|
||||
@@ -131,8 +123,7 @@ internal variants and install paths.
|
||||
|
||||
Other shell-capable agent harnesses are supported too. Copy
|
||||
[`skill/SKILL.md`](skill/SKILL.md) into your harness's skills directory as
|
||||
`browser-skill/SKILL.md` to install the skill manually. DeepSeek Harness uses a
|
||||
dedicated plugin instead — see [DeepSeek Harness plugin](#deepseek-harness-plugin).
|
||||
`browser-skill/SKILL.md` to install the skill manually.
|
||||
|
||||
</details>
|
||||
|
||||
@@ -142,26 +133,6 @@ Start a new Agent session and write a prompt that needs the browser, for example
|
||||
/browser-skill open example.com and summarize what is on the page.
|
||||
```
|
||||
|
||||
## DeepSeek Harness plugin
|
||||
|
||||
Using [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`dsh`)?
|
||||
BrowserSkill ships a first-class dsh plugin on npm as
|
||||
[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin).
|
||||
It injects native `browser_*` tools (no shelling out to `bsk`) and a live Web UI
|
||||
overlay of each Agent Window.
|
||||
|
||||
Add it to a dsh profile, then start that profile:
|
||||
|
||||
```sh
|
||||
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
|
||||
dsh --profile web
|
||||
```
|
||||
|
||||
The plugin carries its own copy of the skill, so `bsk install-skill` is not needed
|
||||
for dsh — but the `bsk` CLI and the browser extension are still prerequisites. See
|
||||
the [plugin README](packages/dsh-plugin-browserskill/README.md) for the tool list,
|
||||
configuration, and the observation overlay.
|
||||
|
||||
## How It Works
|
||||
|
||||
BrowserSkill is a local bridge between your agent harness and your browser.
|
||||
@@ -195,9 +166,7 @@ flowchart TB
|
||||
|
||||
The agent never talks to the browser directly. It asks the `bsk` CLI to perform a
|
||||
browser task; the local daemon routes that request to the extension; the
|
||||
extension runs it in an Agent Window. DeepSeek Harness takes the same path
|
||||
through the [plugin](#deepseek-harness-plugin): the agent calls injected
|
||||
`browser_*` tools, and the plugin invokes `bsk` on its behalf.
|
||||
extension runs it in an Agent Window.
|
||||
|
||||
## For Developers
|
||||
|
||||
@@ -207,7 +176,6 @@ The repository is a Cargo + pnpm workspace:
|
||||
- `crates/bsk-protocol` — shared wire types and JSON schemas
|
||||
- `apps/extension` — browser extension
|
||||
- `packages/ui` and `packages/i18n` — shared extension UI support
|
||||
- `packages/dsh-plugin-browserskill` — DeepSeek Harness plugin (`@wxg-prc-cpg/browser-skill-dsh-plugin`)
|
||||
|
||||
## License
|
||||
|
||||
|
||||
+7
-32
@@ -12,7 +12,7 @@
|
||||
<a href="README.md">English</a> · 中文
|
||||
</p>
|
||||
|
||||
**BrowserSkill** 把 Cursor、Claude Code、Codex、OpenClaw、CodeBuddy、WorkBuddy、Pi、Hermes Agent、DeepSeek Harness 等 AI Agent 连接到你已登录的浏览器。
|
||||
**BrowserSkill** 把 Cursor、Claude Code、Codex、OpenClaw、CodeBuddy、WorkBuddy、Pi、Hermes Agent 等支持 Shell 的 AI Agent 连接到你已登录的浏览器。
|
||||
|
||||
需要 Agent 操作你已打开的标签页？必须显式借用该标签，任务结束后归还，其余浏览器窗口不受影响。
|
||||
|
||||
@@ -54,8 +54,7 @@ BrowserSkill 由两个本地运行组件组成：`bsk` CLI/daemon 和浏览器
|
||||
|
||||
<br>
|
||||
|
||||
先安装 CLI，再从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||
或 [Edge 加载项商店](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) 安装浏览器扩展。
|
||||
先安装 CLI，再从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) 安装浏览器扩展。
|
||||
|
||||
#### 1. 安装 `bsk` CLI
|
||||
|
||||
@@ -65,11 +64,8 @@ BrowserSkill 由两个本地运行组件组成：`bsk` CLI/daemon 和浏览器
|
||||
curl -fsSL https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.sh | sh
|
||||
```
|
||||
|
||||
**Windows**（PowerShell，安装到 `~/.local/bin`）：
|
||||
|
||||
```powershell
|
||||
irm https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.ps1 | iex
|
||||
```
|
||||
**Windows**：从 [最新 CLI release](https://github.com/Tencent/BrowserSkill/releases/latest)
|
||||
下载 `bsk-v<version>-x86_64-pc-windows-msvc.zip`，解压后将 `bsk.exe` 加入 `PATH`。
|
||||
|
||||
验证二进制：
|
||||
|
||||
@@ -79,14 +75,7 @@ bsk --version
|
||||
|
||||
#### 2. 安装浏览器扩展
|
||||
|
||||
在对应浏览器的商店安装 BrowserSkill：
|
||||
|
||||
| 浏览器 | 商店页面 |
|
||||
| --- | --- |
|
||||
| Chrome | [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) |
|
||||
| Microsoft Edge | [Edge 加载项商店](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) |
|
||||
|
||||
其他基于 Chromium 的浏览器，安装 Chrome Web Store 版本即可。
|
||||
从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) 安装 BrowserSkill。
|
||||
|
||||
#### 3. 安装 skill
|
||||
|
||||
@@ -113,7 +102,7 @@ bsk install-skill
|
||||
|
||||
用 <kbd>Space</kbd> 选择需要安装的 Agent harness，然后按 <kbd>Enter</kbd> 安装 skill。运行 `bsk install-skill --list` 可查看 internal 变体及安装路径。
|
||||
|
||||
其他支持 Shell 的 Agent harness 也可使用 BrowserSkill，但需手动将 [`skill/SKILL.md`](skill/SKILL.md) 复制到对应 skills 目录下的 `browser-skill/SKILL.md`。DeepSeek Harness 走独立插件，见 [DeepSeek Harness 插件](#deepseek-harness-插件)。
|
||||
其他支持 Shell 的 Agent harness 也可使用 BrowserSkill，但需手动将 [`skill/SKILL.md`](skill/SKILL.md) 复制到对应 skills 目录下的 `browser-skill/SKILL.md`。
|
||||
|
||||
</details>
|
||||
|
||||
@@ -123,19 +112,6 @@ bsk install-skill
|
||||
/browser-skill open example.com and summarize what is on the page.
|
||||
```
|
||||
|
||||
## DeepSeek Harness 插件
|
||||
|
||||
在用 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness)（`dsh`）？BrowserSkill 提供了官方 dsh 插件，已发布到 npm：[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)。它会注入原生 `browser_*` 工具（无需再通过 Shell 调用 `bsk`），并在 Web UI 中实时观察每个 Agent Window。
|
||||
|
||||
把它装进某个 dsh profile，然后启动该 profile：
|
||||
|
||||
```sh
|
||||
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
|
||||
dsh --profile web
|
||||
```
|
||||
|
||||
插件自带 skill，所以在 dsh 下无需执行 `bsk install-skill`；但 `bsk` CLI 和浏览器扩展仍是前置条件。工具清单、配置项与观察浮层见[插件 README](packages/dsh-plugin-browserskill/README.md)。
|
||||
|
||||
## 工作原理
|
||||
|
||||
BrowserSkill 是 Agent 运行时与浏览器之间的本地桥接层。
|
||||
@@ -167,7 +143,7 @@ flowchart TB
|
||||
style UserWindows fill:#f8fafc,stroke:#cbd5e1,color:#334155
|
||||
```
|
||||
|
||||
Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务；本地 daemon 把请求路由到扩展；扩展在 Agent Window 中执行。DeepSeek Harness 走同一条链路，只是经由 [插件](#deepseek-harness-插件)：Agent 调用注入的 `browser_*` 工具，由插件代为执行 `bsk`。
|
||||
Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务；本地 daemon 把请求路由到扩展；扩展在 Agent Window 中执行。
|
||||
|
||||
## 面向开发者
|
||||
|
||||
@@ -177,7 +153,6 @@ Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务；
|
||||
- `crates/bsk-protocol` — 共享协议类型与 JSON Schema
|
||||
- `apps/extension` — 浏览器扩展
|
||||
- `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持
|
||||
- `packages/dsh-plugin-browserskill` — DeepSeek Harness 插件（`@wxg-prc-cpg/browser-skill-dsh-plugin`）
|
||||
|
||||
## 许可证
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "../chromium-cdp";
|
||||
import { type CdpDebuggerApi, ChromiumCdp } from "../chromium-cdp";
|
||||
|
||||
function fakeChromeEvent<TArgs extends unknown[]>() {
|
||||
const listeners = new Set<(...args: TArgs) => void>();
|
||||
@@ -14,7 +14,7 @@ function fakeChromeEvent<TArgs extends unknown[]>() {
|
||||
}
|
||||
|
||||
function fakeApi() {
|
||||
const onEvent = fakeChromeEvent<[CdpDebuggee, string, unknown]>();
|
||||
const onEvent = fakeChromeEvent<[chrome.debugger.Debuggee, string, unknown]>();
|
||||
const onDetach = fakeChromeEvent<[chrome.debugger.Debuggee, string]>();
|
||||
const api: CdpDebuggerApi = {
|
||||
attach: vi.fn(async () => {}),
|
||||
@@ -29,135 +29,6 @@ function fakeApi() {
|
||||
}
|
||||
|
||||
describe("ChromiumCdp", () => {
|
||||
it("discovers multiple iframe targets and recursively routes nested OOPIF commands", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
async (
|
||||
target: chrome.debugger.Debuggee & { sessionId?: string },
|
||||
method: string,
|
||||
params?: { frameId?: string },
|
||||
) => {
|
||||
if (method === "DOM.getFrameOwner") {
|
||||
return { backendNodeId: params?.frameId === "nested" ? 300 : 200 };
|
||||
}
|
||||
if (method !== "Page.getFrameTree") return {};
|
||||
if (target.sessionId === "right-session") {
|
||||
return {
|
||||
frameTree: {
|
||||
frame: { id: "right", url: "https://right.test" },
|
||||
childFrames: [
|
||||
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
if (target.sessionId === "nested-session") {
|
||||
return { frameTree: { frame: { id: "nested", url: "https://nested.test" } } };
|
||||
}
|
||||
return {
|
||||
frameTree: {
|
||||
frame: { id: "main", url: "https://app.test" },
|
||||
childFrames: [
|
||||
{ frame: { id: "left", parentId: "main", url: "https://left.test" } },
|
||||
{
|
||||
frame: { id: "right", parentId: "main", url: "https://right.test" },
|
||||
childFrames: [
|
||||
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
},
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
await cdp.ensureAttached(4);
|
||||
onEvent.fire({ tabId: 4 }, "Target.attachedToTarget", { sessionId: "right-session" });
|
||||
onEvent.fire({ tabId: 4 } as chrome.debugger.Debuggee, "Target.attachedToTarget", {
|
||||
sessionId: "nested-session",
|
||||
});
|
||||
|
||||
const graph = await cdp.getFrameGraph(4);
|
||||
|
||||
expect(graph.frames).toHaveLength(4);
|
||||
expect(graph.frames.find((frame) => frame.frameId === "left")?.target).toEqual({ tabId: 4 });
|
||||
expect(graph.frames.find((frame) => frame.frameId === "right")?.target.sessionId).toBe(
|
||||
"right-session",
|
||||
);
|
||||
expect(graph.frames.find((frame) => frame.frameId === "nested")?.target.sessionId).toBe(
|
||||
"nested-session",
|
||||
);
|
||||
expect(graph.frames.find((frame) => frame.frameId === "right")?.ownerBackendNodeId).toBe(200);
|
||||
expect(graph.frames.find((frame) => frame.frameId === "nested")?.ownerBackendNodeId).toBe(300);
|
||||
expect(api.sendCommand).toHaveBeenCalledWith(
|
||||
{ tabId: 4, sessionId: "right-session" },
|
||||
"DOM.getFrameOwner",
|
||||
{ frameId: "nested" },
|
||||
);
|
||||
expect(api.sendCommand).toHaveBeenCalledWith(
|
||||
{ tabId: 4, sessionId: "nested-session" },
|
||||
"Target.setAutoAttach",
|
||||
expect.objectContaining({ flatten: true }),
|
||||
);
|
||||
});
|
||||
|
||||
it("waits for recursively attached iframe targets without a depth limit", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
const depth = 6;
|
||||
const scheduled = new Set<string>();
|
||||
const nestedTree = (index: number): Record<string, unknown> => ({
|
||||
frame: {
|
||||
id: index === 0 ? "main" : `frame-${index}`,
|
||||
...(index > 0 ? { parentId: index === 1 ? "main" : `frame-${index - 1}` } : {}),
|
||||
},
|
||||
...(index < depth ? { childFrames: [nestedTree(index + 1)] } : {}),
|
||||
});
|
||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
async (target: CdpDebuggee, method: string, params?: { frameId?: string }) => {
|
||||
if (method === "Target.setAutoAttach") {
|
||||
const parentIndex = target.sessionId
|
||||
? Number(target.sessionId.replace("session-", ""))
|
||||
: 0;
|
||||
const nextIndex = parentIndex + 1;
|
||||
const key = `${target.sessionId ?? "root"}:${nextIndex}`;
|
||||
if (nextIndex <= depth && !scheduled.has(key)) {
|
||||
scheduled.add(key);
|
||||
setTimeout(() => {
|
||||
onEvent.fire(
|
||||
{ tabId: 4, ...(target.sessionId ? { sessionId: target.sessionId } : {}) },
|
||||
"Target.attachedToTarget",
|
||||
{
|
||||
sessionId: `session-${nextIndex}`,
|
||||
targetInfo: { type: "iframe" },
|
||||
},
|
||||
);
|
||||
}, 5);
|
||||
}
|
||||
return {};
|
||||
}
|
||||
if (method === "Page.getFrameTree") {
|
||||
if (!target.sessionId) return { frameTree: nestedTree(0) };
|
||||
const index = Number(target.sessionId.replace("session-", ""));
|
||||
return { frameTree: nestedTree(index) };
|
||||
}
|
||||
if (method === "DOM.getFrameOwner") {
|
||||
return { backendNodeId: Number(params?.frameId?.replace("frame-", "")) + 100 };
|
||||
}
|
||||
return {};
|
||||
},
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
|
||||
const graph = await cdp.getFrameGraph(4);
|
||||
|
||||
expect(graph.frames).toHaveLength(depth + 1);
|
||||
for (let index = 1; index <= depth; index += 1) {
|
||||
expect(
|
||||
graph.frames.find((frame) => frame.frameId === `frame-${index}`)?.target.sessionId,
|
||||
).toBe(`session-${index}`);
|
||||
}
|
||||
});
|
||||
|
||||
it("coalesces concurrent attach calls for the same tab", async () => {
|
||||
const { api } = fakeApi();
|
||||
let releaseAttach!: () => void;
|
||||
|
||||
@@ -1,136 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildFrameGraph, type CdpFrameTreeNode, type CdpFrameTreeSource } from "../frame-graph";
|
||||
|
||||
describe("buildFrameGraph", () => {
|
||||
it("keeps sibling frames distinct and routes nested OOPIFs to their child sessions", () => {
|
||||
const graph = buildFrameGraph([
|
||||
{
|
||||
target: { tabId: 4 },
|
||||
tree: {
|
||||
frame: { id: "main", url: "https://app.test" },
|
||||
childFrames: [
|
||||
{ frame: { id: "left", parentId: "main", url: "https://left.test" } },
|
||||
{
|
||||
frame: { id: "right", parentId: "main", url: "https://right.test" },
|
||||
childFrames: [
|
||||
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
target: { tabId: 4, sessionId: "right-session" },
|
||||
tree: {
|
||||
frame: { id: "right", url: "https://right.test" },
|
||||
childFrames: [{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } }],
|
||||
},
|
||||
},
|
||||
{
|
||||
target: { tabId: 4, sessionId: "nested-session" },
|
||||
tree: { frame: { id: "nested", url: "https://nested.test" } },
|
||||
},
|
||||
]);
|
||||
|
||||
expect(graph?.frames).toHaveLength(4);
|
||||
expect(graph?.frames.find((frame) => frame.frameId === "left")?.target).toEqual({ tabId: 4 });
|
||||
expect(graph?.frames.find((frame) => frame.frameId === "right")).toMatchObject({
|
||||
parentFrameId: "main",
|
||||
target: { tabId: 4, sessionId: "right-session" },
|
||||
});
|
||||
expect(graph?.frames.find((frame) => frame.frameId === "nested")).toMatchObject({
|
||||
parentFrameId: "right",
|
||||
target: { tabId: 4, sessionId: "nested-session" },
|
||||
});
|
||||
});
|
||||
|
||||
it("propagates each target boundary to its same-process descendants", () => {
|
||||
const sources: CdpFrameTreeSource[] = [
|
||||
{
|
||||
target: { tabId: 4 },
|
||||
tree: {
|
||||
frame: { id: "main" },
|
||||
childFrames: [
|
||||
{
|
||||
frame: { id: "oopif-a", parentId: "main" },
|
||||
childFrames: [
|
||||
{
|
||||
frame: { id: "same-process-b", parentId: "oopif-a" },
|
||||
childFrames: [
|
||||
{
|
||||
frame: { id: "oopif-c", parentId: "same-process-b" },
|
||||
childFrames: [{ frame: { id: "same-process-d", parentId: "oopif-c" } }],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
target: { tabId: 4, sessionId: "session-a" },
|
||||
tree: {
|
||||
frame: { id: "oopif-a" },
|
||||
childFrames: [
|
||||
{
|
||||
frame: { id: "same-process-b", parentId: "oopif-a" },
|
||||
childFrames: [{ frame: { id: "oopif-c", parentId: "same-process-b" } }],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
target: { tabId: 4, sessionId: "session-c" },
|
||||
tree: {
|
||||
frame: { id: "oopif-c" },
|
||||
childFrames: [{ frame: { id: "same-process-d", parentId: "oopif-c" } }],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const graph = buildFrameGraph(sources);
|
||||
|
||||
expect(graph?.frames.map((frame) => [frame.frameId, frame.target.sessionId])).toEqual([
|
||||
["main", undefined],
|
||||
["oopif-a", "session-a"],
|
||||
["same-process-b", "session-a"],
|
||||
["oopif-c", "session-c"],
|
||||
["same-process-d", "session-c"],
|
||||
]);
|
||||
expect(graph?.frames.find((frame) => frame.frameId === "oopif-a")?.parentFrameId).toBe("main");
|
||||
expect(graph?.frames.find((frame) => frame.frameId === "oopif-c")?.parentFrameId).toBe(
|
||||
"same-process-b",
|
||||
);
|
||||
|
||||
const reordered = buildFrameGraph([sources[2], sources[1], sources[0]]);
|
||||
expect(reordered).toEqual(graph);
|
||||
});
|
||||
|
||||
it("walks deeply nested frame trees without consuming the JavaScript call stack", () => {
|
||||
const depth = 5_000;
|
||||
const root: CdpFrameTreeNode = { frame: { id: "frame-0" } };
|
||||
let parent = root;
|
||||
for (let index = 1; index <= depth; index += 1) {
|
||||
const child: CdpFrameTreeNode = {
|
||||
frame: { id: `frame-${index}`, parentId: `frame-${index - 1}` },
|
||||
};
|
||||
parent.childFrames = [child];
|
||||
parent = child;
|
||||
}
|
||||
|
||||
const graph = buildFrameGraph([{ target: { tabId: 4 }, tree: root }]);
|
||||
|
||||
expect(graph?.frames).toHaveLength(depth + 1);
|
||||
expect(graph?.frames.at(-1)?.frameId).toBe(`frame-${depth}`);
|
||||
});
|
||||
|
||||
it("does not loop when an in-memory frame tree contains an object cycle", () => {
|
||||
const root: CdpFrameTreeNode = { frame: { id: "main" } };
|
||||
root.childFrames = [root];
|
||||
|
||||
expect(buildFrameGraph([{ target: { tabId: 4 }, tree: root }])?.frames).toEqual([
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -31,15 +31,6 @@ import type {
|
||||
NetworkEntryKind,
|
||||
NetworkResult,
|
||||
} from "@/transport/types";
|
||||
import {
|
||||
buildFrameGraph,
|
||||
type CdpFrameGraph,
|
||||
type CdpFrameTreeNode,
|
||||
type CdpFrameTreeSource,
|
||||
type CdpTarget,
|
||||
} from "./frame-graph";
|
||||
|
||||
export type CdpDebuggee = chrome.debugger.Debuggee & { sessionId?: string };
|
||||
|
||||
/**
|
||||
* Minimal slice of `chrome.debugger` the rest of the extension
|
||||
@@ -47,15 +38,21 @@ export type CdpDebuggee = chrome.debugger.Debuggee & { sessionId?: string };
|
||||
* fake without monkey-patching the real `chrome` global.
|
||||
*/
|
||||
export interface CdpDebuggerApi {
|
||||
attach(target: CdpDebuggee, requiredVersion: string): Promise<void>;
|
||||
detach(target: CdpDebuggee): Promise<void>;
|
||||
sendCommand(target: CdpDebuggee, method: string, commandParams?: object): Promise<unknown>;
|
||||
attach(target: chrome.debugger.Debuggee, requiredVersion: string): Promise<void>;
|
||||
detach(target: chrome.debugger.Debuggee): Promise<void>;
|
||||
sendCommand(
|
||||
target: chrome.debugger.Debuggee,
|
||||
method: string,
|
||||
commandParams?: object,
|
||||
): Promise<unknown>;
|
||||
/**
|
||||
* Fires for every CDP event (`Page.lifecycleEvent`, `DOM.documentUpdated`,
|
||||
* …). The first callback argument is the source debuggee; the second
|
||||
* is the CDP method name; the third is the payload.
|
||||
*/
|
||||
onEvent: chrome.events.Event<(source: CdpDebuggee, method: string, params: unknown) => void>;
|
||||
onEvent: chrome.events.Event<
|
||||
(source: chrome.debugger.Debuggee, method: string, params: unknown) => void
|
||||
>;
|
||||
/**
|
||||
* Fires when Chrome unilaterally detaches us — most commonly because
|
||||
* the tab navigated to a chrome:// URL or the user clicked
|
||||
@@ -113,8 +110,6 @@ const MAX_CONSOLE_STACK_FRAMES = 20;
|
||||
const MAX_NETWORK_BUFFER = 200;
|
||||
const MAX_NETWORK_FIELD_LENGTH = 4096;
|
||||
const MAX_NETWORK_REQUEST_META = 1024;
|
||||
const FRAME_DISCOVERY_TIMEOUT_MS = 1000;
|
||||
const FRAME_DISCOVERY_QUIET_MS = 20;
|
||||
|
||||
interface ParsedDialogOpening {
|
||||
type: JavaScriptDialogType;
|
||||
@@ -136,27 +131,6 @@ interface NetworkRequestMeta {
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
interface FrameDiscoveryState {
|
||||
sessions: Set<string>;
|
||||
pending: Set<Promise<void>>;
|
||||
generation: number;
|
||||
}
|
||||
|
||||
async function settleBeforeDeadline(promises: Promise<void>[], deadline: number): Promise<boolean> {
|
||||
if (promises.length === 0) return true;
|
||||
const remaining = deadline - Date.now();
|
||||
if (remaining <= 0) return false;
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const timedOut = await Promise.race([
|
||||
Promise.allSettled(promises).then(() => false),
|
||||
new Promise<boolean>((resolve) => {
|
||||
timer = setTimeout(() => resolve(true), remaining);
|
||||
}),
|
||||
]);
|
||||
if (timer) clearTimeout(timer);
|
||||
return !timedOut;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrapper around `chrome.debugger` that owns the "attach once per
|
||||
* tabId" cache and exposes typed `send<T>()`.
|
||||
@@ -175,12 +149,10 @@ export class ChromiumCdp {
|
||||
private readonly networkSequences = new Map<number, number>();
|
||||
private readonly networkDomainsEnabledTabs = new Set<number>();
|
||||
private readonly networkRequestMeta = new Map<number, Map<string, NetworkRequestMeta>>();
|
||||
private readonly frameDiscovery = new Map<number, FrameDiscoveryState>();
|
||||
private detachSubscription: { dispose(): void } | null = null;
|
||||
private dialogSubscription: { dispose(): void } | null = null;
|
||||
private consoleSubscription: { dispose(): void } | null = null;
|
||||
private networkSubscription: { dispose(): void } | null = null;
|
||||
private frameTargetSubscription: { dispose(): void } | null = null;
|
||||
|
||||
constructor(api: CdpDebuggerApi = chromeDebuggerApi) {
|
||||
this.api = api;
|
||||
@@ -188,7 +160,6 @@ export class ChromiumCdp {
|
||||
this.bindDialogHandler();
|
||||
this.bindConsoleHandler();
|
||||
this.bindNetworkHandler();
|
||||
this.bindFrameTargetHandler();
|
||||
}
|
||||
|
||||
/** Attach to `tabId` if we haven't already in this driver. */
|
||||
@@ -206,9 +177,6 @@ export class ChromiumCdp {
|
||||
await this.enableConsoleDomains(tabId);
|
||||
await this.enableNetworkDomainBestEffort(tabId);
|
||||
this.attachedTabs.add(tabId);
|
||||
await this.enableFrameDiscovery({ tabId }).catch((err) => {
|
||||
console.debug("[bsk cdp] frame discovery unavailable", { tabId, err });
|
||||
});
|
||||
} catch (err) {
|
||||
// A CDP domain enable failed after the raw attach succeeded
|
||||
// (e.g. `Page.enable` rejects because the tab just navigated to
|
||||
@@ -254,73 +222,6 @@ export class ChromiumCdp {
|
||||
}
|
||||
}
|
||||
|
||||
async sendToTarget<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T> {
|
||||
if (!this.attachedTabs.has(target.tabId)) {
|
||||
await this.ensureAttached(target.tabId);
|
||||
}
|
||||
try {
|
||||
return (await this.api.sendCommand(target, method, params ?? {})) as T;
|
||||
} catch (err) {
|
||||
throw normalizeError(err);
|
||||
}
|
||||
}
|
||||
|
||||
async getFrameGraph(tabId: number): Promise<CdpFrameGraph> {
|
||||
await this.ensureAttached(tabId);
|
||||
await this.enableFrameDiscovery({ tabId }).catch(() => {});
|
||||
await this.drainFrameAttachTasks(tabId);
|
||||
|
||||
const sources: CdpFrameTreeSource[] = [];
|
||||
const root = await this.sendToTarget<{ frameTree?: CdpFrameTreeNode }>(
|
||||
{ tabId },
|
||||
"Page.getFrameTree",
|
||||
{},
|
||||
);
|
||||
if (root.frameTree) sources.push({ target: { tabId }, tree: root.frameTree });
|
||||
|
||||
const sessions = [...(this.frameDiscovery.get(tabId)?.sessions ?? [])];
|
||||
const childTrees = await Promise.all(
|
||||
sessions.map(async (sessionId): Promise<CdpFrameTreeSource | null> => {
|
||||
const target = { tabId, sessionId };
|
||||
try {
|
||||
const reply = await this.sendToTarget<{ frameTree?: CdpFrameTreeNode }>(
|
||||
target,
|
||||
"Page.getFrameTree",
|
||||
{},
|
||||
);
|
||||
return reply.frameTree ? { target, tree: reply.frameTree } : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}),
|
||||
);
|
||||
for (const source of childTrees) {
|
||||
if (source) sources.push(source);
|
||||
}
|
||||
|
||||
const graph = buildFrameGraph(sources);
|
||||
if (!graph) throw new Error("Page.getFrameTree returned no root frame");
|
||||
const frameById = new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
||||
await Promise.all(
|
||||
graph.frames.map(async (frame) => {
|
||||
if (!frame.parentFrameId) return;
|
||||
const parent = frameById.get(frame.parentFrameId);
|
||||
if (!parent) return;
|
||||
try {
|
||||
const owner = await this.sendToTarget<{ backendNodeId?: number }>(
|
||||
parent.target,
|
||||
"DOM.getFrameOwner",
|
||||
{ frameId: frame.frameId },
|
||||
);
|
||||
if (owner.backendNodeId !== undefined) frame.ownerBackendNodeId = owner.backendNodeId;
|
||||
} catch {
|
||||
// The frame may have navigated between tree capture and owner lookup.
|
||||
}
|
||||
}),
|
||||
);
|
||||
return graph;
|
||||
}
|
||||
|
||||
/** Return a cursor marking the current dialog sequence for `tabId`. */
|
||||
dialogCursor(tabId: number): DialogCursor {
|
||||
return this.dialogSequences.get(tabId) ?? 0;
|
||||
@@ -431,7 +332,6 @@ export class ChromiumCdp {
|
||||
this.clearDialogState(tabId);
|
||||
this.clearConsoleState(tabId);
|
||||
this.clearNetworkState(tabId);
|
||||
this.clearFrameState(tabId);
|
||||
try {
|
||||
await this.api.detach({ tabId });
|
||||
} catch (err) {
|
||||
@@ -454,7 +354,7 @@ export class ChromiumCdp {
|
||||
}
|
||||
|
||||
/** Subscribe to all CDP events. Returned disposable removes the listener. */
|
||||
onEvent(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
|
||||
onEvent(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): {
|
||||
dispose(): void;
|
||||
} {
|
||||
this.api.onEvent.addListener(handler);
|
||||
@@ -478,7 +378,6 @@ export class ChromiumCdp {
|
||||
this.networkSequences.clear();
|
||||
this.networkDomainsEnabledTabs.clear();
|
||||
this.networkRequestMeta.clear();
|
||||
this.frameDiscovery.clear();
|
||||
await Promise.all(
|
||||
tabs.map(async (tabId) => {
|
||||
try {
|
||||
@@ -529,104 +428,9 @@ export class ChromiumCdp {
|
||||
}
|
||||
}
|
||||
|
||||
private async enableFrameDiscovery(target: CdpTarget): Promise<void> {
|
||||
await this.api.sendCommand(target, "Target.setAutoAttach", {
|
||||
autoAttach: true,
|
||||
waitForDebuggerOnStart: false,
|
||||
flatten: true,
|
||||
filter: [{ type: "iframe", exclude: false }],
|
||||
});
|
||||
}
|
||||
|
||||
private bindFrameTargetHandler(): void {
|
||||
if (this.frameTargetSubscription) return;
|
||||
const listener = (source: CdpDebuggee, method: string, params: unknown) => {
|
||||
const tabId = source.tabId;
|
||||
if (typeof tabId !== "number") return;
|
||||
const raw = (params ?? {}) as Record<string, unknown>;
|
||||
const sessionId = typeof raw.sessionId === "string" ? raw.sessionId : undefined;
|
||||
if (!sessionId) return;
|
||||
|
||||
if (method === "Target.detachedFromTarget") {
|
||||
const state = this.frameDiscovery.get(tabId);
|
||||
if (state?.sessions.delete(sessionId)) state.generation += 1;
|
||||
return;
|
||||
}
|
||||
if (method !== "Target.attachedToTarget") return;
|
||||
const targetInfo = raw.targetInfo as { type?: string } | undefined;
|
||||
if (targetInfo?.type && targetInfo.type !== "iframe") return;
|
||||
|
||||
const state = this.frameDiscoveryState(tabId);
|
||||
if (state.sessions.has(sessionId)) return;
|
||||
state.sessions.add(sessionId);
|
||||
state.generation += 1;
|
||||
|
||||
const task = this.initializeFrameTarget({ tabId, sessionId });
|
||||
state.pending.add(task);
|
||||
void task.finally(() => {
|
||||
state.pending.delete(task);
|
||||
});
|
||||
};
|
||||
this.api.onEvent.addListener(listener);
|
||||
this.frameTargetSubscription = {
|
||||
dispose: () => this.api.onEvent.removeListener(listener),
|
||||
};
|
||||
}
|
||||
|
||||
private async initializeFrameTarget(target: CdpTarget): Promise<void> {
|
||||
try {
|
||||
await this.enableFrameDiscovery(target);
|
||||
} catch (err) {
|
||||
const state = this.frameDiscovery.get(target.tabId);
|
||||
if (state?.sessions.delete(target.sessionId as string)) state.generation += 1;
|
||||
console.debug("[bsk cdp] child frame target initialization failed", { target, err });
|
||||
}
|
||||
}
|
||||
|
||||
private async drainFrameAttachTasks(tabId: number): Promise<void> {
|
||||
const deadline = Date.now() + FRAME_DISCOVERY_TIMEOUT_MS;
|
||||
while (Date.now() < deadline) {
|
||||
const state = this.frameDiscovery.get(tabId);
|
||||
const generation = state?.generation ?? 0;
|
||||
if (state?.pending.size && !(await settleBeforeDeadline([...state.pending], deadline))) break;
|
||||
|
||||
// Target.setAutoAttach may enqueue the next attachedToTarget event after
|
||||
// its command promise settles. Wait for a short quiet window, then finish
|
||||
// only if neither the state object nor its generation changed.
|
||||
const quietTime = Math.min(FRAME_DISCOVERY_QUIET_MS, deadline - Date.now());
|
||||
if (quietTime <= 0) break;
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, quietTime));
|
||||
const current = this.frameDiscovery.get(tabId);
|
||||
if (
|
||||
current === state &&
|
||||
(current?.generation ?? 0) === generation &&
|
||||
!current?.pending.size
|
||||
) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
console.debug("[bsk cdp] frame discovery did not reach quiescence before timeout", { tabId });
|
||||
}
|
||||
|
||||
private frameDiscoveryState(tabId: number): FrameDiscoveryState {
|
||||
const existing = this.frameDiscovery.get(tabId);
|
||||
if (existing) return existing;
|
||||
const created: FrameDiscoveryState = {
|
||||
sessions: new Set(),
|
||||
pending: new Set(),
|
||||
generation: 0,
|
||||
};
|
||||
this.frameDiscovery.set(tabId, created);
|
||||
return created;
|
||||
}
|
||||
|
||||
private clearFrameState(tabId: number): void {
|
||||
this.frameDiscovery.delete(tabId);
|
||||
}
|
||||
|
||||
private bindDialogHandler(): void {
|
||||
if (this.dialogSubscription) return;
|
||||
const listener = (source: CdpDebuggee, method: string, params: unknown) => {
|
||||
const listener = (source: chrome.debugger.Debuggee, method: string, params: unknown) => {
|
||||
if (method !== "Page.javascriptDialogOpening") return;
|
||||
const tabId = source.tabId;
|
||||
if (typeof tabId !== "number") return;
|
||||
@@ -810,7 +614,6 @@ export class ChromiumCdp {
|
||||
this.clearDialogState(source.tabId);
|
||||
this.clearConsoleState(source.tabId);
|
||||
this.clearNetworkState(source.tabId);
|
||||
this.clearFrameState(source.tabId);
|
||||
}
|
||||
};
|
||||
this.api.onDetach.addListener(listener);
|
||||
@@ -829,8 +632,6 @@ export class ChromiumCdp {
|
||||
this.consoleSubscription = null;
|
||||
this.networkSubscription?.dispose();
|
||||
this.networkSubscription = null;
|
||||
this.frameTargetSubscription?.dispose();
|
||||
this.frameTargetSubscription = null;
|
||||
}
|
||||
|
||||
/** Detach tabs only when no other live session has claimed them. */
|
||||
|
||||
@@ -1,107 +0,0 @@
|
||||
export interface CdpTarget {
|
||||
tabId: number;
|
||||
sessionId?: string;
|
||||
}
|
||||
|
||||
export interface CdpFrame {
|
||||
frameId: string;
|
||||
parentFrameId?: string;
|
||||
ownerBackendNodeId?: number;
|
||||
url?: string;
|
||||
target: CdpTarget;
|
||||
}
|
||||
|
||||
export interface CdpFrameGraph {
|
||||
rootFrameId: string;
|
||||
frames: CdpFrame[];
|
||||
}
|
||||
|
||||
export interface CdpFrameTreeNode {
|
||||
frame: {
|
||||
id: string;
|
||||
parentId?: string;
|
||||
url?: string;
|
||||
};
|
||||
childFrames?: CdpFrameTreeNode[];
|
||||
}
|
||||
|
||||
export interface CdpFrameTreeSource {
|
||||
target: CdpTarget;
|
||||
tree: CdpFrameTreeNode;
|
||||
}
|
||||
|
||||
export function cdpTargetKey(target: CdpTarget): string {
|
||||
return `${target.tabId}:${target.sessionId ?? "root"}`;
|
||||
}
|
||||
|
||||
function mergeFrame(
|
||||
frames: Map<string, CdpFrame>,
|
||||
order: string[],
|
||||
node: CdpFrameTreeNode,
|
||||
target: CdpTarget,
|
||||
): void {
|
||||
const existing = frames.get(node.frame.id);
|
||||
if (!existing) order.push(node.frame.id);
|
||||
const parentFrameId = node.frame.parentId || existing?.parentFrameId;
|
||||
const url = node.frame.url || existing?.url;
|
||||
frames.set(node.frame.id, {
|
||||
frameId: node.frame.id,
|
||||
...(parentFrameId ? { parentFrameId } : {}),
|
||||
...(url ? { url } : {}),
|
||||
target,
|
||||
});
|
||||
}
|
||||
|
||||
function walkFrameTree(
|
||||
source: CdpFrameTreeSource,
|
||||
targetByRootFrameId: ReadonlyMap<string, CdpTarget>,
|
||||
frames: Map<string, CdpFrame>,
|
||||
order: string[],
|
||||
): void {
|
||||
const stack: Array<{ node: CdpFrameTreeNode; inheritedTarget: CdpTarget }> = [
|
||||
{ node: source.tree, inheritedTarget: source.target },
|
||||
];
|
||||
const expanded = new WeakSet<CdpFrameTreeNode>();
|
||||
|
||||
while (stack.length > 0) {
|
||||
const current = stack.pop();
|
||||
if (!current || expanded.has(current.node)) continue;
|
||||
expanded.add(current.node);
|
||||
|
||||
const target = targetByRootFrameId.get(current.node.frame.id) ?? current.inheritedTarget;
|
||||
mergeFrame(frames, order, current.node, target);
|
||||
|
||||
const children = current.node.childFrames ?? [];
|
||||
for (let index = children.length - 1; index >= 0; index -= 1) {
|
||||
stack.push({ node: children[index], inheritedTarget: target });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function buildFrameGraph(sources: CdpFrameTreeSource[]): CdpFrameGraph | null {
|
||||
const rootSource = sources.find((source) => source.target.sessionId === undefined) ?? sources[0];
|
||||
if (!rootSource) return null;
|
||||
|
||||
const frames = new Map<string, CdpFrame>();
|
||||
const order: string[] = [];
|
||||
const targetByRootFrameId = new Map(
|
||||
sources.map((source) => [source.tree.frame.id, source.target] as const),
|
||||
);
|
||||
// The top-level source is authoritative if malformed input reports the same
|
||||
// frame as both a root target and a child target.
|
||||
targetByRootFrameId.set(rootSource.tree.frame.id, rootSource.target);
|
||||
|
||||
walkFrameTree(rootSource, targetByRootFrameId, frames, order);
|
||||
for (const source of sources) {
|
||||
if (source === rootSource) continue;
|
||||
walkFrameTree(source, targetByRootFrameId, frames, order);
|
||||
}
|
||||
|
||||
return {
|
||||
rootFrameId: rootSource.tree.frame.id,
|
||||
frames: order.flatMap((frameId) => {
|
||||
const frame = frames.get(frameId);
|
||||
return frame ? [frame] : [];
|
||||
}),
|
||||
};
|
||||
}
|
||||
@@ -8,7 +8,6 @@ import {
|
||||
useRef,
|
||||
useState,
|
||||
} from "react";
|
||||
import type { HelpHighlightRect } from "@/lib/help-bridge";
|
||||
import logoUrl from "../../assets/logo.png";
|
||||
|
||||
export interface HelpRequestData {
|
||||
@@ -20,10 +19,6 @@ export interface HelpRequestData {
|
||||
displayMode?: "full" | "compact";
|
||||
/** CSS selectors to scroll to + flash-highlight. */
|
||||
selectors: string[];
|
||||
/** Pre-resolved top-viewport rectangles, used for cross-frame targets. */
|
||||
rects?: HelpHighlightRect[];
|
||||
/** Re-resolve cross-frame rectangles after scroll, resize, or layout changes. */
|
||||
refreshRects?: () => Promise<HelpHighlightRect[] | undefined>;
|
||||
onContinue: (note: string) => void;
|
||||
onCancel: () => void;
|
||||
}
|
||||
@@ -54,7 +49,6 @@ const PANEL_WIDTH = 420;
|
||||
const FALLBACK_PANEL_H = 180;
|
||||
const FALLBACK_PANEL_H_COLLAPSED = 44;
|
||||
const EMPTY_SELECTORS: string[] = [];
|
||||
const EMPTY_RECTS: HelpHighlightRect[] = [];
|
||||
|
||||
function clamp(value: number, min: number, max: number): number {
|
||||
return Math.max(min, Math.min(max, value));
|
||||
@@ -185,8 +179,6 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
const { t } = useTranslation("extension");
|
||||
const isCompact = request?.displayMode === "compact";
|
||||
const effectiveSelectors = isCompact ? EMPTY_SELECTORS : (request?.selectors ?? EMPTY_SELECTORS);
|
||||
const [liveRects, setLiveRects] = useState<HelpHighlightRect[] | null>(null);
|
||||
const effectiveRects = isCompact ? EMPTY_RECTS : (liveRects ?? request?.rects ?? EMPTY_RECTS);
|
||||
const [note, setNote] = useState("");
|
||||
const [collapsed, setCollapsed] = useState(false);
|
||||
const [boxes, setBoxes] = useState<Box[]>([]);
|
||||
@@ -209,7 +201,6 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
// Reset the note and collapse state whenever a new request appears.
|
||||
useEffect(() => {
|
||||
setNote("");
|
||||
setLiveRects(null);
|
||||
setCollapsed(false);
|
||||
placementRef.current = null;
|
||||
setDragPos(null);
|
||||
@@ -218,32 +209,6 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
dragStartRef.current = null;
|
||||
}, [request?.id]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!request || isCompact || !request.refreshRects) return;
|
||||
let disposed = false;
|
||||
let resolving = false;
|
||||
const refresh = async () => {
|
||||
if (resolving) return;
|
||||
resolving = true;
|
||||
try {
|
||||
const rects = await request.refreshRects?.();
|
||||
if (!disposed && rects) setLiveRects(rects);
|
||||
} finally {
|
||||
resolving = false;
|
||||
}
|
||||
};
|
||||
const onViewportChange = () => void refresh();
|
||||
window.addEventListener("scroll", onViewportChange, true);
|
||||
window.addEventListener("resize", onViewportChange);
|
||||
const interval = window.setInterval(onViewportChange, 500);
|
||||
return () => {
|
||||
disposed = true;
|
||||
window.removeEventListener("scroll", onViewportChange, true);
|
||||
window.removeEventListener("resize", onViewportChange);
|
||||
window.clearInterval(interval);
|
||||
};
|
||||
}, [request, isCompact]);
|
||||
|
||||
// Scroll the first matched target into view once per distinct request id.
|
||||
useLayoutEffect(() => {
|
||||
if (!request) return;
|
||||
@@ -266,7 +231,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
return;
|
||||
}
|
||||
const update = () => {
|
||||
const nextBoxes = [...measure(effectiveSelectors), ...effectiveRects];
|
||||
const nextBoxes = measure(effectiveSelectors);
|
||||
setBoxes((prev) => (boxesEqual(prev, nextBoxes) ? prev : nextBoxes));
|
||||
};
|
||||
update();
|
||||
@@ -278,7 +243,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
window.removeEventListener("resize", update);
|
||||
window.clearInterval(interval);
|
||||
};
|
||||
}, [request, effectiveSelectors, effectiveRects]);
|
||||
}, [request, effectiveSelectors]);
|
||||
|
||||
const onHeaderPointerDown = useCallback(
|
||||
(e: ReactPointerEvent<HTMLDivElement>) => {
|
||||
@@ -340,7 +305,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
return;
|
||||
}
|
||||
if (userMovedRef.current) return;
|
||||
const measured = boxes.length > 0 ? boxes : [...measure(effectiveSelectors), ...effectiveRects];
|
||||
const measured = boxes.length > 0 ? boxes : measure(effectiveSelectors);
|
||||
const anchor = unionBox(measured);
|
||||
const banner = bannerRef.current;
|
||||
if (!anchor || !banner) {
|
||||
@@ -364,7 +329,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
}
|
||||
const nextPos = placePanel(anchor, panelW, panelH, placementRef.current.placement);
|
||||
setPanelPos((prev) => (panelPosEqual(prev, nextPos) ? prev : nextPos));
|
||||
}, [request, boxes, collapsed, isCompact, effectiveSelectors, effectiveRects]);
|
||||
}, [request, boxes, collapsed, isCompact, effectiveSelectors]);
|
||||
|
||||
if (!request) return null;
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { useTranslation } from "@browser-skill/i18n/react";
|
||||
import type { FrameCaptureFailure, FrameCaptureStatus } from "@/transport/types";
|
||||
import {
|
||||
type PointerEvent as ReactPointerEvent,
|
||||
useCallback,
|
||||
@@ -21,6 +22,8 @@ export interface RecordRequestData {
|
||||
|
||||
type Props = {
|
||||
request: RecordRequestData | null;
|
||||
captureStatus?: FrameCaptureStatus;
|
||||
captureFailures?: FrameCaptureFailure[];
|
||||
};
|
||||
|
||||
const VIEWPORT_MARGIN = 16;
|
||||
@@ -52,7 +55,11 @@ function clampDragPos(
|
||||
};
|
||||
}
|
||||
|
||||
export function RecordOverlay({ request }: Props) {
|
||||
export function RecordOverlay({
|
||||
request,
|
||||
captureStatus = "complete",
|
||||
captureFailures = [],
|
||||
}: Props) {
|
||||
const { t } = useTranslation("extension");
|
||||
const [show, setShow] = useState(false);
|
||||
const [elapsedSeconds, setElapsedSeconds] = useState(0);
|
||||
@@ -154,6 +161,7 @@ export function RecordOverlay({ request }: Props) {
|
||||
|
||||
const introVisible = elapsedSeconds * 1_000 < INTRO_DURATION_MS;
|
||||
const positioned = dragPos !== null;
|
||||
const partialCapture = captureStatus === "partial" || captureFailures.length > 0;
|
||||
const defaultTransform = show
|
||||
? "translateX(-50%) translateY(0)"
|
||||
: "translateX(-50%) translateY(8px)";
|
||||
@@ -208,8 +216,9 @@ export function RecordOverlay({ request }: Props) {
|
||||
data-slot="record-overlay-status"
|
||||
style={{
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 8,
|
||||
flexDirection: "column",
|
||||
alignItems: "flex-start",
|
||||
gap: 2,
|
||||
minWidth: 0,
|
||||
flex: 1,
|
||||
borderRadius: 9999,
|
||||
@@ -218,41 +227,57 @@ export function RecordOverlay({ request }: Props) {
|
||||
paddingRight: 4,
|
||||
}}
|
||||
>
|
||||
<span
|
||||
data-slot="record-overlay-indicator"
|
||||
style={{
|
||||
width: 9,
|
||||
height: 9,
|
||||
borderRadius: "50%",
|
||||
backgroundColor: "#ef4444",
|
||||
flexShrink: 0,
|
||||
animation: "bsk-rec-pulse 1.4s ease-in-out infinite",
|
||||
}}
|
||||
/>
|
||||
<span
|
||||
data-slot="record-overlay-label"
|
||||
style={{
|
||||
display: "block",
|
||||
flex: "0 1 auto",
|
||||
fontSize: 16,
|
||||
fontWeight: 500,
|
||||
color: "#333",
|
||||
whiteSpace: "nowrap",
|
||||
userSelect: "none",
|
||||
overflow: "hidden",
|
||||
textOverflow: "ellipsis",
|
||||
width: introVisible ? "auto" : 48,
|
||||
maxWidth: introVisible ? 280 : 48,
|
||||
fontVariantNumeric: introVisible ? undefined : "tabular-nums",
|
||||
textAlign: introVisible ? "left" : "center",
|
||||
margin: 0,
|
||||
padding: 0,
|
||||
minWidth: 0,
|
||||
transition: "max-width 180ms ease-out",
|
||||
}}
|
||||
>
|
||||
{introVisible ? t("recordOverlay.recording") : formatElapsed(elapsedSeconds)}
|
||||
</span>
|
||||
<div style={{ display: "flex", alignItems: "center", gap: 8, minWidth: 0, width: "100%" }}>
|
||||
<span
|
||||
data-slot="record-overlay-indicator"
|
||||
style={{
|
||||
width: 9,
|
||||
height: 9,
|
||||
borderRadius: "50%",
|
||||
backgroundColor: "#ef4444",
|
||||
flexShrink: 0,
|
||||
animation: "bsk-rec-pulse 1.4s ease-in-out infinite",
|
||||
}}
|
||||
/>
|
||||
<span
|
||||
data-slot="record-overlay-label"
|
||||
style={{
|
||||
display: "block",
|
||||
flex: "0 1 auto",
|
||||
fontSize: 16,
|
||||
fontWeight: 500,
|
||||
color: "#333",
|
||||
whiteSpace: "nowrap",
|
||||
userSelect: "none",
|
||||
overflow: "hidden",
|
||||
textOverflow: "ellipsis",
|
||||
width: introVisible ? "auto" : 48,
|
||||
maxWidth: introVisible ? 280 : 48,
|
||||
fontVariantNumeric: introVisible ? undefined : "tabular-nums",
|
||||
textAlign: introVisible ? "left" : "center",
|
||||
margin: 0,
|
||||
padding: 0,
|
||||
minWidth: 0,
|
||||
transition: "max-width 180ms ease-out",
|
||||
}}
|
||||
>
|
||||
{introVisible ? t("recordOverlay.recording") : formatElapsed(elapsedSeconds)}
|
||||
</span>
|
||||
</div>
|
||||
{partialCapture ? (
|
||||
<span
|
||||
data-slot="record-overlay-partial-warning"
|
||||
style={{
|
||||
fontSize: 12,
|
||||
lineHeight: "16px",
|
||||
color: "#b45309",
|
||||
whiteSpace: "normal",
|
||||
maxWidth: 280,
|
||||
}}
|
||||
>
|
||||
{t("recordOverlay.partialCapture")}
|
||||
</span>
|
||||
) : null}
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { i18n } from "@browser-skill/i18n";
|
||||
import { I18nextProvider } from "@browser-skill/i18n/react";
|
||||
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||
import { cleanup, fireEvent, render, screen } from "@testing-library/react";
|
||||
import { createElement } from "react";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { type HelpRequestData, HelpRequestOverlay } from "../HelpRequestOverlay";
|
||||
@@ -38,36 +38,6 @@ describe("HelpRequestOverlay", () => {
|
||||
expect(screen.getByText("Please complete the captcha")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("renders explicit viewport rectangles for cross-frame targets", () => {
|
||||
const { container } = renderOverlay(
|
||||
baseRequest({ rects: [{ top: 30, left: 40, width: 120, height: 50 }] }),
|
||||
);
|
||||
const highlight = container.querySelector<HTMLElement>("[data-slot='help-highlight']");
|
||||
expect(highlight?.style.top).toBe("30px");
|
||||
expect(highlight?.style.left).toBe("40px");
|
||||
expect(highlight?.style.width).toBe("120px");
|
||||
expect(highlight?.style.height).toBe("50px");
|
||||
});
|
||||
|
||||
it("re-resolves cross-frame rectangles after the viewport changes", async () => {
|
||||
const refreshRects = vi.fn(async () => [{ top: 80, left: 90, width: 140, height: 60 }]);
|
||||
const { container } = renderOverlay(
|
||||
baseRequest({
|
||||
rects: [{ top: 30, left: 40, width: 120, height: 50 }],
|
||||
refreshRects,
|
||||
}),
|
||||
);
|
||||
|
||||
window.dispatchEvent(new Event("scroll"));
|
||||
|
||||
await waitFor(() => {
|
||||
const highlight = container.querySelector<HTMLElement>("[data-slot='help-highlight']");
|
||||
expect(highlight?.style.top).toBe("80px");
|
||||
expect(highlight?.style.left).toBe("90px");
|
||||
});
|
||||
expect(refreshRects).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("keeps the inactive render stable", () => {
|
||||
const { container, rerender } = renderOverlay(null);
|
||||
|
||||
|
||||
@@ -178,4 +178,30 @@ describe("RecordOverlay", () => {
|
||||
expect(pill.style.bottom).toBe("auto");
|
||||
expect(pill.style.transform).toBe("none");
|
||||
});
|
||||
|
||||
it("shows a persistent partial-capture warning when frame capture is incomplete", () => {
|
||||
const { container } = render(
|
||||
createElement(
|
||||
I18nextProvider,
|
||||
{ i18n },
|
||||
createElement(RecordOverlay, {
|
||||
request: { id: "rec-1", onFinish: vi.fn() },
|
||||
captureStatus: "partial",
|
||||
captureFailures: [
|
||||
{
|
||||
reason: "not_injectable",
|
||||
frame_id: 2,
|
||||
url: "chrome://settings",
|
||||
},
|
||||
],
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
const warning = container.querySelector("[data-slot='record-overlay-partial-warning']");
|
||||
expect(warning).toBeTruthy();
|
||||
expect(warning?.textContent).toBe(
|
||||
i18n.t("recordOverlay.partialCapture", { ns: "extension" }),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createHelpRequestData } from "../help-request";
|
||||
|
||||
describe("createHelpRequestData", () => {
|
||||
it("gives live delivery and recovery the same rectangle refresh mechanism", async () => {
|
||||
const finish = vi.fn();
|
||||
const query = vi.fn(async () => ({
|
||||
active: true,
|
||||
request: {
|
||||
requestId: "help-1",
|
||||
prompt: "complete the challenge",
|
||||
selectors: [],
|
||||
rects: [{ top: 80, left: 90, width: 140, height: 60 }],
|
||||
timeoutMs: 1_000,
|
||||
},
|
||||
}));
|
||||
const request = createHelpRequestData(
|
||||
{
|
||||
requestId: "help-1",
|
||||
prompt: "complete the challenge",
|
||||
selectors: [],
|
||||
rects: [{ top: 10, left: 20, width: 30, height: 40 }],
|
||||
timeoutMs: 1_000,
|
||||
},
|
||||
{ finish, query },
|
||||
);
|
||||
|
||||
await expect(request.refreshRects?.()).resolves.toEqual([
|
||||
{ top: 80, left: 90, width: 140, height: 60 },
|
||||
]);
|
||||
request.onContinue(" done ");
|
||||
request.onCancel();
|
||||
|
||||
expect(query).toHaveBeenCalledOnce();
|
||||
expect(finish).toHaveBeenNthCalledWith(1, "help-1", "continued", " done ");
|
||||
expect(finish).toHaveBeenNthCalledWith(2, "help-1", "cancelled");
|
||||
});
|
||||
|
||||
it("ignores rectangle replies for a replaced request", async () => {
|
||||
const request = createHelpRequestData(
|
||||
{
|
||||
requestId: "help-1",
|
||||
prompt: "complete the challenge",
|
||||
selectors: [],
|
||||
timeoutMs: 1_000,
|
||||
},
|
||||
{
|
||||
finish: vi.fn(),
|
||||
query: vi.fn(async () => ({
|
||||
active: true,
|
||||
request: {
|
||||
requestId: "help-2",
|
||||
prompt: "new challenge",
|
||||
selectors: [],
|
||||
rects: [{ top: 1, left: 2, width: 3, height: 4 }],
|
||||
timeoutMs: 1_000,
|
||||
},
|
||||
})),
|
||||
},
|
||||
);
|
||||
|
||||
await expect(request.refreshRects?.()).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
|
||||
import { RECORD_START, RECORD_STOP, type RecordStepPayload } from "@/lib/record-bridge";
|
||||
import { handleRecordContentMessage, startRecordCapture } from "../record-capture";
|
||||
|
||||
vi.stubGlobal("chrome", {
|
||||
@@ -96,6 +96,65 @@ describe("handleRecordContentMessage stop/cancel", () => {
|
||||
expect(dispose).not.toHaveBeenCalled();
|
||||
expect(onStop).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("keeps a failed STOP retryable and redelivers its recorded step", async () => {
|
||||
document.body.innerHTML = `<label for="retry">Draft</label><input id="retry" />`;
|
||||
const sendMessage = vi.mocked(chrome.runtime.sendMessage);
|
||||
sendMessage.mockReset();
|
||||
sendMessage.mockRejectedValueOnce(new Error("service worker unavailable"));
|
||||
sendMessage.mockResolvedValueOnce(undefined);
|
||||
|
||||
let activeRequestId: string | null = null;
|
||||
let capture: ReturnType<typeof startRecordCapture> | null = null;
|
||||
const onStop = vi.fn();
|
||||
const dispatch = (
|
||||
message:
|
||||
| { type: typeof RECORD_START; requestId: string; startedAtMs?: number }
|
||||
| { type: typeof RECORD_STOP; requestId: string },
|
||||
sendResponse?: (response: unknown) => void,
|
||||
) =>
|
||||
handleRecordContentMessage(
|
||||
message,
|
||||
{
|
||||
activeRequestId,
|
||||
capture,
|
||||
setActiveRequestId: (id) => {
|
||||
activeRequestId = id;
|
||||
},
|
||||
setCapture: (next) => {
|
||||
capture = next;
|
||||
},
|
||||
onStart: vi.fn(),
|
||||
onStop,
|
||||
},
|
||||
sendResponse,
|
||||
);
|
||||
|
||||
dispatch({ type: RECORD_START, requestId: "rec-retry" });
|
||||
const input = document.querySelector("input")!;
|
||||
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||
input.value = "dirty final value";
|
||||
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
|
||||
const firstResponse = vi.fn();
|
||||
expect(dispatch({ type: RECORD_STOP, requestId: "rec-retry" }, firstResponse)).toBe(true);
|
||||
await vi.waitFor(() =>
|
||||
expect(firstResponse).toHaveBeenCalledWith({
|
||||
ok: false,
|
||||
error: "failed to deliver one or more recorded steps",
|
||||
}),
|
||||
);
|
||||
expect(activeRequestId).toBe("rec-retry");
|
||||
expect(onStop).not.toHaveBeenCalled();
|
||||
|
||||
const retryResponse = vi.fn();
|
||||
expect(dispatch({ type: RECORD_STOP, requestId: "rec-retry" }, retryResponse)).toBe(true);
|
||||
await vi.waitFor(() => expect(retryResponse).toHaveBeenCalledWith({ ok: true }));
|
||||
|
||||
expect(sendMessage).toHaveBeenCalledTimes(2);
|
||||
expect(activeRequestId).toBeNull();
|
||||
expect(onStop).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("record-capture semantic", () => {
|
||||
@@ -228,6 +287,7 @@ describe("record-capture semantic", () => {
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "hover",
|
||||
target: { role: "button", name: "Open user navigation menu" },
|
||||
geometry: { tag: "button", rect: { x: 900, y: 8, w: 32, h: 32 } },
|
||||
});
|
||||
expect(steps[1]).toMatchObject({
|
||||
op: "click",
|
||||
@@ -1554,3 +1614,79 @@ describe("record-capture semantic", () => {
|
||||
expect(steps.map((s) => s.op)).toEqual(["click"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("record-capture child frame", () => {
|
||||
let steps: RecordStepPayload[];
|
||||
|
||||
beforeEach(() => {
|
||||
steps = [];
|
||||
document.body.innerHTML = `
|
||||
<label for="child-q">Child query</label>
|
||||
<input id="child-q" name="child-q" />
|
||||
<button type="button" aria-label="Child search">Search</button>
|
||||
<select aria-label="Child status">
|
||||
<option value="on">On</option>
|
||||
<option value="off">Off</option>
|
||||
</select>
|
||||
`;
|
||||
});
|
||||
|
||||
it("stamps a document token on start and clears it on dispose", () => {
|
||||
const capture = startRecordCapture("rec-child-token", (step) => steps.push(step), {
|
||||
frameMode: "child",
|
||||
});
|
||||
const token = document.documentElement.getAttribute("data-bsk-record-document");
|
||||
expect(token).toBeTruthy();
|
||||
click(document.querySelector("button")!);
|
||||
expect(steps[0]?.documentToken).toBe(token);
|
||||
expect(steps[0]?.frame_url).toBeTruthy();
|
||||
capture.dispose();
|
||||
expect(document.documentElement.hasAttribute("data-bsk-record-document")).toBe(false);
|
||||
});
|
||||
|
||||
it("records click, fill, select, and press in child frame mode", () => {
|
||||
const capture = startRecordCapture("rec-child-ops", (step) => steps.push(step), {
|
||||
frameMode: "child",
|
||||
});
|
||||
const input = document.querySelector("input")!;
|
||||
const select = document.querySelector("select")!;
|
||||
|
||||
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||
input.value = "child value";
|
||||
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
input.dispatchEvent(new FocusEvent("focusout", { bubbles: true }));
|
||||
select.value = "off";
|
||||
select.dispatchEvent(new Event("change", { bubbles: true }));
|
||||
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||
input.dispatchEvent(new KeyboardEvent("keydown", { key: "Enter", bubbles: true }));
|
||||
click(document.querySelector("button")!);
|
||||
capture.dispose();
|
||||
|
||||
expect(steps.map((s) => s.op)).toEqual(
|
||||
expect.arrayContaining(["fill", "select", "press", "click"]),
|
||||
);
|
||||
expect(steps.every((s) => s.documentToken)).toBe(true);
|
||||
});
|
||||
|
||||
it("commits the last fill when dispose runs in child frame mode", () => {
|
||||
const capture = startRecordCapture("rec-child-fill-stop", (step) => steps.push(step), {
|
||||
frameMode: "child",
|
||||
});
|
||||
const input = document.querySelector("input")!;
|
||||
input.dispatchEvent(new FocusEvent("focusin", { bubbles: true }));
|
||||
input.value = "pending child fill";
|
||||
input.dispatchEvent(new Event("input", { bubbles: true }));
|
||||
capture.dispose();
|
||||
|
||||
expect(steps.some((s) => s.op === "fill" && s.value === "pending child fill")).toBe(true);
|
||||
});
|
||||
|
||||
it("does not emit navigate steps for child frame navigations", () => {
|
||||
const capture = startRecordCapture("rec-child-nav", (step) => steps.push(step), {
|
||||
frameMode: "child",
|
||||
});
|
||||
history.pushState({}, "", "/child-frame-path");
|
||||
capture.dispose();
|
||||
expect(steps.filter((s) => s.op === "navigate")).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,31 +0,0 @@
|
||||
import type { HelpQueryResponse, HelpRequestMessage } from "@/lib/help-bridge";
|
||||
import type { HelpRequestData } from "./HelpRequestOverlay";
|
||||
|
||||
export interface HelpRequestDataDeps {
|
||||
finish(requestId: string, outcome: "continued" | "cancelled", note?: string): void;
|
||||
query(): Promise<HelpQueryResponse | undefined>;
|
||||
}
|
||||
|
||||
/** Build the content-side representation shared by live delivery and recovery. */
|
||||
export function createHelpRequestData(
|
||||
message: Omit<HelpRequestMessage, "type">,
|
||||
deps: HelpRequestDataDeps,
|
||||
): HelpRequestData {
|
||||
return {
|
||||
id: message.requestId,
|
||||
prompt: message.prompt,
|
||||
...(message.title ? { title: message.title } : {}),
|
||||
...(message.displayMode ? { displayMode: message.displayMode } : {}),
|
||||
selectors: message.selectors,
|
||||
rects: message.rects,
|
||||
refreshRects: async () => {
|
||||
const response = await deps.query();
|
||||
return response?.active && response.request?.requestId === message.requestId
|
||||
? response.request.rects
|
||||
: undefined;
|
||||
},
|
||||
onContinue: (note: string) =>
|
||||
deps.finish(message.requestId, "continued", note.trim() ? note : undefined),
|
||||
onCancel: () => deps.finish(message.requestId, "cancelled"),
|
||||
};
|
||||
}
|
||||
@@ -1,9 +1,9 @@
|
||||
import {
|
||||
type CaptureTargetDescriptor,
|
||||
describeEventTarget,
|
||||
describeTarget,
|
||||
resolveClickableElement,
|
||||
resolveHoverElement,
|
||||
type TargetDescriptor,
|
||||
} from "@/lib/describe-target";
|
||||
import {
|
||||
evaluateHoverTrigger,
|
||||
@@ -21,12 +21,18 @@ import {
|
||||
RECORD_STEP,
|
||||
RECORD_STOP,
|
||||
type RecordCancelMessage,
|
||||
type RecordFrameMode,
|
||||
type RecordStartAck,
|
||||
type RecordStartMessage,
|
||||
type RecordStepPayload,
|
||||
type RecordStopAck,
|
||||
type RecordStopMessage,
|
||||
} from "@/lib/record-bridge";
|
||||
import {
|
||||
clearRecordDocumentToken,
|
||||
createRecordDocumentToken,
|
||||
stampRecordDocumentToken,
|
||||
} from "@/lib/record-document-token";
|
||||
import { shouldRecordPress } from "@/lib/trace-reducer";
|
||||
import {
|
||||
closestHoverSurfaceCandidate,
|
||||
@@ -39,17 +45,26 @@ import {
|
||||
|
||||
const pendingStepSends = new Map<string, Set<Promise<boolean>>>();
|
||||
const failedStepDeliveries = new Set<string>();
|
||||
const failedStepPayloads = new Map<string, RecordStepPayload[]>();
|
||||
const pendingStopFlushes = new Map<string, Promise<RecordStopAck>>();
|
||||
const knownRecordRequests = new Set<string>();
|
||||
|
||||
function sendRecordStep(requestId: string, step: RecordStepPayload): void {
|
||||
function deliverRecordStep(requestId: string, step: RecordStepPayload): Promise<boolean> {
|
||||
const payload = { type: RECORD_STEP, requestId, step };
|
||||
const pending = Promise.resolve(chrome.runtime.sendMessage(payload)).then(
|
||||
return Promise.resolve(chrome.runtime.sendMessage(payload)).then(
|
||||
() => true,
|
||||
() => {
|
||||
failedStepDeliveries.add(requestId);
|
||||
const failed = failedStepPayloads.get(requestId) ?? [];
|
||||
failed.push(step);
|
||||
failedStepPayloads.set(requestId, failed);
|
||||
return false;
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function sendRecordStep(requestId: string, step: RecordStepPayload): void {
|
||||
const pending = deliverRecordStep(requestId, step);
|
||||
const sends = pendingStepSends.get(requestId) ?? new Set<Promise<boolean>>();
|
||||
sends.add(pending);
|
||||
pendingStepSends.set(requestId, sends);
|
||||
@@ -65,14 +80,15 @@ export interface RecordCaptureController {
|
||||
|
||||
interface FillSession {
|
||||
element: FillableElement;
|
||||
target: TargetDescriptor;
|
||||
target: CaptureTargetDescriptor;
|
||||
baselineValue: string;
|
||||
lastValue: string;
|
||||
pendingCommit?: "enter" | "suggestion" | "blur";
|
||||
}
|
||||
|
||||
interface HoverCandidate {
|
||||
element: Element;
|
||||
target: TargetDescriptor;
|
||||
target: CaptureTargetDescriptor;
|
||||
recordedAt: number;
|
||||
score: number;
|
||||
eligible: boolean;
|
||||
@@ -157,6 +173,28 @@ function nearbyFillableFromSearchChrome(target: Element): FillableElement | null
|
||||
return null;
|
||||
}
|
||||
|
||||
function captureGeometry(el: Element): RecordStepPayload["geometry"] {
|
||||
const rect = el.getBoundingClientRect();
|
||||
const style = window.getComputedStyle(el);
|
||||
return {
|
||||
rect: { x: rect.x, y: rect.y, w: rect.width, h: rect.height },
|
||||
scrollX: window.scrollX,
|
||||
scrollY: window.scrollY,
|
||||
position: style.position,
|
||||
tag: el.tagName.toLowerCase(),
|
||||
};
|
||||
}
|
||||
|
||||
function geometryForEventTarget(
|
||||
target: EventTarget | null,
|
||||
): RecordStepPayload["geometry"] | undefined {
|
||||
if (!(target instanceof Element)) return undefined;
|
||||
const clickable = target.closest(
|
||||
'a,button,input,select,textarea,[role="button"],[role="link"],[role="menuitem"],[contenteditable="true"]',
|
||||
);
|
||||
return captureGeometry(clickable ?? target);
|
||||
}
|
||||
|
||||
function fillableValue(el: FillableElement): string {
|
||||
if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) {
|
||||
return el.value;
|
||||
@@ -232,7 +270,7 @@ function collectHoverTriggerLabelText(root: Element): string {
|
||||
return normalizeLabelText(text);
|
||||
}
|
||||
|
||||
function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDescriptor {
|
||||
function compactHoverTargetName(el: Element, desc: CaptureTargetDescriptor): CaptureTargetDescriptor {
|
||||
if (!desc.name) return desc;
|
||||
const fullText = normalizeLabelText(el.textContent ?? "");
|
||||
const compactName = collectHoverTriggerLabelText(el);
|
||||
@@ -250,7 +288,7 @@ function compactHoverTargetName(el: Element, desc: TargetDescriptor): TargetDesc
|
||||
return desc;
|
||||
}
|
||||
|
||||
function isWeakHoverTarget(target: TargetDescriptor): boolean {
|
||||
function isWeakHoverTarget(target: CaptureTargetDescriptor): boolean {
|
||||
return !target.role && !target.name && target.tag === "div";
|
||||
}
|
||||
|
||||
@@ -261,9 +299,9 @@ function looksLikeAvatarElement(el: Element): boolean {
|
||||
|
||||
function normalizeHoverTarget(
|
||||
el: Element,
|
||||
desc: TargetDescriptor,
|
||||
desc: CaptureTargetDescriptor,
|
||||
decision: HoverTriggerDecision,
|
||||
): TargetDescriptor {
|
||||
): CaptureTargetDescriptor {
|
||||
if (desc.role === "img" && !desc.name && looksLikeAvatarElement(el)) {
|
||||
return { ...desc, name: "image" };
|
||||
}
|
||||
@@ -277,7 +315,7 @@ function normalizeHoverTarget(
|
||||
return desc;
|
||||
}
|
||||
|
||||
function hoverTriggerSignals(el: Element, desc: TargetDescriptor) {
|
||||
function hoverTriggerSignals(el: Element, desc: CaptureTargetDescriptor) {
|
||||
if (!(el instanceof HTMLElement)) return null;
|
||||
const style = hoverTriggerStyle(el);
|
||||
return {
|
||||
@@ -370,12 +408,37 @@ function scheduleInputCompletionCommit(
|
||||
setTimeout(syncAndCommit, 0);
|
||||
}
|
||||
|
||||
export interface RecordCaptureOptions {
|
||||
frameMode?: RecordFrameMode;
|
||||
}
|
||||
|
||||
export function startRecordCapture(
|
||||
_requestId: string,
|
||||
sendStep: (step: RecordStepPayload) => void,
|
||||
options: RecordCaptureOptions = {},
|
||||
): RecordCaptureController {
|
||||
const frameMode = options.frameMode ?? "top";
|
||||
const emitNavigate = frameMode === "top";
|
||||
const documentToken = createRecordDocumentToken();
|
||||
stampRecordDocumentToken(documentToken);
|
||||
const capturedAtMs = () => Date.now();
|
||||
|
||||
const resolveTopPageUrl = (): string => {
|
||||
try {
|
||||
return window.top?.location.href ?? location.href;
|
||||
} catch {
|
||||
return location.href;
|
||||
}
|
||||
};
|
||||
|
||||
const emitStep = (step: RecordStepPayload) => {
|
||||
sendStep({ page_url: location.href, ...step });
|
||||
sendStep({
|
||||
page_url: resolveTopPageUrl(),
|
||||
frame_url: location.href,
|
||||
documentToken,
|
||||
capturedAtMs: capturedAtMs(),
|
||||
...step,
|
||||
});
|
||||
};
|
||||
const hoverSurfaceStateMap = (states: HoverSurfaceState[]): Map<Element, string> =>
|
||||
new Map(states.map((state) => [state.element, state.signature]));
|
||||
@@ -411,14 +474,17 @@ export function startRecordCapture(
|
||||
emitStep({
|
||||
op: "fill",
|
||||
target: session.target,
|
||||
geometry: captureGeometry(session.element),
|
||||
value,
|
||||
commit: session.pendingCommit ?? "blur",
|
||||
...(isPassword ? { redacted: true } : {}),
|
||||
});
|
||||
};
|
||||
|
||||
const commitFillSession = () => {
|
||||
const commitFillSession = (commit: "enter" | "suggestion" | "blur" = "blur") => {
|
||||
if (!fillSession || composing) return;
|
||||
const session = fillSession;
|
||||
session.pendingCommit = commit;
|
||||
fillSession = null;
|
||||
emitFill(session);
|
||||
committedValues.set(session.element, session.lastValue);
|
||||
@@ -443,6 +509,7 @@ export function startRecordCapture(
|
||||
};
|
||||
|
||||
const emitNavigateIfChanged = (causedByAction?: boolean) => {
|
||||
if (!emitNavigate) return;
|
||||
if (location.href === lastUrl) return;
|
||||
commitFillSession();
|
||||
lastUrl = location.href;
|
||||
@@ -624,6 +691,7 @@ export function startRecordCapture(
|
||||
emitStep({
|
||||
op: "hover",
|
||||
target: hover.target,
|
||||
geometry: captureGeometry(hover.element),
|
||||
});
|
||||
emittedHoverElements.add(hover.element);
|
||||
};
|
||||
@@ -678,6 +746,7 @@ export function startRecordCapture(
|
||||
emitStep({
|
||||
op: "click",
|
||||
target,
|
||||
geometry: geometryForEventTarget(eventTarget(event)),
|
||||
expects_navigation: true,
|
||||
});
|
||||
};
|
||||
@@ -779,7 +848,7 @@ export function startRecordCapture(
|
||||
scheduleInputCompletionCommit(
|
||||
sessionElement,
|
||||
syncFillSessionValue,
|
||||
commitFillSession,
|
||||
() => commitFillSession("suggestion"),
|
||||
(el) => fillSession?.element === el,
|
||||
);
|
||||
return;
|
||||
@@ -839,6 +908,7 @@ export function startRecordCapture(
|
||||
emitStep({
|
||||
op: "select",
|
||||
target: desc,
|
||||
geometry: captureGeometry(target),
|
||||
values,
|
||||
labels,
|
||||
expects_navigation: true,
|
||||
@@ -879,7 +949,7 @@ export function startRecordCapture(
|
||||
};
|
||||
}
|
||||
if (fillable) {
|
||||
commitFillSession();
|
||||
commitFillSession(event.key === "Enter" ? "enter" : "blur");
|
||||
}
|
||||
const desc = describeEventTarget(target);
|
||||
if (!desc && !event.key) return;
|
||||
@@ -887,7 +957,7 @@ export function startRecordCapture(
|
||||
emitStep({
|
||||
op: "press",
|
||||
key: event.key,
|
||||
...(desc ? { target: desc } : {}),
|
||||
...(desc ? { target: desc, geometry: geometryForEventTarget(target) } : {}),
|
||||
...(modifiers.length ? { modifiers } : {}),
|
||||
expects_navigation: event.key === "Enter",
|
||||
});
|
||||
@@ -906,9 +976,12 @@ export function startRecordCapture(
|
||||
const urlObserver = new MutationObserver(() => emitNavigateIfChanged());
|
||||
urlObserver.observe(document, { subtree: true, childList: true });
|
||||
const onUrlEvent = () => emitNavigateIfChanged();
|
||||
// Wrap: passing commitFillSession directly would forward the DOM event as
|
||||
// the `commit` argument and stamp it onto the recorded fill step.
|
||||
const onPageHide = () => commitFillSession();
|
||||
window.addEventListener("hashchange", onUrlEvent);
|
||||
window.addEventListener("popstate", onUrlEvent);
|
||||
window.addEventListener("pagehide", commitFillSession);
|
||||
window.addEventListener("pagehide", onPageHide);
|
||||
|
||||
const originalPushState = history.pushState;
|
||||
const originalReplaceState = history.replaceState;
|
||||
@@ -924,6 +997,7 @@ export function startRecordCapture(
|
||||
return {
|
||||
dispose() {
|
||||
commitFillSession();
|
||||
clearRecordDocumentToken();
|
||||
document.removeEventListener("click", onClick, true);
|
||||
document.removeEventListener("mouseover", onMouseOver, true);
|
||||
document.removeEventListener("focusin", onFocusIn, true);
|
||||
@@ -936,7 +1010,7 @@ export function startRecordCapture(
|
||||
urlObserver.disconnect();
|
||||
window.removeEventListener("hashchange", onUrlEvent);
|
||||
window.removeEventListener("popstate", onUrlEvent);
|
||||
window.removeEventListener("pagehide", commitFillSession);
|
||||
window.removeEventListener("pagehide", onPageHide);
|
||||
history.pushState = originalPushState;
|
||||
history.replaceState = originalReplaceState;
|
||||
},
|
||||
@@ -956,8 +1030,8 @@ export function handleRecordContentMessage(
|
||||
capture: RecordCaptureController | null;
|
||||
setActiveRequestId(id: string | null): void;
|
||||
setCapture(capture: RecordCaptureController | null): void;
|
||||
onStart(requestId: string, startedAtMs?: number): void;
|
||||
onStop(): void;
|
||||
onStart?(requestId: string, startedAtMs?: number): void;
|
||||
onStop?(): void;
|
||||
},
|
||||
sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
|
||||
): boolean {
|
||||
@@ -965,15 +1039,23 @@ export function handleRecordContentMessage(
|
||||
if (!knownRecordRequests.has(message.requestId)) {
|
||||
knownRecordRequests.add(message.requestId);
|
||||
failedStepDeliveries.delete(message.requestId);
|
||||
failedStepPayloads.delete(message.requestId);
|
||||
}
|
||||
state.capture?.dispose();
|
||||
const frameMode = message.frameMode ?? "top";
|
||||
state.setCapture(
|
||||
startRecordCapture(message.requestId, (step) => {
|
||||
sendRecordStep(message.requestId, step);
|
||||
}),
|
||||
startRecordCapture(
|
||||
message.requestId,
|
||||
(step) => {
|
||||
sendRecordStep(message.requestId, step);
|
||||
},
|
||||
{ frameMode },
|
||||
),
|
||||
);
|
||||
state.setActiveRequestId(message.requestId);
|
||||
state.onStart(message.requestId, message.startedAtMs);
|
||||
if (message.showOverlay !== false && frameMode === "top") {
|
||||
state.onStart?.(message.requestId, message.startedAtMs);
|
||||
}
|
||||
sendResponse?.({ ok: true });
|
||||
return sendResponse !== undefined;
|
||||
}
|
||||
@@ -988,31 +1070,55 @@ export function handleRecordContentMessage(
|
||||
state.capture?.dispose();
|
||||
state.setCapture(null);
|
||||
const finishStop = () => {
|
||||
state.onStop();
|
||||
state.onStop?.();
|
||||
state.setActiveRequestId(null);
|
||||
};
|
||||
if (isRecordStopMessage(message) && sendResponse) {
|
||||
const existingFlush = pendingStopFlushes.get(message.requestId);
|
||||
if (existingFlush) {
|
||||
void existingFlush.then(sendResponse);
|
||||
return true;
|
||||
}
|
||||
|
||||
const failedBeforeStop = [...(failedStepPayloads.get(message.requestId) ?? [])];
|
||||
if (failedBeforeStop.length > 0) {
|
||||
failedStepPayloads.delete(message.requestId);
|
||||
failedStepDeliveries.delete(message.requestId);
|
||||
}
|
||||
const pending = [...(pendingStepSends.get(message.requestId) ?? [])];
|
||||
void Promise.all(pending).then((delivered) => {
|
||||
finishStop();
|
||||
const succeeded = delivered.every(Boolean) && !failedStepDeliveries.has(message.requestId);
|
||||
const flush = Promise.all(pending).then(async (delivered): Promise<RecordStopAck> => {
|
||||
const retried = await Promise.all(
|
||||
failedBeforeStop.map((step) => deliverRecordStep(message.requestId, step)),
|
||||
);
|
||||
const succeeded =
|
||||
delivered.every(Boolean) &&
|
||||
retried.every(Boolean) &&
|
||||
!failedStepDeliveries.has(message.requestId);
|
||||
if (succeeded) {
|
||||
failedStepDeliveries.delete(message.requestId);
|
||||
failedStepPayloads.delete(message.requestId);
|
||||
knownRecordRequests.delete(message.requestId);
|
||||
finishStop();
|
||||
}
|
||||
return succeeded
|
||||
? { ok: true }
|
||||
: {
|
||||
ok: false,
|
||||
error: "failed to deliver one or more recorded steps",
|
||||
};
|
||||
});
|
||||
pendingStopFlushes.set(message.requestId, flush);
|
||||
void flush.then(sendResponse).finally(() => {
|
||||
if (pendingStopFlushes.get(message.requestId) === flush) {
|
||||
pendingStopFlushes.delete(message.requestId);
|
||||
}
|
||||
sendResponse(
|
||||
succeeded
|
||||
? { ok: true }
|
||||
: {
|
||||
ok: false,
|
||||
error: "failed to deliver one or more recorded steps",
|
||||
},
|
||||
);
|
||||
});
|
||||
return true;
|
||||
}
|
||||
if (isRecordCancelMessage(message)) {
|
||||
failedStepDeliveries.delete(message.requestId);
|
||||
failedStepPayloads.delete(message.requestId);
|
||||
pendingStopFlushes.delete(message.requestId);
|
||||
knownRecordRequests.delete(message.requestId);
|
||||
}
|
||||
finishStop();
|
||||
|
||||
@@ -33,6 +33,9 @@ import {
|
||||
defaultBorrowChromeWindows,
|
||||
requestBorrowConfirmation,
|
||||
} from "@/tools/borrow-confirmation";
|
||||
import {
|
||||
type RecordCaptureStatusMessage,
|
||||
} from "@/lib/record-bridge";
|
||||
import { ToolDispatcher } from "@/tools/dispatcher";
|
||||
import {
|
||||
attachRecordFinishListener,
|
||||
@@ -172,8 +175,27 @@ export default defineBackground(() => {
|
||||
dispatcher.start();
|
||||
const recordDeps = {
|
||||
tabsApi: chrome.tabs,
|
||||
cdp,
|
||||
sendToTab: (tabId: number, msg: Parameters<typeof chrome.tabs.sendMessage>[1]) =>
|
||||
chrome.tabs.sendMessage(tabId, msg),
|
||||
sendToDocument: (
|
||||
tabId: number,
|
||||
documentId: string,
|
||||
msg: Parameters<typeof chrome.tabs.sendMessage>[1],
|
||||
) => chrome.tabs.sendMessage(tabId, msg, { documentId }),
|
||||
getAllFrames: async (tabId: number) => {
|
||||
if (!chrome.webNavigation?.getAllFrames) return [];
|
||||
const frames = await chrome.webNavigation.getAllFrames({ tabId });
|
||||
return (frames ?? []).map((frame) => ({
|
||||
frameId: frame.frameId,
|
||||
documentId: frame.documentId,
|
||||
url: frame.url,
|
||||
parentFrameId: frame.parentFrameId,
|
||||
}));
|
||||
},
|
||||
broadcastCaptureStatus: async (tabId: number, msg: RecordCaptureStatusMessage) => {
|
||||
await chrome.tabs.sendMessage(tabId, msg, { frameId: 0 });
|
||||
},
|
||||
bypassOverlay: async (tabId: number, enabled: boolean) => {
|
||||
try {
|
||||
await chrome.tabs.sendMessage(tabId, {
|
||||
@@ -188,7 +210,7 @@ export default defineBackground(() => {
|
||||
// Message listeners stay up (cheap; fire only for record message types).
|
||||
// Tab / webNavigation observation attaches lazily while a recording is
|
||||
// active — see ensureBrowserObservationListeners in tools/record.ts.
|
||||
attachRecordStepListener();
|
||||
attachRecordStepListener(recordDeps);
|
||||
attachRecordFinishListener(recordDeps);
|
||||
attachRecordQueryListener(recordDeps);
|
||||
if (typeof chrome.notifications?.onClicked?.addListener === "function") {
|
||||
|
||||
@@ -7,7 +7,6 @@ import { BorrowConfirmationOverlay } from "@/content/BorrowConfirmationOverlay";
|
||||
import { ControlOverlay } from "@/content/ControlOverlay";
|
||||
import { createCaptureSuppressController } from "@/content/capture-suppress";
|
||||
import { HelpRequestOverlay } from "@/content/HelpRequestOverlay";
|
||||
import { createHelpRequestData } from "@/content/help-request";
|
||||
import overlayCss from "@/content/overlay.css?inline";
|
||||
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
|
||||
import { RecordOverlay } from "@/content/RecordOverlay";
|
||||
@@ -45,12 +44,15 @@ import {
|
||||
} from "@/lib/overlay-bridge";
|
||||
import { sendInterrupt } from "@/lib/overlay-interrupt-client";
|
||||
import {
|
||||
isRecordCaptureStatusMessage,
|
||||
RECORD_FINISH,
|
||||
RECORD_QUERY,
|
||||
type RecordCaptureStatusMessage,
|
||||
type RecordQueryResponse,
|
||||
type RecordStartAck,
|
||||
type RecordStopAck,
|
||||
} from "@/lib/record-bridge";
|
||||
import type { FrameCaptureFailure, FrameCaptureStatus } from "@/transport/types";
|
||||
import type {
|
||||
BorrowCancelMessage,
|
||||
BorrowRequestMessage,
|
||||
@@ -71,6 +73,8 @@ export default defineContentScript({
|
||||
const overlays = new OverlayController();
|
||||
let recordCapture: RecordCaptureController | null = null;
|
||||
let activeRecordRequestId: string | null = null;
|
||||
let recordCaptureStatus: FrameCaptureStatus = "complete";
|
||||
let recordCaptureFailures: FrameCaptureFailure[] = [];
|
||||
let reactRoot: ReactDOM.Root | null = null;
|
||||
let overlayHost: HTMLElement | null = null;
|
||||
let overlayContainer: HTMLElement | null = null;
|
||||
@@ -189,7 +193,11 @@ export default defineContentScript({
|
||||
requests: overlayState.borrowRequests,
|
||||
}),
|
||||
React.createElement(HelpRequestOverlay, { request: overlayState.activeHelp }),
|
||||
React.createElement(RecordOverlay, { request: overlayState.activeRecord }),
|
||||
React.createElement(RecordOverlay, {
|
||||
request: overlayState.activeRecord,
|
||||
captureStatus: recordCaptureStatus,
|
||||
captureFailures: recordCaptureFailures,
|
||||
}),
|
||||
React.createElement(ControlOverlay, {
|
||||
visible: controlOverlayVisible,
|
||||
interrupting: overlayState.interrupting,
|
||||
@@ -261,7 +269,8 @@ export default defineContentScript({
|
||||
| CaptureSuppressMessage
|
||||
| OverlayAgentOverlayResetMessage
|
||||
| OverlayAgentStateMessage
|
||||
| OverlayAutomationBypassMessage,
|
||||
| OverlayAutomationBypassMessage
|
||||
| RecordCaptureStatusMessage,
|
||||
_sender: chrome.runtime.MessageSender,
|
||||
sendResponse: (response: BorrowResponseMessage | HelpAckMessage | CaptureSuppressAck) => void,
|
||||
) => {
|
||||
@@ -269,6 +278,13 @@ export default defineContentScript({
|
||||
return captureSuppress.handleMessage(message, sendResponse);
|
||||
}
|
||||
|
||||
if (isRecordCaptureStatusMessage(message)) {
|
||||
recordCaptureStatus = message.status;
|
||||
recordCaptureFailures = message.failures ?? [];
|
||||
renderAll();
|
||||
return false;
|
||||
}
|
||||
|
||||
if (isRecordContentMessage(message)) {
|
||||
const needsAsync = handleRecordContentMessage(
|
||||
message,
|
||||
@@ -296,6 +312,8 @@ export default defineContentScript({
|
||||
},
|
||||
onStop: () => {
|
||||
overlays.clearAgentRecordRequest(activeRecordRequestId ?? undefined);
|
||||
recordCaptureStatus = "complete";
|
||||
recordCaptureFailures = [];
|
||||
renderAll();
|
||||
},
|
||||
},
|
||||
@@ -345,7 +363,16 @@ export default defineContentScript({
|
||||
|
||||
if (isHelpRequestMessage(message)) {
|
||||
const helpMsg = message as HelpRequestMessage;
|
||||
const previousHelp = mountHelpRequest(helpMsg);
|
||||
const previousHelp = overlays.setAgentHelpRequest({
|
||||
id: helpMsg.requestId,
|
||||
prompt: helpMsg.prompt,
|
||||
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
||||
...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}),
|
||||
selectors: helpMsg.selectors,
|
||||
onContinue: (note: string) =>
|
||||
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
|
||||
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
|
||||
});
|
||||
if (previousHelp && previousHelp.id !== helpMsg.requestId) {
|
||||
void sendHelpFinish(previousHelp.id, "cancelled");
|
||||
}
|
||||
@@ -398,18 +425,17 @@ export default defineContentScript({
|
||||
});
|
||||
}
|
||||
|
||||
function mountHelpRequest(helpMsg: Omit<HelpRequestMessage, "type">) {
|
||||
return overlays.setAgentHelpRequest(
|
||||
createHelpRequestData(helpMsg, {
|
||||
finish: (requestId, outcome, note) => {
|
||||
void sendHelpFinish(requestId, outcome, note);
|
||||
},
|
||||
query: () =>
|
||||
chrome.runtime.sendMessage({ type: HELP_QUERY }) as Promise<
|
||||
HelpQueryResponse | undefined
|
||||
>,
|
||||
}),
|
||||
);
|
||||
function mountHelpRequest(helpMsg: Omit<HelpRequestMessage, "type">): void {
|
||||
overlays.setAgentHelpRequest({
|
||||
id: helpMsg.requestId,
|
||||
prompt: helpMsg.prompt,
|
||||
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
||||
...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}),
|
||||
selectors: helpMsg.selectors,
|
||||
onContinue: (note: string) =>
|
||||
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
|
||||
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
|
||||
});
|
||||
}
|
||||
|
||||
async function queryActiveHelpWithRetry(): Promise<void> {
|
||||
@@ -442,6 +468,8 @@ export default defineContentScript({
|
||||
) {
|
||||
const requestId = recordQuery.requestId;
|
||||
const startedAtMs = recordQuery.startedAtMs;
|
||||
recordCaptureStatus = recordQuery.captureStatus ?? "complete";
|
||||
recordCaptureFailures = recordQuery.captureFailures ?? [];
|
||||
overlays.setAgentRecordRequest({
|
||||
id: requestId,
|
||||
...(typeof startedAtMs === "number" ? { startedAtMs } : {}),
|
||||
|
||||
@@ -100,6 +100,24 @@ describe("App", () => {
|
||||
expect(screen.queryByText("录制你的操作，供 Agent 参考")).toBeNull();
|
||||
});
|
||||
|
||||
it("renders localized copy for a version failure instead of protocol details", () => {
|
||||
mockUseConnectionState.mockReturnValue({
|
||||
snapshot: {
|
||||
...baseSnapshot,
|
||||
lastError:
|
||||
"[handshake] daemon rejected handshake: version_too_old — peer protocol 1.0 is below local min_compatible_protocol 1.1",
|
||||
},
|
||||
statusState: "disconnected",
|
||||
setLabel,
|
||||
setConnectionEnabled,
|
||||
});
|
||||
|
||||
render(<App />);
|
||||
|
||||
const error = screen.getByText(/bsk 与 BrowserSkill 扩展版本不兼容/);
|
||||
expect(error.textContent).not.toContain("min_compatible_protocol");
|
||||
});
|
||||
|
||||
it("shows single-line compact metadata and copies the instance id", async () => {
|
||||
mockUseConnectionState.mockReturnValue({
|
||||
snapshot: {
|
||||
@@ -200,6 +218,13 @@ describe("App", () => {
|
||||
expect(copied).toContain("--browser 03c3e47f");
|
||||
expect(copied).toContain('--purpose "发布 wiki 文档"');
|
||||
expect(copied).not.toMatch(/bsk record start[^\n]*--url/);
|
||||
expect(copied).toContain("./trace");
|
||||
expect(copied).toContain("trace.json");
|
||||
expect(copied).toContain("pages/");
|
||||
expect(copied).toContain("总结我做了什么");
|
||||
expect(copied).not.toContain("不要重跑");
|
||||
expect(copied).not.toContain("pages + steps");
|
||||
expect(copied).not.toContain("@eN");
|
||||
// Button label stays static; a transient toast confirms the copy.
|
||||
expect(copyButton.textContent).toContain("复制录制指令");
|
||||
await waitFor(() => expect(screen.getByRole("status").textContent).toContain("已复制"));
|
||||
|
||||
@@ -83,6 +83,9 @@ export function App() {
|
||||
const daemonProtocol = snapshot.handshake?.protocol_version ?? "—";
|
||||
const extensionVersion = snapshot.extensionVersion || "—";
|
||||
const instanceId = snapshot.instanceId || "—";
|
||||
const errorMessage = snapshot.lastError?.includes("version_too_old")
|
||||
? t("popup.versionIncompatibleError")
|
||||
: snapshot.lastError;
|
||||
|
||||
const copyInstanceId = async () => {
|
||||
if (!snapshot.instanceId || !navigator.clipboard?.writeText) return;
|
||||
@@ -252,12 +255,12 @@ export function App() {
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{snapshot.lastError && (
|
||||
{errorMessage && (
|
||||
<div
|
||||
className="rounded-lg border border-destructive/25 bg-destructive/10 px-3 py-2 text-xs leading-snug text-destructive"
|
||||
data-slot="popup-error"
|
||||
>
|
||||
{snapshot.lastError}
|
||||
{errorMessage}
|
||||
</div>
|
||||
)}
|
||||
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
import {
|
||||
handleRecordContentMessage,
|
||||
isRecordContentMessage,
|
||||
type RecordCaptureController,
|
||||
} from "@/content/record-capture";
|
||||
import {
|
||||
RECORD_QUERY,
|
||||
type RecordQueryResponse,
|
||||
} from "@/lib/record-bridge";
|
||||
|
||||
/** Capture-only content script for child frames (no React overlay). */
|
||||
export default defineContentScript({
|
||||
matches: ["<all_urls>"],
|
||||
runAt: "document_end",
|
||||
allFrames: true,
|
||||
matchAboutBlank: true,
|
||||
matchOriginAsFallback: true,
|
||||
|
||||
main(ctx) {
|
||||
if (window.top === window) return;
|
||||
|
||||
let recordCapture: RecordCaptureController | null = null;
|
||||
let activeRecordRequestId: string | null = null;
|
||||
|
||||
const onMessage = (
|
||||
message: unknown,
|
||||
_sender: chrome.runtime.MessageSender,
|
||||
sendResponse: (response: unknown) => void,
|
||||
) => {
|
||||
if (isRecordContentMessage(message)) {
|
||||
const needsAsync = handleRecordContentMessage(
|
||||
message,
|
||||
{
|
||||
activeRequestId: activeRecordRequestId,
|
||||
capture: recordCapture,
|
||||
setActiveRequestId: (id) => {
|
||||
activeRecordRequestId = id;
|
||||
},
|
||||
setCapture: (capture) => {
|
||||
recordCapture = capture;
|
||||
},
|
||||
},
|
||||
sendResponse as never,
|
||||
);
|
||||
return needsAsync;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
async function queryActiveRecord(): Promise<void> {
|
||||
try {
|
||||
const recordQuery = (await chrome.runtime.sendMessage({
|
||||
type: RECORD_QUERY,
|
||||
})) as RecordQueryResponse | undefined;
|
||||
if (
|
||||
recordQuery?.active &&
|
||||
typeof recordQuery.requestId === "string" &&
|
||||
activeRecordRequestId === null
|
||||
) {
|
||||
// Background rearm sends RECORD_START; nothing to mount locally.
|
||||
}
|
||||
} catch (err) {
|
||||
console.debug("[bsk record-frame] record query failed", err);
|
||||
}
|
||||
}
|
||||
|
||||
chrome.runtime.onMessage.addListener(onMessage);
|
||||
void queryActiveRecord();
|
||||
|
||||
ctx.onInvalidated(() => {
|
||||
chrome.runtime.onMessage.removeListener(onMessage);
|
||||
recordCapture?.dispose();
|
||||
recordCapture = null;
|
||||
activeRecordRequestId = null;
|
||||
});
|
||||
},
|
||||
});
|
||||
@@ -27,19 +27,19 @@ function handshake(
|
||||
|
||||
describe("computeConnectedState (protocol-based compat)", () => {
|
||||
it("returns connected when protocol strings match", () => {
|
||||
expect(computeConnectedState(handshake("1.0", "1.0"), MIN_COMPATIBLE_PROTOCOL)).toEqual({
|
||||
expect(computeConnectedState(handshake("1.1", "1.1"), MIN_COMPATIBLE_PROTOCOL)).toEqual({
|
||||
kind: "connected",
|
||||
});
|
||||
});
|
||||
|
||||
it("returns version_skew when daemon protocol minor is newer", () => {
|
||||
expect(computeConnectedState(handshake("1.1", "1.0"))).toEqual({
|
||||
expect(computeConnectedState(handshake("1.2", "1.1"))).toEqual({
|
||||
kind: "version_skew",
|
||||
});
|
||||
});
|
||||
|
||||
it("returns version_skew when daemon protocol string differs but floor is satisfied", () => {
|
||||
expect(computeConnectedState(handshake("1", "1.0"))).toEqual({
|
||||
expect(computeConnectedState(handshake("1.1.0", "1.1"))).toEqual({
|
||||
kind: "version_skew",
|
||||
});
|
||||
});
|
||||
@@ -48,16 +48,19 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
||||
const result = computeConnectedState(handshake("2.0", "1.0"));
|
||||
expect(result.kind).toBe("rejected");
|
||||
if (result.kind === "rejected") {
|
||||
expect(result.reason).toContain("protocol-major mismatch");
|
||||
expect(result.reason).toContain("bsk CLI");
|
||||
expect(result.reason).toContain("BrowserSkill extension");
|
||||
expect(result.reason).not.toContain("protocol");
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects when extension is below daemon min_compatible_protocol", () => {
|
||||
const result = computeConnectedState(handshake("1.0", "1.5"));
|
||||
const result = computeConnectedState(handshake("1.1", "1.5"));
|
||||
expect(result.kind).toBe("rejected");
|
||||
if (result.kind === "rejected") {
|
||||
expect(result.reason).toContain("min_compatible_protocol");
|
||||
expect(result.reason).toContain("1.5");
|
||||
expect(result.reason).toContain("BrowserSkill extension is out of date");
|
||||
expect(result.reason).toContain("Update and reload the extension");
|
||||
expect(result.reason).not.toContain("min_compatible_protocol");
|
||||
}
|
||||
});
|
||||
|
||||
@@ -65,7 +68,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
||||
const result = computeConnectedState({
|
||||
server: "browser-skill-daemon",
|
||||
version: "0.1.0",
|
||||
protocol_version: "1.0",
|
||||
protocol_version: "1.1",
|
||||
min_compatible_peer: "0.1.0",
|
||||
});
|
||||
expect(result).toEqual({ kind: "connected" });
|
||||
@@ -75,16 +78,23 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
||||
const result = computeConnectedState(handshake("1.0", "1.0"), "1.1");
|
||||
expect(result.kind).toBe("rejected");
|
||||
if (result.kind === "rejected") {
|
||||
expect(result.reason).toContain("below extension min_compatible_protocol");
|
||||
expect(result.reason).toContain("bsk CLI is out of date");
|
||||
expect(result.reason).toContain("bsk daemon restart");
|
||||
expect(result.reason).not.toContain("min_compatible_protocol");
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects a pre-Trace-v3 daemon using the configured extension floor", () => {
|
||||
const result = computeConnectedState(handshake("1.0", "1.0"));
|
||||
expect(result.kind).toBe("rejected");
|
||||
});
|
||||
|
||||
it("rejects malformed daemon min_compatible_protocol with a daemon-floor reason", () => {
|
||||
const result = computeConnectedState(handshake("1.0", "not-a-protocol"));
|
||||
const result = computeConnectedState(handshake("1.1", "not-a-protocol"));
|
||||
expect(result.kind).toBe("rejected");
|
||||
if (result.kind === "rejected") {
|
||||
expect(result.reason).toContain("daemon min_compatible_protocol");
|
||||
expect(result.reason).toContain("not-a-protocol");
|
||||
expect(result.reason).toContain("could not verify version compatibility");
|
||||
expect(result.reason).not.toContain("min_compatible_protocol");
|
||||
}
|
||||
});
|
||||
|
||||
@@ -242,11 +252,11 @@ describe("ConnectionController connectionEnabled", () => {
|
||||
const second = transport.send.mock.calls[1]?.[0] as { id: string };
|
||||
expect(second.id).not.toBe(first.id);
|
||||
|
||||
transport.emitMessage({ id: first.id, result: handshake("1.0", "1.0") });
|
||||
transport.emitMessage({ id: first.id, result: handshake("1.1", "1.1") });
|
||||
await Promise.resolve();
|
||||
expect(controller.snapshot().state).not.toBe("connected");
|
||||
|
||||
transport.emitMessage({ id: second.id, result: handshake("1.0", "1.0") });
|
||||
transport.emitMessage({ id: second.id, result: handshake("1.1", "1.1") });
|
||||
await vi.waitFor(() => expect(controller.snapshot().state).toBe("connected"));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -32,7 +32,6 @@ describe("help-bridge", () => {
|
||||
prompt: "log in",
|
||||
displayMode: "compact",
|
||||
selectors: ["#login"],
|
||||
rects: [{ top: 10, left: 20, width: 100, height: 40 }],
|
||||
timeoutMs: 1000,
|
||||
}),
|
||||
).toBe(true);
|
||||
|
||||
@@ -0,0 +1,172 @@
|
||||
import type { RenderedRef } from "@browser-skill/vom";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { CapturedNode } from "@/tools/vom/capture";
|
||||
import { matchTarget } from "../match-target";
|
||||
|
||||
function node(backendNodeId: number, overrides: Partial<CapturedNode> = {}): CapturedNode {
|
||||
return {
|
||||
backendNodeId,
|
||||
parentBackendNodeId: null,
|
||||
tag: "button",
|
||||
attrs: {},
|
||||
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("matchTarget", () => {
|
||||
const refs: RenderedRef[] = [
|
||||
{ ref: "e1", backendNodeId: 42, role: "button", name: "发布", line: 3 },
|
||||
];
|
||||
|
||||
it("matches a unique node by viewport coordinates", () => {
|
||||
const target = matchTarget({
|
||||
geometry: {
|
||||
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
scrollX: 0,
|
||||
scrollY: 0,
|
||||
position: "static",
|
||||
tag: "button",
|
||||
},
|
||||
captured: [node(42)],
|
||||
refs,
|
||||
fallback: { tag: "button", name: "发布" },
|
||||
});
|
||||
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
|
||||
});
|
||||
|
||||
it("matches a non-fixed node after the top frame scrolls", () => {
|
||||
const target = matchTarget({
|
||||
geometry: {
|
||||
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
scrollX: 40,
|
||||
scrollY: 300,
|
||||
position: "static",
|
||||
tag: "button",
|
||||
},
|
||||
captured: [
|
||||
node(42, {
|
||||
documentRect: { x: 50, y: 320, w: 100, h: 30 },
|
||||
localRect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
}),
|
||||
],
|
||||
refs,
|
||||
fallback: { tag: "button", name: "发布" },
|
||||
});
|
||||
|
||||
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
|
||||
});
|
||||
|
||||
it("matches a static node by document coordinates when scroll changed after observation", () => {
|
||||
const target = matchTarget({
|
||||
geometry: {
|
||||
rect: { x: 10, y: 50, w: 100, h: 30 },
|
||||
scrollX: 0,
|
||||
scrollY: 250,
|
||||
position: "static",
|
||||
tag: "button",
|
||||
},
|
||||
captured: [
|
||||
node(42, {
|
||||
documentRect: { x: 10, y: 300, w: 100, h: 30 },
|
||||
localRect: { x: 10, y: 200, w: 100, h: 30 },
|
||||
rect: { x: 10, y: 200, w: 100, h: 30 },
|
||||
}),
|
||||
],
|
||||
refs,
|
||||
fallback: { tag: "button", name: "发布" },
|
||||
});
|
||||
|
||||
expect(target).toEqual({ ref: "e1", role: "button", name: "发布" });
|
||||
});
|
||||
|
||||
it("returns unmatched when multiple candidates match", () => {
|
||||
const target = matchTarget({
|
||||
geometry: {
|
||||
rect: { x: 10, y: 20, w: 100, h: 30 },
|
||||
scrollX: 0,
|
||||
scrollY: 0,
|
||||
position: "static",
|
||||
tag: "button",
|
||||
},
|
||||
captured: [node(42), node(43)],
|
||||
refs,
|
||||
});
|
||||
expect(target.unmatched).toBe(true);
|
||||
});
|
||||
|
||||
it("uses viewport coordinates for fixed elements", () => {
|
||||
const target = matchTarget({
|
||||
geometry: {
|
||||
rect: { x: 5, y: 5, w: 80, h: 24 },
|
||||
scrollX: 100,
|
||||
scrollY: 200,
|
||||
position: "fixed",
|
||||
tag: "button",
|
||||
},
|
||||
captured: [node(42, { position: "fixed", rect: { x: 5, y: 5, w: 80, h: 24 } })],
|
||||
refs,
|
||||
});
|
||||
expect(target.ref).toBe("e1");
|
||||
});
|
||||
|
||||
it("isolates geometry matching to the owner iframe backend node", () => {
|
||||
const sharedRect = { x: 10, y: 20, w: 100, h: 30 };
|
||||
const iframeRefs: RenderedRef[] = [
|
||||
{ ref: "e1", backendNodeId: 42, role: "button", name: "Top", line: 1 },
|
||||
{ ref: "e2", backendNodeId: 99, role: "button", name: "Iframe", line: 2 },
|
||||
];
|
||||
const target = matchTarget({
|
||||
geometry: {
|
||||
rect: sharedRect,
|
||||
scrollX: 0,
|
||||
scrollY: 0,
|
||||
position: "static",
|
||||
tag: "button",
|
||||
ownerFrameBackendNodeId: 13,
|
||||
},
|
||||
captured: [
|
||||
node(42, { rect: sharedRect, documentRect: sharedRect }),
|
||||
node(99, {
|
||||
rect: sharedRect,
|
||||
documentRect: sharedRect,
|
||||
ownerFrameBackendNodeId: 13,
|
||||
}),
|
||||
],
|
||||
refs: iframeRefs,
|
||||
fallback: { tag: "button", name: "Iframe" },
|
||||
});
|
||||
expect(target).toEqual({ ref: "e2", role: "button", name: "Iframe" });
|
||||
});
|
||||
|
||||
it("matches iframe static elements using frame scroll and document coordinates", () => {
|
||||
const iframeRefs: RenderedRef[] = [
|
||||
{ ref: "e2", backendNodeId: 99, role: "button", name: "Iframe", line: 2 },
|
||||
];
|
||||
const target = matchTarget({
|
||||
geometry: {
|
||||
rect: { x: 10, y: 50, w: 100, h: 30 },
|
||||
scrollX: 0,
|
||||
scrollY: 120,
|
||||
position: "static",
|
||||
tag: "button",
|
||||
ownerFrameBackendNodeId: 13,
|
||||
},
|
||||
captured: [
|
||||
node(99, {
|
||||
ownerFrameBackendNodeId: 13,
|
||||
documentRect: { x: 10, y: 170, w: 100, h: 30 },
|
||||
localRect: { x: 10, y: 50, w: 100, h: 30 },
|
||||
rect: { x: 10, y: 50, w: 100, h: 30 },
|
||||
}),
|
||||
],
|
||||
refs: iframeRefs,
|
||||
fallback: { tag: "button", name: "Iframe" },
|
||||
});
|
||||
expect(target).toEqual({ ref: "e2", role: "button", name: "Iframe" });
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,116 @@
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
import type { CapturedNode } from "@/tools/vom/capture";
|
||||
import type { DraftTraceStep } from "@/transport/types";
|
||||
import { resetStateIdCounterForTests } from "../record-constants";
|
||||
import {
|
||||
applyTargetMatching,
|
||||
buildTraceV3,
|
||||
createObservationState,
|
||||
rememberStepOnPage,
|
||||
} from "../record-observation";
|
||||
import { registerObservation } from "../trace-reducer";
|
||||
|
||||
const URL = "https://example.com/login";
|
||||
|
||||
function capturedInput(): CapturedNode {
|
||||
return {
|
||||
backendNodeId: 42,
|
||||
parentBackendNodeId: null,
|
||||
tag: "input",
|
||||
attrs: {},
|
||||
rect: { x: 20, y: 40, w: 200, h: 30 },
|
||||
localRect: { x: 20, y: 40, w: 200, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
};
|
||||
}
|
||||
|
||||
function finalizedFillBody(value: string, redactValues: boolean): string {
|
||||
const obs = createObservationState({ redactValues });
|
||||
const rawVomText = '@vom 1\ntextbox "Password" value="••••••" [ref=e1]';
|
||||
const stateId = registerObservation(obs.stateRegistry, { url: URL, rawVomText });
|
||||
obs.lastSettled = {
|
||||
stateId,
|
||||
captured: [capturedInput()],
|
||||
matchNodes: [capturedInput()],
|
||||
documentTokenOwners: new Map(),
|
||||
refs: [{ ref: "e1", backendNodeId: 42, role: "textbox", name: "Password", line: 1 }],
|
||||
url: URL,
|
||||
vomText: rawVomText,
|
||||
};
|
||||
|
||||
const draft: DraftTraceStep = {
|
||||
op: "fill",
|
||||
target: { unmatched: true },
|
||||
value,
|
||||
geometry: {
|
||||
rect: { x: 20, y: 40, w: 200, h: 30 },
|
||||
scrollX: 0,
|
||||
scrollY: 0,
|
||||
position: "static",
|
||||
tag: "input",
|
||||
},
|
||||
};
|
||||
applyTargetMatching(obs, draft, 1);
|
||||
draft.postStateId = stateId;
|
||||
rememberStepOnPage(obs, draft, 1);
|
||||
|
||||
return (
|
||||
buildTraceV3({
|
||||
obs,
|
||||
steps: [draft],
|
||||
startedAt: "2026-08-12T00:00:00.000Z",
|
||||
stoppedBy: "user_finish",
|
||||
bskVersion: "test",
|
||||
}).states[0]?.body ?? ""
|
||||
);
|
||||
}
|
||||
|
||||
describe("record observation annotations", () => {
|
||||
beforeEach(() => {
|
||||
resetStateIdCounterForTests();
|
||||
});
|
||||
|
||||
it("omits a fill literal from finalized state bodies when values are redacted", () => {
|
||||
const secret = "hunter2-private";
|
||||
const body = finalizedFillBody(secret, true);
|
||||
|
||||
expect(body).toContain("step 1: fill");
|
||||
expect(body).not.toContain(secret);
|
||||
});
|
||||
|
||||
it("keeps fill details in ordinary recording annotations", () => {
|
||||
const value = "ordinary text";
|
||||
const body = finalizedFillBody(value, false);
|
||||
|
||||
expect(body).toContain(`step 1: fill: ${JSON.stringify(value)}`);
|
||||
});
|
||||
});
|
||||
|
||||
describe("applyTargetMatching without a settled observation", () => {
|
||||
it("keeps the capture role/name instead of a bare unmatched target", () => {
|
||||
const obs = createObservationState();
|
||||
const draft: DraftTraceStep = {
|
||||
op: "hover",
|
||||
target: { unmatched: true },
|
||||
captureTarget: { tag: "button", role: "button", name: "新建" },
|
||||
geometry: {
|
||||
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||
scrollX: 0,
|
||||
scrollY: 0,
|
||||
position: "static",
|
||||
tag: "button",
|
||||
},
|
||||
};
|
||||
|
||||
applyTargetMatching(obs, draft, 1);
|
||||
|
||||
expect(draft.preStateId).toBeUndefined();
|
||||
expect(draft.target).toEqual({
|
||||
role: "button",
|
||||
name: "新建",
|
||||
unmatched: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -12,12 +12,42 @@ describe("recording-step-buffer", () => {
|
||||
expect(buffer.steps).toEqual([
|
||||
{
|
||||
op: "click",
|
||||
target: { tag: "button", role: "button", name: "发布" },
|
||||
target: { unmatched: true },
|
||||
captureTarget: { tag: "button", role: "button", name: "发布" },
|
||||
},
|
||||
]);
|
||||
expect(buffer.pendingNavigation).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps the hovered element description as a capture fallback", () => {
|
||||
const buffer = { steps: [], pendingNavigation: false };
|
||||
appendRecordedPayload(buffer, {
|
||||
op: "hover",
|
||||
target: { tag: "button", role: "button", name: "新建" },
|
||||
geometry: {
|
||||
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||
scrollX: 0,
|
||||
scrollY: 0,
|
||||
position: "static",
|
||||
tag: "button",
|
||||
},
|
||||
});
|
||||
expect(buffer.steps).toEqual([
|
||||
{
|
||||
op: "hover",
|
||||
target: { unmatched: true },
|
||||
captureTarget: { tag: "button", role: "button", name: "新建" },
|
||||
geometry: {
|
||||
rect: { x: 900, y: 8, w: 60, h: 32 },
|
||||
scrollX: 0,
|
||||
scrollY: 0,
|
||||
position: "static",
|
||||
tag: "button",
|
||||
},
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("annotates navigated_to on action-caused navigation instead of wait_for_navigation", () => {
|
||||
const buffer = {
|
||||
steps: [
|
||||
@@ -71,10 +101,41 @@ describe("recording-step-buffer", () => {
|
||||
currentUrl: "https://example.com/a",
|
||||
pendingNavigation: false,
|
||||
};
|
||||
observeRecordedNavigation(buffer, "https://example.com/b", false);
|
||||
const result = observeRecordedNavigation(buffer, "https://example.com/b", false);
|
||||
expect(result).toEqual({ kind: "appended", index: 0 });
|
||||
expect(buffer.steps[0]).toMatchObject({
|
||||
op: "navigate",
|
||||
url: "https://example.com/b",
|
||||
});
|
||||
});
|
||||
|
||||
it("asks the recorder to coalesce redirect hops instead of emitting each one", () => {
|
||||
const buffer = {
|
||||
steps: [],
|
||||
currentUrl: "https://passport.example/login",
|
||||
pendingNavigation: false,
|
||||
};
|
||||
const hop1 = observeRecordedNavigation(
|
||||
buffer,
|
||||
"https://passport.example/callback",
|
||||
false,
|
||||
"link",
|
||||
["server_redirect"],
|
||||
);
|
||||
expect(hop1).toEqual({
|
||||
kind: "coalesce_redirect",
|
||||
url: "https://passport.example/callback",
|
||||
});
|
||||
expect(buffer.steps).toEqual([]);
|
||||
|
||||
const hop2 = observeRecordedNavigation(buffer, "https://app.example/dashboard", false, "link", [
|
||||
"client_redirect",
|
||||
]);
|
||||
expect(hop2).toEqual({
|
||||
kind: "coalesce_redirect",
|
||||
url: "https://app.example/dashboard",
|
||||
});
|
||||
expect(buffer.steps).toEqual([]);
|
||||
expect(buffer.currentUrl).toBe("https://app.example/dashboard");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { DraftTraceStep } from "@/transport/types";
|
||||
import { reduceTraceSteps, resolveTraceStartUrl, shouldRecordPress } from "../trace-reducer";
|
||||
import { resetStateIdCounterForTests } from "@/lib/record-constants";
|
||||
import {
|
||||
reduceTraceSteps,
|
||||
registerObservation,
|
||||
resolveTraceStartUrl,
|
||||
shouldRecordPress,
|
||||
} from "../trace-reducer";
|
||||
|
||||
describe("shouldRecordPress", () => {
|
||||
it("keeps Enter and Escape", () => {
|
||||
@@ -17,86 +22,85 @@ describe("shouldRecordPress", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("reduceTraceSteps", () => {
|
||||
it("builds steps with pages dictionary and page id references", () => {
|
||||
const drafts: DraftTraceStep[] = [
|
||||
{
|
||||
op: "navigate",
|
||||
url: "https://example.com/search?q=hello&utm_source=x",
|
||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
||||
},
|
||||
{
|
||||
op: "fill",
|
||||
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
||||
value: "browser skill",
|
||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
||||
},
|
||||
{
|
||||
op: "press",
|
||||
key: "Enter",
|
||||
target: { tag: "input", role: "textbox", name: "搜索", name_attr: "q" },
|
||||
navigated_to: "https://example.com/results/42",
|
||||
page_url: "https://example.com/search?q=hello&utm_source=x",
|
||||
},
|
||||
{
|
||||
op: "click",
|
||||
target: { tag: "button", role: "button", name: "发布" },
|
||||
navigated_to: "https://example.com/p/99",
|
||||
page_url: "https://example.com/results/42",
|
||||
},
|
||||
{
|
||||
op: "press",
|
||||
key: "a",
|
||||
page_url: "https://example.com/p/99",
|
||||
},
|
||||
];
|
||||
|
||||
const { pages, steps } = reduceTraceSteps(
|
||||
drafts,
|
||||
"https://example.com/search?q=hello&utm_source=x",
|
||||
);
|
||||
expect(JSON.stringify(steps)).not.toContain("parameters");
|
||||
expect(JSON.stringify(steps)).not.toContain("intent");
|
||||
expect(JSON.stringify(steps)).not.toContain("summary");
|
||||
expect(steps.map((s) => s.op)).toEqual(["navigate", "fill", "press", "click"]);
|
||||
expect(pages.map((p) => p.url)).toEqual([
|
||||
"https://example.com/search?q=hello&utm_source=x",
|
||||
"https://example.com/results/42",
|
||||
"https://example.com/p/99",
|
||||
]);
|
||||
|
||||
expect(steps[0]).toMatchObject({
|
||||
id: 1,
|
||||
op: "navigate",
|
||||
page: "p1",
|
||||
to: "https://example.com/search?q=hello&utm_source=x",
|
||||
describe("reduceTraceSteps v3", () => {
|
||||
it("builds steps with state/result references and deduped states", () => {
|
||||
resetStateIdCounterForTests();
|
||||
const registry = new Map();
|
||||
const body = "@vom 1\nL1 page";
|
||||
const s1 = registerObservation(registry, {
|
||||
url: "https://example.com/search",
|
||||
rawVomText: body,
|
||||
});
|
||||
const s2 = registerObservation(registry, {
|
||||
url: "https://example.com/results/42",
|
||||
rawVomText: "@vom 1\nL1 results",
|
||||
});
|
||||
|
||||
const { states, steps } = reduceTraceSteps(
|
||||
[
|
||||
{
|
||||
op: "navigate",
|
||||
url: "https://example.com/search",
|
||||
preStateId: s1,
|
||||
postStateId: s1,
|
||||
},
|
||||
{
|
||||
op: "fill",
|
||||
target: { ref: "e1", role: "textbox", name: "搜索" },
|
||||
value: "browser skill",
|
||||
commit: "enter",
|
||||
preStateId: s1,
|
||||
postStateId: s1,
|
||||
},
|
||||
{
|
||||
op: "press",
|
||||
key: "Enter",
|
||||
target: { ref: "e1", role: "textbox", name: "搜索" },
|
||||
preStateId: s1,
|
||||
postStateId: s2,
|
||||
navigated_to: "https://example.com/results/42",
|
||||
},
|
||||
{
|
||||
op: "click",
|
||||
target: { ref: "e2", role: "button", name: "发布" },
|
||||
preStateId: s2,
|
||||
postStateId: s2,
|
||||
},
|
||||
],
|
||||
registry,
|
||||
);
|
||||
|
||||
expect(states).toHaveLength(2);
|
||||
expect(steps.map((s) => s.op)).toEqual(["navigate", "fill", "press", "click"]);
|
||||
expect(steps[1]).toMatchObject({
|
||||
op: "fill",
|
||||
page: "p1",
|
||||
value: "browser skill",
|
||||
state: s1,
|
||||
result: { state: s1 },
|
||||
commit: "enter",
|
||||
});
|
||||
|
||||
expect(steps[2]).toMatchObject({
|
||||
op: "press",
|
||||
key: "Enter",
|
||||
page: "p1",
|
||||
effect: { navigated_to: "p2" },
|
||||
});
|
||||
|
||||
expect(steps[3]).toMatchObject({
|
||||
op: "click",
|
||||
page: "p2",
|
||||
effect: { navigated_to: "p3" },
|
||||
state: s1,
|
||||
result: { state: s2 },
|
||||
});
|
||||
expect(JSON.stringify(steps)).not.toContain("page");
|
||||
expect(JSON.stringify(steps)).not.toContain("effect");
|
||||
});
|
||||
|
||||
it("collapses consecutive navigations", () => {
|
||||
const { steps } = reduceTraceSteps([
|
||||
{ op: "navigate", url: "https://a.example/redirect1" },
|
||||
{ op: "navigate", url: "https://a.example/final" },
|
||||
]);
|
||||
resetStateIdCounterForTests();
|
||||
const registry = new Map();
|
||||
const s1 = registerObservation(registry, {
|
||||
url: "https://a.example/redirect1",
|
||||
rawVomText: "a",
|
||||
});
|
||||
const { steps } = reduceTraceSteps(
|
||||
[
|
||||
{ op: "navigate", url: "https://a.example/redirect1", preStateId: s1, postStateId: s1 },
|
||||
{ op: "navigate", url: "https://a.example/final", preStateId: s1, postStateId: s1 },
|
||||
],
|
||||
registry,
|
||||
);
|
||||
expect(steps).toHaveLength(1);
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "navigate",
|
||||
@@ -104,73 +108,124 @@ describe("reduceTraceSteps", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps committed empty fill steps", () => {
|
||||
const { steps } = reduceTraceSteps(
|
||||
it("keeps the origin state and cause of the hop that started a redirect chain", () => {
|
||||
resetStateIdCounterForTests();
|
||||
const registry = new Map();
|
||||
const start = registerObservation(registry, {
|
||||
url: "https://example.com/",
|
||||
rawVomText: "start",
|
||||
});
|
||||
const hop = registerObservation(registry, {
|
||||
url: "https://iwiki.woa.com/",
|
||||
rawVomText: "hop",
|
||||
});
|
||||
const landing = registerObservation(registry, {
|
||||
url: "https://iwiki.woa.com/dashboard",
|
||||
rawVomText: "landing",
|
||||
});
|
||||
|
||||
const { steps, stepIdByDraftId } = reduceTraceSteps(
|
||||
[
|
||||
{
|
||||
op: "fill",
|
||||
target: { tag: "input", role: "textbox", name: "Search query" },
|
||||
value: "",
|
||||
page_url: "https://example.com/search",
|
||||
op: "navigate",
|
||||
url: "https://iwiki.woa.com/",
|
||||
preStateId: start,
|
||||
postStateId: hop,
|
||||
transitionType: "typed",
|
||||
},
|
||||
{
|
||||
op: "navigate",
|
||||
url: "https://iwiki.woa.com/dashboard",
|
||||
preStateId: hop,
|
||||
postStateId: landing,
|
||||
transitionType: "link",
|
||||
},
|
||||
],
|
||||
"https://example.com/search",
|
||||
registry,
|
||||
);
|
||||
|
||||
expect(steps).toEqual([
|
||||
expect.objectContaining({
|
||||
op: "fill",
|
||||
value: "",
|
||||
}),
|
||||
]);
|
||||
expect(steps).toHaveLength(1);
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "navigate",
|
||||
id: 1,
|
||||
state: start,
|
||||
result: { state: landing },
|
||||
to: "https://iwiki.woa.com/dashboard",
|
||||
cause: "user_typed",
|
||||
});
|
||||
// Both drafts fold into step 1, so anything tagged with either draft id
|
||||
// must resolve to it.
|
||||
expect(stepIdByDraftId.get(1)).toBe(1);
|
||||
expect(stepIdByDraftId.get(2)).toBe(1);
|
||||
});
|
||||
|
||||
it("maps select navigated_to onto effect.navigated_to (page id)", () => {
|
||||
const { pages, steps } = reduceTraceSteps(
|
||||
it("maps draft ids to published step ids across dropped drafts", () => {
|
||||
resetStateIdCounterForTests();
|
||||
const registry = new Map();
|
||||
const s1 = registerObservation(registry, { url: "https://a.example/", rawVomText: "a" });
|
||||
|
||||
const { steps, stepIdByDraftId } = reduceTraceSteps(
|
||||
[
|
||||
{
|
||||
op: "select",
|
||||
target: { tag: "select", role: "combobox", name: "分类" },
|
||||
values: ["tech"],
|
||||
labels: ["技术"],
|
||||
navigated_to: "https://example.com/list?cat=tech",
|
||||
page_url: "https://example.com/list",
|
||||
},
|
||||
// Dropped: bare typing press.
|
||||
{ op: "press", key: "a", preStateId: s1, postStateId: s1 },
|
||||
{ op: "click", target: { ref: "e2" }, preStateId: s1, postStateId: s1 },
|
||||
],
|
||||
"https://example.com/list",
|
||||
registry,
|
||||
);
|
||||
expect(pages.map((p) => p.url)).toEqual([
|
||||
"https://example.com/list",
|
||||
"https://example.com/list?cat=tech",
|
||||
]);
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "select",
|
||||
page: "p1",
|
||||
selection: [{ value: "tech", label: "技术" }],
|
||||
effect: { navigated_to: "p2" },
|
||||
});
|
||||
|
||||
expect(steps.map((step) => step.op)).toEqual(["click"]);
|
||||
expect(stepIdByDraftId.get(1)).toBeUndefined();
|
||||
expect(stepIdByDraftId.get(2)).toBe(1);
|
||||
});
|
||||
|
||||
it("keeps committed empty fills", () => {
|
||||
resetStateIdCounterForTests();
|
||||
const registry = new Map();
|
||||
const s1 = registerObservation(registry, { url: "https://a.example/", rawVomText: "a" });
|
||||
|
||||
const { steps } = reduceTraceSteps(
|
||||
[
|
||||
{ op: "fill", target: { ref: "e1", role: "textbox", name: "Search" }, value: "", preStateId: s1, postStateId: s1 },
|
||||
{ op: "click", target: { ref: "e2", role: "button", name: "Apply" }, preStateId: s1, postStateId: s1 },
|
||||
],
|
||||
registry,
|
||||
);
|
||||
|
||||
expect(steps.map((step) => step.op)).toEqual(["fill", "click"]);
|
||||
expect(steps[0]).toMatchObject({ op: "fill", value: "" });
|
||||
});
|
||||
|
||||
it("keeps hover steps before menu clicks", () => {
|
||||
resetStateIdCounterForTests();
|
||||
const registry = new Map();
|
||||
const state = registerObservation(registry, {
|
||||
url: "https://example.com/app",
|
||||
rawVomText: '@e1 button "Account"\n@e2 link "Profile"',
|
||||
});
|
||||
const { steps } = reduceTraceSteps(
|
||||
[
|
||||
{
|
||||
op: "hover",
|
||||
target: { tag: "span", role: "button", name: "Account" },
|
||||
target: { ref: "e1", role: "button", name: "Account" },
|
||||
page_url: "https://example.com/app",
|
||||
preStateId: state,
|
||||
postStateId: state,
|
||||
},
|
||||
{
|
||||
op: "click",
|
||||
target: { tag: "a", role: "link", name: "Profile" },
|
||||
target: { ref: "e2", role: "link", name: "Profile" },
|
||||
page_url: "https://example.com/app",
|
||||
preStateId: state,
|
||||
postStateId: state,
|
||||
},
|
||||
],
|
||||
"https://example.com/app",
|
||||
registry,
|
||||
);
|
||||
expect(steps.map((s) => s.op)).toEqual(["hover", "click"]);
|
||||
expect(steps[0]).toMatchObject({
|
||||
op: "hover",
|
||||
page: "p1",
|
||||
state,
|
||||
result: { state },
|
||||
target: { name: "Account" },
|
||||
});
|
||||
});
|
||||
@@ -178,7 +233,7 @@ describe("reduceTraceSteps", () => {
|
||||
it("resolveTraceStartUrl prefers explicit start URL", () => {
|
||||
expect(
|
||||
resolveTraceStartUrl(
|
||||
[{ op: "navigate", url: "https://example.com/other" }],
|
||||
[{ op: "navigate", url: "https://example.com/other", preStateId: "s1", postStateId: "s1" }],
|
||||
"https://example.com/start",
|
||||
),
|
||||
).toBe("https://example.com/start");
|
||||
|
||||
@@ -313,6 +313,15 @@ export type HandshakeVerdict =
|
||||
| { kind: "version_skew" }
|
||||
| { kind: "rejected"; reason: string };
|
||||
|
||||
const UPDATE_BOTH_VERSION_MESSAGE =
|
||||
"The installed bsk CLI and BrowserSkill extension are incompatible. Update both, reload the extension, then run `bsk daemon restart`.";
|
||||
const UPDATE_BSK_VERSION_MESSAGE =
|
||||
"The installed bsk CLI is out of date for this BrowserSkill extension. Update bsk, then run `bsk daemon restart`.";
|
||||
const UPDATE_EXTENSION_VERSION_MESSAGE =
|
||||
"This BrowserSkill extension is out of date for the installed bsk CLI. Update and reload the extension, then try again.";
|
||||
const INVALID_VERSION_MESSAGE =
|
||||
"BrowserSkill could not verify version compatibility. Update bsk and the BrowserSkill extension, reload the extension, then run `bsk daemon restart`.";
|
||||
|
||||
/**
|
||||
* Symmetric extension-side **protocol** compat check.
|
||||
*
|
||||
@@ -339,19 +348,19 @@ export function computeConnectedState(
|
||||
if (daemonMajor === null || ourMajor === null || daemonMajor !== ourMajor) {
|
||||
return {
|
||||
kind: "rejected",
|
||||
reason: `protocol-major mismatch (daemon protocol v${handshake.protocol_version}, extension protocol v${PROTOCOL_VERSION})`,
|
||||
reason: UPDATE_BOTH_VERSION_MESSAGE,
|
||||
};
|
||||
}
|
||||
if (compareProtocol(handshake.protocol_version, localMinCompatibleProtocol) === null) {
|
||||
return {
|
||||
kind: "rejected",
|
||||
reason: `daemon protocol v${handshake.protocol_version} is unparseable`,
|
||||
reason: INVALID_VERSION_MESSAGE,
|
||||
};
|
||||
}
|
||||
if (compareProtocol(handshake.protocol_version, localMinCompatibleProtocol)! < 0) {
|
||||
return {
|
||||
kind: "rejected",
|
||||
reason: `daemon protocol v${handshake.protocol_version} below extension min_compatible_protocol ${localMinCompatibleProtocol}`,
|
||||
reason: UPDATE_BSK_VERSION_MESSAGE,
|
||||
};
|
||||
}
|
||||
const peerFloor = handshake.min_compatible_protocol;
|
||||
@@ -359,13 +368,13 @@ export function computeConnectedState(
|
||||
if (compareProtocol(PROTOCOL_VERSION, peerFloor) === null) {
|
||||
return {
|
||||
kind: "rejected",
|
||||
reason: `daemon min_compatible_protocol ${peerFloor} is unparseable`,
|
||||
reason: INVALID_VERSION_MESSAGE,
|
||||
};
|
||||
}
|
||||
if (compareProtocol(PROTOCOL_VERSION, peerFloor)! < 0) {
|
||||
return {
|
||||
kind: "rejected",
|
||||
reason: `extension protocol v${PROTOCOL_VERSION} below daemon min_compatible_protocol ${peerFloor}`,
|
||||
reason: UPDATE_EXTENSION_VERSION_MESSAGE,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,8 @@
|
||||
* `{ "tag": "div" }` / “点击div” is useless and must not be recorded.
|
||||
*/
|
||||
|
||||
export interface TargetDescriptor {
|
||||
/** Content-script capture descriptor before VOM geometric matching. */
|
||||
export interface CaptureTargetDescriptor {
|
||||
role?: string;
|
||||
name?: string;
|
||||
tag: string;
|
||||
@@ -247,7 +248,7 @@ export function resolveHoverElement(target: Element): Element | null {
|
||||
* (visible name), or at least a form `name_attr` for checkbox/radio.
|
||||
* Recording “点击div” with no name fails this bar.
|
||||
*/
|
||||
export function isMeaningfulClickTarget(target: TargetDescriptor): boolean {
|
||||
export function isMeaningfulClickTarget(target: CaptureTargetDescriptor): boolean {
|
||||
const name = target.name?.trim();
|
||||
if (name && isActionableLabel(name)) return true;
|
||||
if (
|
||||
@@ -323,7 +324,7 @@ function nearbyLabelText(el: Element): string | undefined {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
export function describeTarget(el: Element): TargetDescriptor {
|
||||
export function describeTarget(el: Element): CaptureTargetDescriptor {
|
||||
const tag = el.tagName.toLowerCase();
|
||||
const role = inferRole(el);
|
||||
const name = accessibleName(el);
|
||||
@@ -350,7 +351,7 @@ export function describeTarget(el: Element): TargetDescriptor {
|
||||
};
|
||||
}
|
||||
|
||||
export function describeEventTarget(target: EventTarget | null): TargetDescriptor | null {
|
||||
export function describeEventTarget(target: EventTarget | null): CaptureTargetDescriptor | null {
|
||||
if (!(target instanceof Element)) return null;
|
||||
const clickable = resolveClickableElement(target);
|
||||
if (!clickable) return null;
|
||||
|
||||
@@ -0,0 +1,62 @@
|
||||
import type { Step } from "@/transport/types";
|
||||
import { OBSERVATION_FILE_VERSION } from "./record-constants";
|
||||
|
||||
export interface ObservationAnnotation {
|
||||
stepId: number;
|
||||
op: Step["op"];
|
||||
line: number;
|
||||
stateId: string;
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
function formatAnnotation({ stepId, op, detail }: ObservationAnnotation): string {
|
||||
const suffix = detail ? `: ${detail}` : "";
|
||||
return ` ⟵ step ${stepId}: ${op}${suffix}`;
|
||||
}
|
||||
|
||||
/** Serialize a page observation file (front matter + VOM body + step annotations). */
|
||||
export function formatObservationFile(input: {
|
||||
stateId: string;
|
||||
url: string;
|
||||
title?: string;
|
||||
stepsHere: number[];
|
||||
body: string;
|
||||
annotations?: ObservationAnnotation[];
|
||||
}): string {
|
||||
const lines: string[] = [
|
||||
`# bsk-observation ${OBSERVATION_FILE_VERSION}`,
|
||||
`state: ${input.stateId}`,
|
||||
`url: ${input.url}`,
|
||||
];
|
||||
if (input.title) lines.push(`title: ${input.title}`);
|
||||
if (input.stepsHere.length > 0) {
|
||||
lines.push(`steps_here: [${input.stepsHere.join(", ")}]`);
|
||||
}
|
||||
lines.push("---");
|
||||
|
||||
const bodyLines = input.body.split("\n");
|
||||
const annotationMap = new Map<number, ObservationAnnotation[]>();
|
||||
for (const ann of input.annotations ?? []) {
|
||||
const bucket = annotationMap.get(ann.line) ?? [];
|
||||
bucket.push(ann);
|
||||
annotationMap.set(ann.line, bucket);
|
||||
}
|
||||
|
||||
for (let i = 0; i < bodyLines.length; i += 1) {
|
||||
let line = bodyLines[i] ?? "";
|
||||
const anns = annotationMap.get(i);
|
||||
if (anns) {
|
||||
for (const ann of anns) {
|
||||
line += formatAnnotation(ann);
|
||||
}
|
||||
}
|
||||
lines.push(line);
|
||||
}
|
||||
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
/** Hash input: VOM body **before** annotations are inserted. */
|
||||
export function observationBodyForHash(vomText: string): string {
|
||||
return vomText;
|
||||
}
|
||||
@@ -16,13 +16,6 @@ export const HELP_ACK = "bsk-help-ack";
|
||||
export const HELP_QUERY = "bsk-help-query";
|
||||
export const HELP_FINISH = "bsk-help-finish";
|
||||
|
||||
export interface HelpHighlightRect {
|
||||
top: number;
|
||||
left: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface HelpRequestMessage {
|
||||
type: typeof HELP_REQUEST;
|
||||
requestId: string;
|
||||
@@ -33,8 +26,6 @@ export interface HelpRequestMessage {
|
||||
displayMode?: "full" | "compact";
|
||||
/** CSS selectors to scroll to + flash-highlight (may be empty). */
|
||||
selectors: string[];
|
||||
/** Top-viewport rectangles for targets that cannot be represented by a root selector. */
|
||||
rects?: HelpHighlightRect[];
|
||||
timeoutMs: number;
|
||||
}
|
||||
|
||||
@@ -83,16 +74,6 @@ export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage {
|
||||
(m.displayMode === undefined || m.displayMode === "full" || m.displayMode === "compact") &&
|
||||
Array.isArray(m.selectors) &&
|
||||
m.selectors.every((selector) => typeof selector === "string") &&
|
||||
(m.rects === undefined ||
|
||||
(Array.isArray(m.rects) &&
|
||||
m.rects.every(
|
||||
(rect) =>
|
||||
typeof rect === "object" &&
|
||||
rect !== null &&
|
||||
["top", "left", "width", "height"].every((key) =>
|
||||
Number.isFinite((rect as Record<string, unknown>)[key]),
|
||||
),
|
||||
))) &&
|
||||
typeof m.timeoutMs === "number"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
import type { RenderedRef } from "@browser-skill/vom";
|
||||
import type { CapturedNode } from "@/tools/vom/capture";
|
||||
import type { CaptureGeometry, TargetDescriptor } from "@/transport/types";
|
||||
import type { CaptureTargetDescriptor } from "./describe-target";
|
||||
import { GEOM_MATCH_TOLERANCE_PX } from "./record-constants";
|
||||
|
||||
export interface MatchTargetInput {
|
||||
geometry: CaptureGeometry;
|
||||
captured: CapturedNode[];
|
||||
refs: RenderedRef[];
|
||||
fallback?: CaptureTargetDescriptor;
|
||||
}
|
||||
|
||||
function withinTolerance(a: number, b: number): boolean {
|
||||
return Math.abs(a - b) <= GEOM_MATCH_TOLERANCE_PX;
|
||||
}
|
||||
|
||||
function rectsMatch(
|
||||
a: { x: number; y: number; w: number; h: number },
|
||||
b: { x: number; y: number; w: number; h: number },
|
||||
): boolean {
|
||||
return (
|
||||
withinTolerance(a.x, b.x) &&
|
||||
withinTolerance(a.y, b.y) &&
|
||||
withinTolerance(a.w, b.w) &&
|
||||
withinTolerance(a.h, b.h)
|
||||
);
|
||||
}
|
||||
|
||||
function nodeViewportRect(
|
||||
node: CapturedNode,
|
||||
): { x: number; y: number; w: number; h: number } | null {
|
||||
return node.localRect ?? node.rect;
|
||||
}
|
||||
|
||||
function matchingRects(
|
||||
geometry: CaptureGeometry,
|
||||
node: CapturedNode,
|
||||
): {
|
||||
target: { x: number; y: number; w: number; h: number };
|
||||
node: { x: number; y: number; w: number; h: number } | null;
|
||||
} {
|
||||
const viewportPosition = geometry.position === "fixed" || geometry.position === "sticky";
|
||||
if (viewportPosition || !node.documentRect) {
|
||||
return { target: geometry.rect, node: nodeViewportRect(node) };
|
||||
}
|
||||
return {
|
||||
target: {
|
||||
x: geometry.rect.x + geometry.scrollX,
|
||||
y: geometry.rect.y + geometry.scrollY,
|
||||
w: geometry.rect.w,
|
||||
h: geometry.rect.h,
|
||||
},
|
||||
node: node.documentRect,
|
||||
};
|
||||
}
|
||||
|
||||
function tagMatches(geometryTag: string, nodeTag: string): boolean {
|
||||
return geometryTag.toLowerCase() === nodeTag.toLowerCase();
|
||||
}
|
||||
|
||||
/** Best description available when the element cannot be located in the VOM. */
|
||||
export function fallbackDescriptor(fallback?: CaptureTargetDescriptor): TargetDescriptor {
|
||||
if (!fallback) {
|
||||
return { unmatched: true };
|
||||
}
|
||||
return {
|
||||
...(fallback.role ? { role: fallback.role } : {}),
|
||||
...(fallback.name ? { name: fallback.name } : {}),
|
||||
unmatched: true,
|
||||
};
|
||||
}
|
||||
|
||||
/** Locate the interacted element in the last settled observation by geometry. */
|
||||
export function matchTarget(input: MatchTargetInput): TargetDescriptor {
|
||||
const ownerFrame = input.geometry.ownerFrameBackendNodeId ?? null;
|
||||
|
||||
const candidates = input.captured.filter((node) => {
|
||||
if (ownerFrame !== (node.ownerFrameBackendNodeId ?? null)) return false;
|
||||
if (!tagMatches(input.geometry.tag, node.tag)) return false;
|
||||
const rects = matchingRects(input.geometry, node);
|
||||
if (!rects.node) return false;
|
||||
return rectsMatch(rects.target, rects.node);
|
||||
});
|
||||
|
||||
if (candidates.length !== 1) {
|
||||
return fallbackDescriptor(input.fallback);
|
||||
}
|
||||
|
||||
const backendNodeId = candidates[0]!.backendNodeId;
|
||||
const refEntry = input.refs.find((r) => r.backendNodeId === backendNodeId);
|
||||
if (!refEntry) {
|
||||
return fallbackDescriptor(input.fallback);
|
||||
}
|
||||
|
||||
return {
|
||||
ref: refEntry.ref,
|
||||
...(refEntry.role ? { role: refEntry.role } : {}),
|
||||
...(refEntry.name ? { name: refEntry.name } : {}),
|
||||
...(refEntry.ctx ? { ctx: refEntry.ctx } : {}),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
// Deciding *when* a page is worth observing.
|
||||
//
|
||||
// A fixed delay cannot serve both a modal that toggles in one frame and a
|
||||
// route change that renders for a second. Instead of guessing, ask the page:
|
||||
// a document that has stopped mutating and is no longer loading is done
|
||||
// reacting to whatever the user just did.
|
||||
|
||||
import type { CdpRunner } from "@/tools/shared";
|
||||
import { SETTLE_MAX_MS, SETTLE_MIN_MS, SETTLE_POLL_MS, SETTLE_QUIET_MS } from "./record-constants";
|
||||
|
||||
/**
|
||||
* Installed once per document. Records only the timestamp of the last DOM
|
||||
* change so each poll stays O(1) no matter how large the page is.
|
||||
*/
|
||||
const QUIET_PROBE = `(() => {
|
||||
const scope = window;
|
||||
let probe = scope.__bskRecordQuiet;
|
||||
if (!probe) {
|
||||
probe = { changedAt: Date.now() };
|
||||
const observer = new MutationObserver(() => {
|
||||
probe.changedAt = Date.now();
|
||||
});
|
||||
observer.observe(document.documentElement, {
|
||||
subtree: true,
|
||||
childList: true,
|
||||
attributes: true,
|
||||
characterData: true,
|
||||
});
|
||||
scope.__bskRecordQuiet = probe;
|
||||
}
|
||||
return { idleMs: Date.now() - probe.changedAt, readyState: document.readyState };
|
||||
})()`;
|
||||
|
||||
export type SettleOutcome =
|
||||
/** The page stopped changing on its own. */
|
||||
| "quiet"
|
||||
/** Still changing when the budget ran out; observe it as it is. */
|
||||
| "timeout"
|
||||
/** A newer action took over; this observation is no longer wanted. */
|
||||
| "cancelled";
|
||||
|
||||
interface QuietProbe {
|
||||
idleMs: number;
|
||||
readyState: string;
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function readQuietProbe(cdp: CdpRunner, tabId: number): Promise<QuietProbe | null> {
|
||||
try {
|
||||
const reply = await cdp.send<{ result?: { value?: unknown } }>(tabId, "Runtime.evaluate", {
|
||||
expression: QUIET_PROBE,
|
||||
returnByValue: true,
|
||||
});
|
||||
const value = reply.result?.value;
|
||||
if (!value || typeof value !== "object") return null;
|
||||
const { idleMs, readyState } = value as { idleMs?: unknown; readyState?: unknown };
|
||||
if (typeof idleMs !== "number" || typeof readyState !== "string") return null;
|
||||
return { idleMs, readyState };
|
||||
} catch {
|
||||
// An unreadable page is a page mid-swap; that is a reason to keep waiting,
|
||||
// not a reason to give up.
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/** Wait until the page has finished reacting, or until the budget runs out. */
|
||||
export async function waitForPageSettled(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
options: { cancelled?: () => boolean } = {},
|
||||
): Promise<SettleOutcome> {
|
||||
const startedAt = Date.now();
|
||||
const floor = startedAt + SETTLE_MIN_MS;
|
||||
const deadline = startedAt + SETTLE_MAX_MS;
|
||||
|
||||
for (;;) {
|
||||
if (options.cancelled?.()) return "cancelled";
|
||||
await sleep(SETTLE_POLL_MS);
|
||||
if (options.cancelled?.()) return "cancelled";
|
||||
|
||||
const probe = await readQuietProbe(cdp, tabId);
|
||||
const now = Date.now();
|
||||
if (now < floor) continue;
|
||||
if (now >= deadline) return "timeout";
|
||||
if (!probe) continue;
|
||||
if (probe.readyState === "loading") continue;
|
||||
if (probe.idleMs >= SETTLE_QUIET_MS) return "quiet";
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,8 @@
|
||||
* service worker and a tab's content script.
|
||||
*/
|
||||
|
||||
import type { TargetDescriptor } from "./describe-target";
|
||||
import type { CaptureTargetDescriptor } from "./describe-target";
|
||||
import type { FrameCaptureFailure, FrameCaptureStatus } from "@/transport/types";
|
||||
|
||||
export const RECORD_START = "bsk-record-start";
|
||||
export const RECORD_STEP = "bsk-record-step";
|
||||
@@ -11,6 +12,7 @@ export const RECORD_STOP = "bsk-record-stop";
|
||||
export const RECORD_CANCEL = "bsk-record-cancel";
|
||||
export const RECORD_FINISH = "bsk-record-finish";
|
||||
export const RECORD_QUERY = "bsk-record-query";
|
||||
export const RECORD_CAPTURE_STATUS = "bsk-record-capture-status";
|
||||
|
||||
export interface RecordStartAck {
|
||||
ok: true;
|
||||
@@ -27,8 +29,12 @@ export interface RecordQueryResponse {
|
||||
requestId?: string;
|
||||
/** Epoch ms when the recording began; see `RecordStartMessage.startedAtMs`. */
|
||||
startedAtMs?: number;
|
||||
captureStatus?: FrameCaptureStatus;
|
||||
captureFailures?: FrameCaptureFailure[];
|
||||
}
|
||||
|
||||
export type RecordFrameMode = "top" | "child";
|
||||
|
||||
export interface RecordStartMessage {
|
||||
type: typeof RECORD_START;
|
||||
requestId: string;
|
||||
@@ -38,11 +44,15 @@ export interface RecordStartMessage {
|
||||
* content-script remounts instead of restarting per page.
|
||||
*/
|
||||
startedAtMs?: number;
|
||||
/** When true, the top-frame overlay should mount. Child frames omit this. */
|
||||
showOverlay?: boolean;
|
||||
/** Whether this document runs top-frame or child-frame capture semantics. */
|
||||
frameMode?: RecordFrameMode;
|
||||
}
|
||||
|
||||
export interface RecordStepPayload {
|
||||
op: "click" | "hover" | "fill" | "press" | "select" | "navigate";
|
||||
target?: TargetDescriptor;
|
||||
op: "click" | "hover" | "fill" | "press" | "select" | "navigate" | "scroll";
|
||||
target?: CaptureTargetDescriptor;
|
||||
value?: string;
|
||||
key?: string;
|
||||
modifiers?: Array<"alt" | "ctrl" | "meta" | "shift">;
|
||||
@@ -50,12 +60,31 @@ export interface RecordStepPayload {
|
||||
labels?: string[];
|
||||
url?: string;
|
||||
redacted?: boolean;
|
||||
/** Page URL when the step was captured. */
|
||||
commit?: "enter" | "suggestion" | "blur";
|
||||
/** Top-level tab URL when the step was captured. */
|
||||
page_url?: string;
|
||||
/** URL of the frame document where the interaction occurred. */
|
||||
frame_url?: string;
|
||||
/** Ephemeral token stamped on the active document root; not persisted in trace steps. */
|
||||
documentToken?: string;
|
||||
/** Epoch ms when the step was captured in the content script. */
|
||||
capturedAtMs?: number;
|
||||
/** Document geometry at capture time for background geometric matching. */
|
||||
geometry?: {
|
||||
rect: { x: number; y: number; w: number; h: number };
|
||||
scrollX: number;
|
||||
scrollY: number;
|
||||
position: string;
|
||||
tag: string;
|
||||
ownerFrameBackendNodeId?: number | null;
|
||||
};
|
||||
/** Capture-only hint; never persisted unless converted to navigated_to. */
|
||||
expects_navigation?: boolean;
|
||||
/** Whether an observed URL change was synchronously caused by the action. */
|
||||
navigation_caused_by_action?: boolean;
|
||||
/** Raw webNavigation transition metadata for cause mapping. */
|
||||
transitionType?: string;
|
||||
transitionQualifiers?: string[];
|
||||
}
|
||||
|
||||
export interface RecordStepMessage {
|
||||
@@ -79,6 +108,12 @@ export interface RecordFinishMessage {
|
||||
requestId: string;
|
||||
}
|
||||
|
||||
export interface RecordCaptureStatusMessage {
|
||||
type: typeof RECORD_CAPTURE_STATUS;
|
||||
status: FrameCaptureStatus;
|
||||
failures?: FrameCaptureFailure[];
|
||||
}
|
||||
|
||||
export function isRecordStartMessage(msg: unknown): msg is RecordStartMessage {
|
||||
if (typeof msg !== "object" || msg === null) return false;
|
||||
const m = msg as Record<string, unknown>;
|
||||
@@ -109,6 +144,16 @@ export function isRecordQueryMessage(msg: unknown): msg is RecordQueryMessage {
|
||||
return m.type === RECORD_QUERY;
|
||||
}
|
||||
|
||||
export function isRecordCaptureStatusMessage(msg: unknown): msg is RecordCaptureStatusMessage {
|
||||
if (typeof msg !== "object" || msg === null) return false;
|
||||
const m = msg as Record<string, unknown>;
|
||||
return (
|
||||
m.type === RECORD_CAPTURE_STATUS &&
|
||||
(m.status === "complete" || m.status === "partial") &&
|
||||
(m.failures === undefined || Array.isArray(m.failures))
|
||||
);
|
||||
}
|
||||
|
||||
export function isRecordStepMessage(msg: unknown): msg is RecordStepMessage {
|
||||
if (typeof msg !== "object" || msg === null) return false;
|
||||
const m = msg as Record<string, unknown>;
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
import type { FillCommit, NavigationCause } from "@/transport/types";
|
||||
|
||||
/**
|
||||
* Floor before a post-action observation may be taken. A page usually has not
|
||||
* started reacting in the first frames after an action, and capturing then
|
||||
* would record the page as it was, not as the action left it.
|
||||
*/
|
||||
export const SETTLE_MIN_MS = 150;
|
||||
|
||||
/** How long the DOM must stop changing before a page counts as settled. */
|
||||
export const SETTLE_QUIET_MS = 250;
|
||||
|
||||
/** Upper bound on waiting for a page to settle; animations never stop. */
|
||||
export const SETTLE_MAX_MS = 2_000;
|
||||
|
||||
/** How often to ask the page whether it has gone quiet. */
|
||||
export const SETTLE_POLL_MS = 60;
|
||||
|
||||
/** Minimum interval between consecutive observations on the same tab. */
|
||||
export const OBSERVATION_MIN_INTERVAL_MS = 200;
|
||||
|
||||
/** Delay before retrying a capture that lost its execution context. */
|
||||
export const CAPTURE_RETRY_DELAY_MS = 250;
|
||||
|
||||
/** Document-coordinate matching tolerance in CSS pixels. */
|
||||
export const GEOM_MATCH_TOLERANCE_PX = 2;
|
||||
|
||||
/** Default max tokens per page observation when CLI omits `--max-page-tokens`. */
|
||||
export const DEFAULT_MAX_PAGE_TOKENS = 3000;
|
||||
|
||||
/** VOM observation file format version (front matter header). */
|
||||
export const OBSERVATION_FILE_VERSION = 1;
|
||||
|
||||
/** FNV-1a 64-bit offset basis. */
|
||||
const FNV_OFFSET = 0xcbf29ce484222325n;
|
||||
/** FNV-1a 64-bit prime. */
|
||||
const FNV_PRIME = 0x100000001b3n;
|
||||
|
||||
/** Content-hash for state deduplication (sync, non-cryptographic). */
|
||||
export function fnv1a64(text: string): string {
|
||||
let hash = FNV_OFFSET;
|
||||
for (let i = 0; i < text.length; i += 1) {
|
||||
hash ^= BigInt(text.charCodeAt(i));
|
||||
hash = (hash * FNV_PRIME) & 0xffffffffffffffffn;
|
||||
}
|
||||
return hash.toString(16).padStart(16, "0");
|
||||
}
|
||||
|
||||
let nextStateSerial = 0;
|
||||
|
||||
/** Monotonic state id generator (`s1`, `s2`, …). */
|
||||
export function nextStateId(): string {
|
||||
nextStateSerial += 1;
|
||||
return `s${nextStateSerial}`;
|
||||
}
|
||||
|
||||
/** Test seam: reset the state id counter. */
|
||||
export function resetStateIdCounterForTests(): void {
|
||||
nextStateSerial = 0;
|
||||
}
|
||||
|
||||
const REDIRECT_QUALIFIERS = new Set(["client_redirect", "server_redirect"]);
|
||||
|
||||
const TRANSITION_TO_CAUSE: Record<string, NavigationCause> = {
|
||||
typed: "user_typed",
|
||||
generated: "user_typed",
|
||||
keyword: "user_typed",
|
||||
keyword_generated: "user_typed",
|
||||
link: "link",
|
||||
form_submit: "form_submit",
|
||||
reload: "reload",
|
||||
auto_bookmark: "browser",
|
||||
start_page: "browser",
|
||||
};
|
||||
|
||||
export interface NavigationTransitionMeta {
|
||||
transitionType?: string;
|
||||
transitionQualifiers?: string[];
|
||||
navigationActionPending?: boolean;
|
||||
}
|
||||
|
||||
/** Map webNavigation metadata to protocol `NavigationCause`. Returns null for redirects. */
|
||||
export function mapNavigationCause(meta: NavigationTransitionMeta): NavigationCause | null {
|
||||
const qualifiers = meta.transitionQualifiers ?? [];
|
||||
if (qualifiers.includes("forward_back")) return "history";
|
||||
if (qualifiers.some((q) => REDIRECT_QUALIFIERS.has(q))) return null;
|
||||
if (qualifiers.includes("from_address_bar")) return "user_typed";
|
||||
|
||||
const type = meta.transitionType ?? "";
|
||||
const mapped = TRANSITION_TO_CAUSE[type];
|
||||
if (mapped) return mapped;
|
||||
|
||||
if (!type && meta.navigationActionPending === false) return "script";
|
||||
return "browser";
|
||||
}
|
||||
|
||||
export const DEFAULT_FILL_COMMIT: FillCommit = "blur";
|
||||
@@ -0,0 +1,24 @@
|
||||
/** Attribute written on `<html>` while a document is actively recording. */
|
||||
export const RECORD_DOCUMENT_TOKEN_ATTR = "data-bsk-record-document";
|
||||
|
||||
export function createRecordDocumentToken(): string {
|
||||
if (typeof crypto !== "undefined" && typeof crypto.randomUUID === "function") {
|
||||
return crypto.randomUUID();
|
||||
}
|
||||
return `doc-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
|
||||
}
|
||||
|
||||
export function stampRecordDocumentToken(token: string): void {
|
||||
document.documentElement.setAttribute(RECORD_DOCUMENT_TOKEN_ATTR, token);
|
||||
}
|
||||
|
||||
export function clearRecordDocumentToken(): void {
|
||||
document.documentElement.removeAttribute(RECORD_DOCUMENT_TOKEN_ATTR);
|
||||
}
|
||||
|
||||
export function readRecordDocumentTokenFromAttrs(
|
||||
attrs: Record<string, string> | undefined,
|
||||
): string | undefined {
|
||||
const token = attrs?.[RECORD_DOCUMENT_TOKEN_ATTR];
|
||||
return token?.trim() ? token : undefined;
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
import {
|
||||
RECORD_CANCEL,
|
||||
RECORD_START,
|
||||
RECORD_STOP,
|
||||
type RecordCancelMessage,
|
||||
type RecordStartAck,
|
||||
type RecordStartMessage,
|
||||
type RecordStopAck,
|
||||
type RecordStopMessage,
|
||||
} from "@/lib/record-bridge";
|
||||
import type { FrameCaptureFailure, FrameCaptureFailureReason } from "@/transport/types";
|
||||
|
||||
export interface WebNavigationFrameInfo {
|
||||
frameId: number;
|
||||
documentId?: string;
|
||||
url?: string;
|
||||
parentFrameId: number;
|
||||
}
|
||||
|
||||
export interface ArmedDocument {
|
||||
tabId: number;
|
||||
frameId: number;
|
||||
documentId: string;
|
||||
url?: string;
|
||||
parentFrameId: number;
|
||||
}
|
||||
|
||||
export interface RecordFrameSendDeps {
|
||||
sendToDocument(
|
||||
tabId: number,
|
||||
documentId: string,
|
||||
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
|
||||
): Promise<unknown>;
|
||||
getAllFrames(tabId: number): Promise<WebNavigationFrameInfo[]>;
|
||||
}
|
||||
|
||||
const RECORD_START_RETRIES = 3;
|
||||
const RECORD_START_RETRY_DELAY_MS = 500;
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
export function isFrameUrlRestricted(url: string | undefined): boolean {
|
||||
if (!url) return false;
|
||||
const lower = url.toLowerCase();
|
||||
return (
|
||||
lower.startsWith("chrome://") ||
|
||||
lower.startsWith("chrome-extension://") ||
|
||||
lower.startsWith("edge://") ||
|
||||
lower.startsWith("devtools://") ||
|
||||
lower.startsWith("devtools:") ||
|
||||
lower.startsWith("https://chrome.google.com/webstore")
|
||||
);
|
||||
}
|
||||
|
||||
function isRecordStartAck(response: unknown): response is RecordStartAck {
|
||||
return (
|
||||
typeof response === "object" &&
|
||||
response !== null &&
|
||||
"ok" in response &&
|
||||
(response as RecordStartAck).ok === true
|
||||
);
|
||||
}
|
||||
|
||||
function isRecordStopAckOk(response: unknown): boolean {
|
||||
return (
|
||||
typeof response === "object" &&
|
||||
response !== null &&
|
||||
"ok" in response &&
|
||||
(response as RecordStopAck).ok === true
|
||||
);
|
||||
}
|
||||
|
||||
export function makeFrameCaptureFailure(
|
||||
reason: FrameCaptureFailureReason,
|
||||
input: { frameId: number; documentId?: string; url?: string; detail?: string },
|
||||
): FrameCaptureFailure {
|
||||
return {
|
||||
reason,
|
||||
frame_id: input.frameId,
|
||||
...(input.documentId ? { document_id: input.documentId } : {}),
|
||||
...(input.url ? { url: input.url } : {}),
|
||||
...(input.detail ? { detail: input.detail } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function listInjectableFrames(
|
||||
tabId: number,
|
||||
deps: RecordFrameSendDeps,
|
||||
): Promise<{ injectable: WebNavigationFrameInfo[]; failures: FrameCaptureFailure[] }> {
|
||||
const failures: FrameCaptureFailure[] = [];
|
||||
let frames: WebNavigationFrameInfo[] = [];
|
||||
try {
|
||||
frames = await deps.getAllFrames(tabId);
|
||||
} catch (err) {
|
||||
failures.push(
|
||||
makeFrameCaptureFailure("arm_failed", {
|
||||
frameId: 0,
|
||||
detail: err instanceof Error ? err.message : "webNavigation.getAllFrames failed",
|
||||
}),
|
||||
);
|
||||
return { injectable: [], failures };
|
||||
}
|
||||
|
||||
const injectable = frames.filter((frame) => {
|
||||
if (!frame.documentId) {
|
||||
failures.push(
|
||||
makeFrameCaptureFailure("not_injectable", {
|
||||
frameId: frame.frameId,
|
||||
url: frame.url,
|
||||
detail: "missing documentId",
|
||||
}),
|
||||
);
|
||||
return false;
|
||||
}
|
||||
if (isFrameUrlRestricted(frame.url)) {
|
||||
failures.push(
|
||||
makeFrameCaptureFailure("not_injectable", {
|
||||
frameId: frame.frameId,
|
||||
documentId: frame.documentId,
|
||||
url: frame.url,
|
||||
}),
|
||||
);
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
});
|
||||
return { injectable, failures };
|
||||
}
|
||||
|
||||
export async function sendRecordStartWithAckToDocument(
|
||||
tabId: number,
|
||||
documentId: string,
|
||||
msg: RecordStartMessage,
|
||||
deps: RecordFrameSendDeps,
|
||||
): Promise<void> {
|
||||
let lastError: unknown;
|
||||
for (let attempt = 0; attempt < RECORD_START_RETRIES; attempt += 1) {
|
||||
try {
|
||||
const response = await deps.sendToDocument(tabId, documentId, msg);
|
||||
if (isRecordStartAck(response)) return;
|
||||
lastError = new Error("content script did not ack RECORD_START");
|
||||
} catch (err) {
|
||||
lastError = err;
|
||||
}
|
||||
if (attempt + 1 < RECORD_START_RETRIES) {
|
||||
await sleep(RECORD_START_RETRY_DELAY_MS);
|
||||
}
|
||||
}
|
||||
throw lastError ?? new Error("failed to start recording in content script");
|
||||
}
|
||||
|
||||
export async function armDocumentCapture(
|
||||
tabId: number,
|
||||
frame: WebNavigationFrameInfo,
|
||||
startMsg: RecordStartMessage,
|
||||
deps: RecordFrameSendDeps,
|
||||
): Promise<{ armed?: ArmedDocument; failure?: FrameCaptureFailure }> {
|
||||
if (!frame.documentId) {
|
||||
return {
|
||||
failure: makeFrameCaptureFailure("not_injectable", {
|
||||
frameId: frame.frameId,
|
||||
url: frame.url,
|
||||
detail: "missing documentId",
|
||||
}),
|
||||
};
|
||||
}
|
||||
if (isFrameUrlRestricted(frame.url)) {
|
||||
return {
|
||||
failure: makeFrameCaptureFailure("not_injectable", {
|
||||
frameId: frame.frameId,
|
||||
documentId: frame.documentId,
|
||||
url: frame.url,
|
||||
}),
|
||||
};
|
||||
}
|
||||
try {
|
||||
await sendRecordStartWithAckToDocument(tabId, frame.documentId, startMsg, deps);
|
||||
return {
|
||||
armed: {
|
||||
tabId,
|
||||
frameId: frame.frameId,
|
||||
documentId: frame.documentId,
|
||||
url: frame.url,
|
||||
parentFrameId: frame.parentFrameId,
|
||||
},
|
||||
};
|
||||
} catch (err) {
|
||||
return {
|
||||
failure: makeFrameCaptureFailure("arm_failed", {
|
||||
frameId: frame.frameId,
|
||||
documentId: frame.documentId,
|
||||
url: frame.url,
|
||||
detail: err instanceof Error ? err.message : "RECORD_START failed",
|
||||
}),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export async function armAllDocumentsForTab(
|
||||
tabId: number,
|
||||
startMsg: RecordStartMessage,
|
||||
deps: RecordFrameSendDeps,
|
||||
): Promise<{ armed: ArmedDocument[]; failures: FrameCaptureFailure[] }> {
|
||||
const { injectable, failures } = await listInjectableFrames(tabId, deps);
|
||||
const armed: ArmedDocument[] = [];
|
||||
for (const frame of injectable) {
|
||||
const result = await armDocumentCapture(tabId, frame, startMsg, deps);
|
||||
if (result.armed) armed.push(result.armed);
|
||||
if (result.failure) failures.push(result.failure);
|
||||
}
|
||||
return { armed, failures };
|
||||
}
|
||||
|
||||
export async function stopDocumentCapture(
|
||||
armed: ArmedDocument,
|
||||
stopMsg: RecordStopMessage,
|
||||
deps: RecordFrameSendDeps,
|
||||
): Promise<{ ok: true } | FrameCaptureFailure> {
|
||||
try {
|
||||
const response = await deps.sendToDocument(armed.tabId, armed.documentId, stopMsg);
|
||||
if (isRecordStopAckOk(response)) return { ok: true };
|
||||
return makeFrameCaptureFailure("flush_failed", {
|
||||
frameId: armed.frameId,
|
||||
documentId: armed.documentId,
|
||||
url: armed.url,
|
||||
detail: "content script did not confirm recorded steps",
|
||||
});
|
||||
} catch (err) {
|
||||
return makeFrameCaptureFailure("flush_failed", {
|
||||
frameId: armed.frameId,
|
||||
documentId: armed.documentId,
|
||||
url: armed.url,
|
||||
detail: err instanceof Error ? err.message : "RECORD_STOP failed",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function cancelDocumentCapture(
|
||||
armed: ArmedDocument,
|
||||
cancelMsg: RecordCancelMessage,
|
||||
deps: RecordFrameSendDeps,
|
||||
): Promise<void> {
|
||||
try {
|
||||
await deps.sendToDocument(armed.tabId, armed.documentId, cancelMsg);
|
||||
} catch {
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,673 @@
|
||||
import type { RenderedRef } from "@browser-skill/vom";
|
||||
import { formatObservationFile, type ObservationAnnotation } from "@/lib/format-observation-file";
|
||||
import { fallbackDescriptor, matchTarget } from "@/lib/match-target";
|
||||
import { waitForPageSettled } from "@/lib/page-settled";
|
||||
import {
|
||||
CAPTURE_RETRY_DELAY_MS,
|
||||
DEFAULT_MAX_PAGE_TOKENS,
|
||||
OBSERVATION_MIN_INTERVAL_MS,
|
||||
} from "@/lib/record-constants";
|
||||
import { registerObservation, type StateRegistryEntry } from "@/lib/trace-reducer";
|
||||
import { captureVomObservation } from "@/tools/capture-vom-observation";
|
||||
import type { CdpRunner, ChromeTabsApi } from "@/tools/shared";
|
||||
import type { CapturedNode } from "@/tools/vom/capture";
|
||||
import type { DraftTraceStep, Step, StopReason, Trace, TraceState } from "@/transport/types";
|
||||
import { reduceTraceSteps, resolveTraceStartUrl } from "./trace-reducer";
|
||||
|
||||
export interface LastSettledObservation {
|
||||
stateId: string;
|
||||
captured: CapturedNode[];
|
||||
matchNodes: CapturedNode[];
|
||||
documentTokenOwners: Map<string, number | null>;
|
||||
refs: RenderedRef[];
|
||||
url: string;
|
||||
title?: string;
|
||||
vomText: string;
|
||||
}
|
||||
|
||||
/** A post-action observation that has been queued but not yet written down. */
|
||||
interface PendingSettle {
|
||||
draftIndex: number;
|
||||
/** Set when a newer action has already decided where this step landed. */
|
||||
cancelled: boolean;
|
||||
}
|
||||
|
||||
/** Latest URL in an in-flight redirect chain awaiting coalesce + settle. */
|
||||
export interface PendingRedirectLanding {
|
||||
url: string;
|
||||
/** Bumped on every hop so an in-flight `waitForPageSettled` can cancel. */
|
||||
generation: number;
|
||||
}
|
||||
|
||||
export interface RecordingObservationState {
|
||||
stateRegistry: Map<string, StateRegistryEntry>;
|
||||
lastSettled: LastSettledObservation | null;
|
||||
/** Latest token→owner map from the most recent observation capture. */
|
||||
documentTokenOwners: Map<string, number | null>;
|
||||
maxPageTokens: number;
|
||||
redactValues: boolean;
|
||||
lastCaptureAtMs: number;
|
||||
/**
|
||||
* Tail of the settle chain. Observations run one at a time so that a slow
|
||||
* capture can never land after a faster one and report the pages out of the
|
||||
* order the user visited them.
|
||||
*/
|
||||
settleQueue: Promise<void>;
|
||||
/** Queued and in-flight settles, by draft index, so newer actions can supersede them. */
|
||||
settles: Map<number, PendingSettle>;
|
||||
stepAnnotations: Map<number, ObservationAnnotation[]>;
|
||||
/**
|
||||
* OAuth / server redirects are coalesced here: intermediate hops only update
|
||||
* `url`+`generation`, and one `navigate` is emitted after the page settles.
|
||||
*/
|
||||
pendingRedirect: PendingRedirectLanding | null;
|
||||
redirectFlushQueue: Promise<void>;
|
||||
}
|
||||
|
||||
export function createObservationState(options?: {
|
||||
maxPageTokens?: number;
|
||||
redactValues?: boolean;
|
||||
}): RecordingObservationState {
|
||||
return {
|
||||
stateRegistry: new Map(),
|
||||
lastSettled: null,
|
||||
documentTokenOwners: new Map(),
|
||||
maxPageTokens: options?.maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS,
|
||||
redactValues: options?.redactValues ?? false,
|
||||
lastCaptureAtMs: 0,
|
||||
settleQueue: Promise.resolve(),
|
||||
settles: new Map(),
|
||||
stepAnnotations: new Map(),
|
||||
pendingRedirect: null,
|
||||
redirectFlushQueue: Promise.resolve(),
|
||||
};
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function readTabMeta(
|
||||
tabsApi: ChromeTabsApi,
|
||||
tabId: number,
|
||||
): Promise<{ url: string; title?: string }> {
|
||||
try {
|
||||
const tab = await tabsApi.get(tabId);
|
||||
return { url: tab.url ?? "about:blank", title: tab.title };
|
||||
} catch {
|
||||
return { url: "about:blank" };
|
||||
}
|
||||
}
|
||||
|
||||
export async function captureAndRegisterObservation(
|
||||
obs: RecordingObservationState,
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
urlOverride?: string,
|
||||
): Promise<LastSettledObservation> {
|
||||
const now = Date.now();
|
||||
const waitMs = Math.max(0, OBSERVATION_MIN_INTERVAL_MS - (now - obs.lastCaptureAtMs));
|
||||
if (waitMs > 0) await sleep(waitMs);
|
||||
|
||||
const { url, title } = urlOverride
|
||||
? { url: urlOverride, title: undefined }
|
||||
: await readTabMeta(tabsApi, tabId);
|
||||
const meta = urlOverride ? await readTabMeta(tabsApi, tabId) : { url, title };
|
||||
const resolvedTitle = title ?? meta.title;
|
||||
|
||||
const rendered = await captureVomObservation(cdp, tabId, url, {
|
||||
maxTokens: obs.maxPageTokens,
|
||||
redactValues: obs.redactValues,
|
||||
conditionalSurfaceProbe: false,
|
||||
});
|
||||
|
||||
const stateId = registerObservation(obs.stateRegistry, {
|
||||
url,
|
||||
title: resolvedTitle,
|
||||
rawVomText: rendered.text,
|
||||
truncated: rendered.truncated,
|
||||
});
|
||||
|
||||
obs.lastCaptureAtMs = Date.now();
|
||||
obs.documentTokenOwners = rendered.documentTokenOwners;
|
||||
const settled: LastSettledObservation = {
|
||||
stateId,
|
||||
captured: rendered.captured,
|
||||
matchNodes: rendered.matchNodes,
|
||||
documentTokenOwners: rendered.documentTokenOwners,
|
||||
refs: rendered.refs,
|
||||
url,
|
||||
title: resolvedTitle,
|
||||
vomText: rendered.text,
|
||||
};
|
||||
obs.lastSettled = settled;
|
||||
return settled;
|
||||
}
|
||||
|
||||
/**
|
||||
* Bind a draft to the page it was performed on: origin state, geometric target
|
||||
* match, and the inline annotation. All three read the *current* observation,
|
||||
* which is only the page the user acted on while the draft is still fresh —
|
||||
* once the action settles, `lastSettled` is the destination page instead.
|
||||
*/
|
||||
export function applyTargetMatching(
|
||||
obs: RecordingObservationState,
|
||||
draft: DraftTraceStep,
|
||||
stepId: number,
|
||||
): void {
|
||||
if (!obs.lastSettled) {
|
||||
// Initial observation can fail or race the first action. Keep whatever
|
||||
// the content script captured so the exported step is still teachable.
|
||||
if ("target" in draft && "captureTarget" in draft) {
|
||||
draft.target = fallbackDescriptor(draft.captureTarget);
|
||||
}
|
||||
rememberStepAnnotation(obs, stepId, draft);
|
||||
return;
|
||||
}
|
||||
draft.preStateId = obs.lastSettled.stateId;
|
||||
|
||||
if ("target" in draft) {
|
||||
draft.target =
|
||||
"geometry" in draft && draft.geometry
|
||||
? matchTarget({
|
||||
geometry: draft.geometry,
|
||||
captured: obs.lastSettled.matchNodes,
|
||||
refs: obs.lastSettled.refs,
|
||||
fallback: draft.captureTarget,
|
||||
})
|
||||
: // No geometry to match on, but the capture still knew what the user
|
||||
// touched — keep that instead of an anonymous unmatched target.
|
||||
fallbackDescriptor(draft.captureTarget);
|
||||
}
|
||||
rememberStepAnnotation(obs, stepId, draft);
|
||||
}
|
||||
|
||||
function fillDetailForDraft(
|
||||
obs: RecordingObservationState,
|
||||
draft: DraftTraceStep,
|
||||
): string | undefined {
|
||||
if (obs.redactValues) return undefined;
|
||||
if (draft.op === "fill") return JSON.stringify(draft.value);
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function refLineForDraft(
|
||||
obs: RecordingObservationState,
|
||||
draft: DraftTraceStep,
|
||||
): number | undefined {
|
||||
if (!("target" in draft) || !obs.lastSettled) return undefined;
|
||||
const targetRef = draft.target?.ref;
|
||||
if (!targetRef) return undefined;
|
||||
return obs.lastSettled.refs.find((r) => r.ref === targetRef)?.line;
|
||||
}
|
||||
|
||||
function rememberStepAnnotation(
|
||||
obs: RecordingObservationState,
|
||||
stepId: number,
|
||||
draft: DraftTraceStep,
|
||||
): void {
|
||||
const line = refLineForDraft(obs, draft);
|
||||
if (line === undefined || draft.op === "navigate" || draft.op === "scroll") return;
|
||||
if (!draft.preStateId) return;
|
||||
const ann: ObservationAnnotation = {
|
||||
stepId,
|
||||
op: draft.op,
|
||||
line,
|
||||
stateId: draft.preStateId,
|
||||
detail: fillDetailForDraft(obs, draft),
|
||||
};
|
||||
const bucket = obs.stepAnnotations.get(line) ?? [];
|
||||
bucket.push(ann);
|
||||
obs.stepAnnotations.set(line, bucket);
|
||||
}
|
||||
|
||||
/** Record the step under the page it was performed on, not the one it led to. */
|
||||
export function rememberStepOnPage(
|
||||
obs: RecordingObservationState,
|
||||
draft: DraftTraceStep,
|
||||
stepId: number,
|
||||
): void {
|
||||
const stateId = draft.preStateId;
|
||||
if (!stateId) return;
|
||||
const entry = obs.stateRegistry.get(stateId);
|
||||
if (entry && !entry.stepsHere.includes(stepId)) {
|
||||
entry.stepsHere.push(stepId);
|
||||
}
|
||||
}
|
||||
|
||||
/** Drop an in-flight redirect coalesce (e.g. a real navigate superseded it). */
|
||||
export function clearPendingRedirectLanding(obs: RecordingObservationState): void {
|
||||
obs.pendingRedirect = null;
|
||||
}
|
||||
|
||||
function enqueueRedirectFlush(obs: RecordingObservationState, task: () => Promise<void>): void {
|
||||
obs.redirectFlushQueue = obs.redirectFlushQueue.then(task, task).catch(() => {});
|
||||
}
|
||||
|
||||
/**
|
||||
* Remember a redirect hop and schedule a settle-then-emit of a single navigate
|
||||
* to the final URL. Later hops only bump `generation` / replace `url`.
|
||||
*/
|
||||
export function scheduleRedirectLandingFlush(
|
||||
obs: RecordingObservationState,
|
||||
steps: DraftTraceStep[],
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
url: string,
|
||||
options: { cancelled?: () => boolean } = {},
|
||||
): void {
|
||||
const generation = (obs.pendingRedirect?.generation ?? 0) + 1;
|
||||
obs.pendingRedirect = { url, generation };
|
||||
enqueueRedirectFlush(obs, () =>
|
||||
coalesceRedirectLanding(obs, steps, cdp, tabId, tabsApi, options),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Drain any coalesced redirect so the next action matches against the real
|
||||
* landing page. Safe to call when nothing is pending.
|
||||
*/
|
||||
export async function flushPendingRedirectLanding(
|
||||
obs: RecordingObservationState,
|
||||
steps: DraftTraceStep[],
|
||||
cdp: CdpRunner | undefined,
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
options: { cancelled?: () => boolean } = {},
|
||||
): Promise<void> {
|
||||
if (obs.pendingRedirect && cdp) {
|
||||
enqueueRedirectFlush(obs, () =>
|
||||
coalesceRedirectLanding(obs, steps, cdp, tabId, tabsApi, options),
|
||||
);
|
||||
}
|
||||
await obs.redirectFlushQueue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait until the redirect chain stops changing the document, then emit one
|
||||
* `navigate` (cause `browser`) to the tab's final URL and settle its observation.
|
||||
*/
|
||||
async function coalesceRedirectLanding(
|
||||
obs: RecordingObservationState,
|
||||
steps: DraftTraceStep[],
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
options: { cancelled?: () => boolean },
|
||||
): Promise<void> {
|
||||
while (obs.pendingRedirect) {
|
||||
if (options.cancelled?.()) {
|
||||
obs.pendingRedirect = null;
|
||||
return;
|
||||
}
|
||||
|
||||
const snap = obs.pendingRedirect;
|
||||
const outcome = await waitForPageSettled(cdp, tabId, {
|
||||
cancelled: () =>
|
||||
!!options.cancelled?.() ||
|
||||
obs.pendingRedirect === null ||
|
||||
obs.pendingRedirect.generation !== snap.generation,
|
||||
});
|
||||
|
||||
if (options.cancelled?.()) {
|
||||
obs.pendingRedirect = null;
|
||||
return;
|
||||
}
|
||||
// A newer hop cancelled this wait — loop and settle against the latest URL.
|
||||
if (outcome === "cancelled") continue;
|
||||
if (!obs.pendingRedirect || obs.pendingRedirect.generation !== snap.generation) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const finalUrl = (await readTabMeta(tabsApi, tabId)).url || snap.url;
|
||||
obs.pendingRedirect = null;
|
||||
|
||||
if (!finalUrl || finalUrl === "about:blank") return;
|
||||
if (obs.lastSettled?.url === finalUrl) return;
|
||||
|
||||
const last = steps[steps.length - 1];
|
||||
if (last?.op === "navigate" && last.url === finalUrl) {
|
||||
if (!last.postStateId) {
|
||||
const draftIndex = steps.length - 1;
|
||||
scheduleDraftSettle(obs, draftIndex, draftIndex + 1, cdp, tabId, tabsApi, steps);
|
||||
await obs.settleQueue;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (outcome === "timeout") {
|
||||
console.debug(
|
||||
`[bsk record] redirect landing still changing after settle budget; ` +
|
||||
`recording navigate to ${finalUrl}`,
|
||||
);
|
||||
}
|
||||
|
||||
const draft: DraftTraceStep = {
|
||||
op: "navigate",
|
||||
url: finalUrl,
|
||||
page_url: finalUrl,
|
||||
cause: "browser",
|
||||
preStateId: obs.lastSettled?.stateId,
|
||||
};
|
||||
steps.push(draft);
|
||||
const draftIndex = steps.length - 1;
|
||||
const stepId = draftIndex + 1;
|
||||
rememberStepOnPage(obs, draft, stepId);
|
||||
scheduleDraftSettle(obs, draftIndex, stepId, cdp, tabId, tabsApi, steps);
|
||||
// Callers that flush before the next action need `lastSettled` to already
|
||||
// be the landing page so target matching does not use the pre-redirect view.
|
||||
await obs.settleQueue;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A capture that lands mid-navigation fails on a destroyed execution context.
|
||||
* That is transient, so give it one more chance before the step has to fall
|
||||
* back to a stale state.
|
||||
*/
|
||||
async function captureWithRetry(
|
||||
obs: RecordingObservationState,
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
urlOverride?: string,
|
||||
): Promise<LastSettledObservation> {
|
||||
try {
|
||||
return await captureAndRegisterObservation(obs, cdp, tabId, tabsApi, urlOverride);
|
||||
} catch (err) {
|
||||
console.debug("[bsk record] observation failed, retrying once", err);
|
||||
await sleep(CAPTURE_RETRY_DELAY_MS);
|
||||
return captureAndRegisterObservation(obs, cdp, tabId, tabsApi, urlOverride);
|
||||
}
|
||||
}
|
||||
|
||||
export async function settleDraftObservation(
|
||||
obs: RecordingObservationState,
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
steps: DraftTraceStep[],
|
||||
draftIndex: number,
|
||||
pending?: PendingSettle,
|
||||
): Promise<void> {
|
||||
const cancelled = () => pending?.cancelled === true;
|
||||
const outcome = await waitForPageSettled(cdp, tabId, { cancelled });
|
||||
if (outcome === "cancelled") return;
|
||||
if (outcome === "timeout") {
|
||||
console.debug(
|
||||
`[bsk record] step ${draftIndex + 1} was still changing the page after the settle ` +
|
||||
"budget; observing it as it stands",
|
||||
);
|
||||
}
|
||||
|
||||
const started = steps[draftIndex];
|
||||
if (!started) return;
|
||||
|
||||
// The URL attached to the draft may only be an intermediate redirect hop.
|
||||
// Once the page is settled, register the capture against live tab metadata.
|
||||
const settled = await captureWithRetry(obs, cdp, tabId, tabsApi);
|
||||
// The observation still counts as the latest view of the page even when this
|
||||
// step no longer wants it — the next action will start from it.
|
||||
if (cancelled()) return;
|
||||
// Re-read the slot: a navigation observed while the capture ran may have
|
||||
// rewritten this draft, and the observation belongs to whatever occupies the
|
||||
// slot now — writing to the object we started with would strand it.
|
||||
const draft = steps[draftIndex] ?? started;
|
||||
draft.postStateId = settled.stateId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recover landing states from the shape of the recording itself: wherever the
|
||||
* next action was performed is, by definition, where the previous one landed.
|
||||
* Only the immediately following draft counts — a later one would claim a page
|
||||
* that several unobserved actions away, which is worse than admitting we saw
|
||||
* no change.
|
||||
*/
|
||||
export function inferMissingPostStates(steps: DraftTraceStep[]): void {
|
||||
for (let i = 0; i < steps.length - 1; i += 1) {
|
||||
const draft = steps[i];
|
||||
const next = steps[i + 1];
|
||||
if (!draft || !next || draft.postStateId || !next.preStateId) continue;
|
||||
draft.postStateId = next.preStateId;
|
||||
console.debug(
|
||||
`[bsk record] step ${i + 1} (${draft.op}) had no post-action observation; ` +
|
||||
`using where step ${i + 2} started (${next.preStateId})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Last chance for the closing actions of a recording. Stopping right after the
|
||||
* final action is normal, and so is that action navigating away, so take one
|
||||
* final look at the page. Only trailing drafts may claim it: an earlier step
|
||||
* did not land on whatever the user happens to be looking at when they stop.
|
||||
*/
|
||||
export async function settleUnsettledDrafts(
|
||||
obs: RecordingObservationState,
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
steps: DraftTraceStep[],
|
||||
): Promise<void> {
|
||||
const trailing: DraftTraceStep[] = [];
|
||||
for (let i = steps.length - 1; i >= 0; i -= 1) {
|
||||
const draft = steps[i];
|
||||
if (!draft || draft.postStateId) break;
|
||||
trailing.push(draft);
|
||||
}
|
||||
if (trailing.length === 0) return;
|
||||
|
||||
let settled: LastSettledObservation;
|
||||
try {
|
||||
settled = await captureWithRetry(obs, cdp, tabId, tabsApi);
|
||||
} catch (err) {
|
||||
console.warn("[bsk record] final observation at stop failed", err);
|
||||
return;
|
||||
}
|
||||
for (const draft of trailing) draft.postStateId = settled.stateId;
|
||||
console.debug(
|
||||
`[bsk record] settled ${trailing.length} trailing step(s) against the page at stop ` +
|
||||
`(${settled.stateId} ${settled.url})`,
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Performing a new action ends the previous one: the page the user reached for
|
||||
* is, by definition, where the earlier step left them. Taking the landing from
|
||||
* the newer step also keeps the trace monotonic — a capture that finished late
|
||||
* would otherwise credit an earlier step with a page that only exists because
|
||||
* of the newer action.
|
||||
*/
|
||||
function supersedeEarlierSettles(
|
||||
obs: RecordingObservationState,
|
||||
draftIndex: number,
|
||||
steps: DraftTraceStep[],
|
||||
): void {
|
||||
const landing = steps[draftIndex]?.preStateId;
|
||||
for (const [index, pending] of obs.settles) {
|
||||
if (index >= draftIndex) continue;
|
||||
pending.cancelled = true;
|
||||
obs.settles.delete(index);
|
||||
|
||||
const draft = steps[index];
|
||||
if (!draft || draft.postStateId) continue;
|
||||
if (!landing) continue;
|
||||
draft.postStateId = landing;
|
||||
console.debug(
|
||||
`[bsk record] step ${index + 1} (${draft.op}) was still settling when step ` +
|
||||
`${draftIndex + 1} started; landing it on ${landing}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function scheduleDraftSettle(
|
||||
obs: RecordingObservationState,
|
||||
draftIndex: number,
|
||||
stepId: number,
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
tabsApi: ChromeTabsApi,
|
||||
steps: DraftTraceStep[],
|
||||
): void {
|
||||
supersedeEarlierSettles(obs, draftIndex, steps);
|
||||
|
||||
// Rescheduling the same step (a navigation showed up after the action)
|
||||
// replaces the pending observation rather than racing it.
|
||||
const superseded = obs.settles.get(draftIndex);
|
||||
if (superseded) superseded.cancelled = true;
|
||||
|
||||
const pending: PendingSettle = { draftIndex, cancelled: false };
|
||||
obs.settles.set(draftIndex, pending);
|
||||
|
||||
enqueueSettle(obs, async () => {
|
||||
if (pending.cancelled) return;
|
||||
try {
|
||||
await settleDraftObservation(obs, cdp, tabId, tabsApi, steps, draftIndex, pending);
|
||||
} catch (err) {
|
||||
// Recoverable: stop-time repair still gives the step a landing page.
|
||||
const op = steps[draftIndex]?.op ?? "?";
|
||||
console.warn(`[bsk record] post-action observation failed for step ${stepId} (${op})`, err);
|
||||
} finally {
|
||||
if (obs.settles.get(draftIndex) === pending) obs.settles.delete(draftIndex);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function enqueueSettle(obs: RecordingObservationState, task: () => Promise<void>): void {
|
||||
obs.settleQueue = obs.settleQueue.then(task, task).catch(() => {});
|
||||
}
|
||||
|
||||
export function cancelPendingSettles(obs: RecordingObservationState): void {
|
||||
for (const pending of obs.settles.values()) pending.cancelled = true;
|
||||
obs.settles.clear();
|
||||
}
|
||||
|
||||
/** Bound the drain loop so a self-rescheduling settle cannot block stop. */
|
||||
const MAX_SETTLE_FLUSH_ROUNDS = 10;
|
||||
|
||||
/**
|
||||
* Drain settle work before exporting, including work queued while draining —
|
||||
* a navigation observed during the last capture schedules another one, and
|
||||
* the trace would drop it if stop did not wait.
|
||||
*/
|
||||
export async function flushPendingSettles(obs: RecordingObservationState): Promise<void> {
|
||||
for (let round = 0; round < MAX_SETTLE_FLUSH_ROUNDS; round += 1) {
|
||||
const drained = obs.settleQueue;
|
||||
await drained;
|
||||
if (obs.settleQueue === drained) return;
|
||||
}
|
||||
console.warn("[bsk record] settle queue kept growing at stop; exporting what has been observed");
|
||||
}
|
||||
|
||||
function finalizeStateBodies(
|
||||
entries: StateRegistryEntry[],
|
||||
annotationsByState: Map<string, ObservationAnnotation[]>,
|
||||
stepIdByDraftId: Map<number, number>,
|
||||
idByOldId: Map<string, string>,
|
||||
): TraceState[] {
|
||||
return entries.map((entry) => {
|
||||
const id = idByOldId.get(entry.id) ?? entry.id;
|
||||
const body = formatObservationFile({
|
||||
stateId: id,
|
||||
url: entry.url,
|
||||
title: entry.title,
|
||||
stepsHere: remapStepIds(entry.stepsHere, stepIdByDraftId),
|
||||
body: entry.rawVomText,
|
||||
annotations: annotationsByState.get(entry.id) ?? [],
|
||||
});
|
||||
return {
|
||||
id,
|
||||
url: entry.url,
|
||||
...(entry.title ? { title: entry.title } : {}),
|
||||
body,
|
||||
...(entry.truncated ? { truncated: true } : {}),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
/** Draft ids only become step ids after collapsing and filtering. */
|
||||
function remapStepIds(draftIds: number[], stepIdByDraftId: Map<number, number>): number[] {
|
||||
const mapped = new Set<number>();
|
||||
for (const draftId of draftIds) {
|
||||
const stepId = stepIdByDraftId.get(draftId);
|
||||
if (stepId !== undefined) mapped.add(stepId);
|
||||
}
|
||||
return [...mapped].sort((a, b) => a - b);
|
||||
}
|
||||
|
||||
function collectAnnotationsByState(
|
||||
obs: RecordingObservationState,
|
||||
stepIdByDraftId: Map<number, number>,
|
||||
): Map<string, ObservationAnnotation[]> {
|
||||
const byState = new Map<string, ObservationAnnotation[]>();
|
||||
for (const anns of obs.stepAnnotations.values()) {
|
||||
for (const ann of anns) {
|
||||
const stepId = stepIdByDraftId.get(ann.stepId);
|
||||
if (stepId === undefined) continue;
|
||||
const bucket = byState.get(ann.stateId) ?? [];
|
||||
bucket.push({ ...ann, stepId });
|
||||
byState.set(ann.stateId, bucket);
|
||||
}
|
||||
}
|
||||
return byState;
|
||||
}
|
||||
|
||||
/**
|
||||
* Keep only the pages the published steps point at. Redirect hops and
|
||||
* mid-load captures land in the registry too, and shipping them would invite
|
||||
* a reader to treat a page the flow merely passed through as a real stop.
|
||||
* With no steps at all the first observation is the whole artifact, so it
|
||||
* stays.
|
||||
*/
|
||||
function selectPublishedStates(
|
||||
registry: Map<string, StateRegistryEntry>,
|
||||
steps: Step[],
|
||||
): StateRegistryEntry[] {
|
||||
const entries = [...registry.values()];
|
||||
if (steps.length === 0) return entries.slice(0, 1);
|
||||
const referenced = new Set<string>();
|
||||
for (const step of steps) {
|
||||
referenced.add(step.state);
|
||||
referenced.add(step.result.state);
|
||||
}
|
||||
return entries.filter((entry) => referenced.has(entry.id));
|
||||
}
|
||||
|
||||
export function buildTraceV3(input: {
|
||||
obs: RecordingObservationState;
|
||||
steps: DraftTraceStep[];
|
||||
startedAt: string;
|
||||
purpose?: string;
|
||||
startUrl?: string;
|
||||
stoppedBy: StopReason;
|
||||
bskVersion: string;
|
||||
frameCapture?: Trace["frame_capture"];
|
||||
}): Trace {
|
||||
const { steps, stepIdByDraftId } = reduceTraceSteps(input.steps, input.obs.stateRegistry);
|
||||
const published = selectPublishedStates(input.obs.stateRegistry, steps);
|
||||
// Renumber so the shipped dictionary reads s1..sN without holes where the
|
||||
// dropped captures used to be.
|
||||
const idByOldId = new Map(published.map((entry, index) => [entry.id, `s${index + 1}`]));
|
||||
for (const step of steps) {
|
||||
step.state = idByOldId.get(step.state) ?? step.state;
|
||||
step.result.state = idByOldId.get(step.result.state) ?? step.result.state;
|
||||
}
|
||||
const annotationsByState = collectAnnotationsByState(input.obs, stepIdByDraftId);
|
||||
const states = finalizeStateBodies(published, annotationsByState, stepIdByDraftId, idByOldId);
|
||||
const startUrl = resolveTraceStartUrl(input.steps, input.startUrl, states);
|
||||
return {
|
||||
version: 3,
|
||||
...(input.purpose ? { purpose: input.purpose } : {}),
|
||||
recorded_at: new Date().toISOString(),
|
||||
started_at: input.startedAt,
|
||||
stopped_by: input.stoppedBy,
|
||||
entry: { start_url: startUrl },
|
||||
recorder: { bsk: input.bskVersion, vom: 1 },
|
||||
...(input.frameCapture ? { frame_capture: input.frameCapture } : {}),
|
||||
states,
|
||||
steps,
|
||||
};
|
||||
}
|
||||
@@ -1,23 +1,29 @@
|
||||
import type { DraftTraceStep } from "@/transport/types";
|
||||
import type { DraftTraceStep, NavigationCause } from "@/transport/types";
|
||||
import type { RecordStepPayload } from "./record-bridge";
|
||||
import { mapNavigationCause } from "./record-constants";
|
||||
|
||||
export interface RecordingStepBuffer {
|
||||
steps: DraftTraceStep[];
|
||||
currentUrl?: string;
|
||||
pendingNavigation: boolean;
|
||||
pendingNavigationDeadline?: number;
|
||||
lastTransitionType?: string;
|
||||
lastTransitionQualifiers?: string[];
|
||||
}
|
||||
|
||||
const NAVIGATION_TRIGGER_WINDOW_MS = 3_000;
|
||||
|
||||
function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
||||
const pageUrl = payload.page_url;
|
||||
const geometry = payload.geometry;
|
||||
switch (payload.op) {
|
||||
case "click":
|
||||
return payload.target
|
||||
? {
|
||||
op: "click",
|
||||
target: payload.target,
|
||||
target: { unmatched: true },
|
||||
captureTarget: payload.target,
|
||||
...(geometry ? { geometry } : {}),
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
}
|
||||
: null;
|
||||
@@ -25,7 +31,9 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
||||
return payload.target
|
||||
? {
|
||||
op: "hover",
|
||||
target: payload.target,
|
||||
target: { unmatched: true },
|
||||
captureTarget: payload.target,
|
||||
...(geometry ? { geometry } : {}),
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
}
|
||||
: null;
|
||||
@@ -33,8 +41,11 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
||||
return payload.target
|
||||
? {
|
||||
op: "fill",
|
||||
target: payload.target,
|
||||
target: { unmatched: true },
|
||||
captureTarget: payload.target,
|
||||
...(geometry ? { geometry } : {}),
|
||||
value: payload.value ?? "",
|
||||
...(payload.commit ? { commit: payload.commit } : {}),
|
||||
...(payload.redacted ? { redacted: true } : {}),
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
}
|
||||
@@ -44,7 +55,10 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
||||
? {
|
||||
op: "press",
|
||||
key: payload.key,
|
||||
...(payload.target ? { target: payload.target } : {}),
|
||||
...(payload.target
|
||||
? { target: { unmatched: true }, captureTarget: payload.target }
|
||||
: {}),
|
||||
...(geometry ? { geometry } : {}),
|
||||
...(payload.modifiers?.length ? { modifiers: payload.modifiers } : {}),
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
}
|
||||
@@ -53,37 +67,69 @@ function toDraftStep(payload: RecordStepPayload): DraftTraceStep | null {
|
||||
return payload.target && payload.values
|
||||
? {
|
||||
op: "select",
|
||||
target: payload.target,
|
||||
target: { unmatched: true },
|
||||
captureTarget: payload.target,
|
||||
...(geometry ? { geometry } : {}),
|
||||
values: payload.values,
|
||||
...(payload.labels?.length ? { labels: payload.labels } : {}),
|
||||
...(pageUrl ? { page_url: pageUrl } : {}),
|
||||
}
|
||||
: null;
|
||||
case "scroll":
|
||||
return { op: "scroll", ...(pageUrl ? { page_url: pageUrl } : {}) };
|
||||
case "navigate":
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): boolean {
|
||||
/** Annotate the latest action with `navigated_to`. Returns its index, or -1. */
|
||||
function annotateLastStepNavigation(buffer: RecordingStepBuffer, url: string): number {
|
||||
for (let i = buffer.steps.length - 1; i >= 0; i -= 1) {
|
||||
const step = buffer.steps[i];
|
||||
if (!step) continue;
|
||||
if (step.op === "click" || step.op === "press" || step.op === "select") {
|
||||
buffer.steps[i] = { ...step, navigated_to: url };
|
||||
return true;
|
||||
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
||||
// Mutate in place: an in-flight settle holds this object, and replacing
|
||||
// it would strand that settle's observation on the discarded copy.
|
||||
step.navigated_to = url;
|
||||
return i;
|
||||
}
|
||||
break;
|
||||
}
|
||||
return false;
|
||||
return -1;
|
||||
}
|
||||
|
||||
function causeFromBuffer(buffer: RecordingStepBuffer): NavigationCause | null {
|
||||
return mapNavigationCause({
|
||||
transitionType: buffer.lastTransitionType,
|
||||
transitionQualifiers: buffer.lastTransitionQualifiers,
|
||||
});
|
||||
}
|
||||
|
||||
/** Outcome of folding a main-frame URL change into the draft buffer. */
|
||||
export type NavigationObserveResult =
|
||||
| { kind: "noop" }
|
||||
| { kind: "annotated"; index: number }
|
||||
| { kind: "appended"; index: number }
|
||||
/**
|
||||
* Redirect hop: do not emit a step yet. The recorder coalesces the chain and
|
||||
* emits one `navigate` after `waitForPageSettled` on the final URL.
|
||||
*/
|
||||
| { kind: "coalesce_redirect"; url: string };
|
||||
|
||||
export function observeRecordedNavigation(
|
||||
buffer: RecordingStepBuffer,
|
||||
url: string,
|
||||
causedByAction?: boolean,
|
||||
): void {
|
||||
if (!url || url === buffer.currentUrl) return;
|
||||
transitionType?: string,
|
||||
transitionQualifiers?: string[],
|
||||
): NavigationObserveResult {
|
||||
if (!url || url === buffer.currentUrl) return { kind: "noop" };
|
||||
buffer.currentUrl = url;
|
||||
if (transitionType !== undefined) buffer.lastTransitionType = transitionType;
|
||||
if (transitionQualifiers !== undefined) {
|
||||
buffer.lastTransitionQualifiers = transitionQualifiers;
|
||||
}
|
||||
|
||||
const pendingIsCurrent =
|
||||
buffer.pendingNavigation &&
|
||||
(buffer.pendingNavigationDeadline === undefined ||
|
||||
@@ -92,40 +138,69 @@ export function observeRecordedNavigation(
|
||||
if (causedByAction === true || (causedByAction === undefined && pendingIsCurrent)) {
|
||||
buffer.pendingNavigation = false;
|
||||
buffer.pendingNavigationDeadline = undefined;
|
||||
if (!annotateLastStepNavigation(buffer, url)) {
|
||||
buffer.steps.push({
|
||||
op: "navigate",
|
||||
url,
|
||||
page_url: url,
|
||||
});
|
||||
const annotatedIndex = annotateLastStepNavigation(buffer, url);
|
||||
if (annotatedIndex >= 0) {
|
||||
return { kind: "annotated", index: annotatedIndex };
|
||||
}
|
||||
return;
|
||||
const cause = causeFromBuffer(buffer);
|
||||
if (cause === null) {
|
||||
return { kind: "coalesce_redirect", url };
|
||||
}
|
||||
buffer.steps.push({
|
||||
op: "navigate",
|
||||
url,
|
||||
page_url: url,
|
||||
cause,
|
||||
transitionType: buffer.lastTransitionType,
|
||||
transitionQualifiers: buffer.lastTransitionQualifiers,
|
||||
});
|
||||
return { kind: "appended", index: buffer.steps.length - 1 };
|
||||
}
|
||||
|
||||
buffer.pendingNavigation = false;
|
||||
buffer.pendingNavigationDeadline = undefined;
|
||||
const cause = causeFromBuffer(buffer);
|
||||
if (cause === null) {
|
||||
return { kind: "coalesce_redirect", url };
|
||||
}
|
||||
buffer.steps.push({
|
||||
op: "navigate",
|
||||
url,
|
||||
page_url: url,
|
||||
cause,
|
||||
transitionType: buffer.lastTransitionType,
|
||||
transitionQualifiers: buffer.lastTransitionQualifiers,
|
||||
});
|
||||
return { kind: "appended", index: buffer.steps.length - 1 };
|
||||
}
|
||||
|
||||
/**
|
||||
* Buffer one captured payload. Returns the index of the draft it appended, or
|
||||
* `null` when the payload produced none (an unnamed click target, a navigation
|
||||
* that only annotated the previous draft, …). Callers must not fall back to
|
||||
* "the last draft" on `null`: that would re-match an already recorded step
|
||||
* against a newer observation and overwrite its origin state.
|
||||
*/
|
||||
export function appendRecordedPayload(
|
||||
buffer: RecordingStepBuffer,
|
||||
payload: RecordStepPayload,
|
||||
): void {
|
||||
): number | null {
|
||||
if (payload.op === "navigate") {
|
||||
if (payload.url) {
|
||||
observeRecordedNavigation(buffer, payload.url, payload.navigation_caused_by_action);
|
||||
}
|
||||
return;
|
||||
if (!payload.url) return null;
|
||||
const result = observeRecordedNavigation(
|
||||
buffer,
|
||||
payload.url,
|
||||
payload.navigation_caused_by_action,
|
||||
payload.transitionType,
|
||||
payload.transitionQualifiers,
|
||||
);
|
||||
return result.kind === "appended" ? result.index : null;
|
||||
}
|
||||
const step = toDraftStep({
|
||||
...payload,
|
||||
page_url: payload.page_url ?? buffer.currentUrl,
|
||||
});
|
||||
if (!step) return;
|
||||
if (!step) return null;
|
||||
buffer.steps.push(step);
|
||||
if (step.op === "click" || step.op === "press" || step.op === "select" || step.op === "fill") {
|
||||
buffer.pendingNavigation = payload.expects_navigation === true;
|
||||
@@ -133,4 +208,5 @@ export function appendRecordedPayload(
|
||||
? Date.now() + NAVIGATION_TRIGGER_WINDOW_MS
|
||||
: undefined;
|
||||
}
|
||||
return buffer.steps.length - 1;
|
||||
}
|
||||
|
||||
@@ -1,4 +1,17 @@
|
||||
import type { DraftTraceStep, PageRef, SelectedOption, Step } from "@/transport/types";
|
||||
import type {
|
||||
DraftTraceStep,
|
||||
FillCommit,
|
||||
NavigationCause,
|
||||
Step,
|
||||
TraceState,
|
||||
} from "@/transport/types";
|
||||
import {
|
||||
DEFAULT_FILL_COMMIT,
|
||||
fnv1a64,
|
||||
mapNavigationCause,
|
||||
nextStateId,
|
||||
resetStateIdCounterForTests,
|
||||
} from "./record-constants";
|
||||
|
||||
const CLIPBOARD_KEYS = new Set(["a", "c", "v", "x", "A", "C", "V", "X"]);
|
||||
const MODIFIER_ONLY_KEYS = new Set(["Meta", "Control", "Alt", "Shift", "OS", "Hyper", "Super"]);
|
||||
@@ -12,7 +25,6 @@ export function shouldRecordPress(
|
||||
const hasCtrlOrMeta = mods.includes("ctrl") || mods.includes("meta");
|
||||
if (hasCtrlOrMeta && CLIPBOARD_KEYS.has(key)) return false;
|
||||
if (key === "Enter" || key === "Escape") return true;
|
||||
// Drop bare character typing — FillSession already records the value.
|
||||
if (key.length === 1 && !hasCtrlOrMeta && !mods.includes("alt")) return false;
|
||||
return false;
|
||||
}
|
||||
@@ -22,195 +34,191 @@ function shouldIncludeDraft(step: DraftTraceStep): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Collapse consecutive navigations to the last hop. */
|
||||
function collapseNavigations(steps: DraftTraceStep[]): DraftTraceStep[] {
|
||||
const out: DraftTraceStep[] = [];
|
||||
for (const step of steps) {
|
||||
/** A reduced draft plus the capture-order ids of every draft folded into it. */
|
||||
interface CollapsedDraft {
|
||||
draft: DraftTraceStep;
|
||||
draftIds: number[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Collapse a redirect chain into a single hop.
|
||||
*
|
||||
* The surviving step keeps the *first* hop's origin state and cause — that is
|
||||
* where the user was and why they left — and the *last* hop's destination, so
|
||||
* the intermediate URL the browser only passed through never becomes the
|
||||
* step's `state` or `to`.
|
||||
*/
|
||||
function collapseNavigations(steps: DraftTraceStep[]): CollapsedDraft[] {
|
||||
const out: CollapsedDraft[] = [];
|
||||
steps.forEach((step, index) => {
|
||||
const draftId = index + 1;
|
||||
const prev = out[out.length - 1];
|
||||
if (step.op === "navigate" && prev?.op === "navigate") {
|
||||
out[out.length - 1] = step;
|
||||
continue;
|
||||
if (step.op === "navigate" && prev?.draft.op === "navigate") {
|
||||
prev.draft = {
|
||||
...prev.draft,
|
||||
url: step.url,
|
||||
postStateId: step.postStateId ?? prev.draft.postStateId,
|
||||
};
|
||||
prev.draftIds.push(draftId);
|
||||
return;
|
||||
}
|
||||
out.push(step);
|
||||
}
|
||||
out.push({ draft: step, draftIds: [draftId] });
|
||||
});
|
||||
return out;
|
||||
}
|
||||
|
||||
function collectUrls(steps: DraftTraceStep[], startUrl?: string): string[] {
|
||||
const urls: string[] = [];
|
||||
if (startUrl) urls.push(startUrl);
|
||||
for (const step of steps) {
|
||||
if (step.op === "navigate") {
|
||||
urls.push(step.url);
|
||||
continue;
|
||||
}
|
||||
if ("page_url" in step && step.page_url) urls.push(step.page_url);
|
||||
if ("navigated_to" in step && step.navigated_to) urls.push(step.navigated_to);
|
||||
}
|
||||
const seen = new Set<string>();
|
||||
const unique: string[] = [];
|
||||
for (const url of urls) {
|
||||
if (!url || seen.has(url)) continue;
|
||||
seen.add(url);
|
||||
unique.push(url);
|
||||
}
|
||||
return unique;
|
||||
export interface StateRegistryEntry extends TraceState {
|
||||
contentHash: string;
|
||||
rawVomText: string;
|
||||
stepsHere: number[];
|
||||
}
|
||||
|
||||
function buildPageRegistry(
|
||||
steps: DraftTraceStep[],
|
||||
startUrl?: string,
|
||||
): { pages: PageRef[]; urlToId: Map<string, string> } {
|
||||
const urls = collectUrls(steps, startUrl);
|
||||
const urlToId = new Map<string, string>();
|
||||
const pages = urls.map((url, index) => {
|
||||
const id = `p${index + 1}`;
|
||||
urlToId.set(url, id);
|
||||
return { id, url };
|
||||
});
|
||||
return { pages, urlToId };
|
||||
export function stateDedupKey(body: string, url: string): string {
|
||||
return `${fnv1a64(body)}:${url}`;
|
||||
}
|
||||
|
||||
function pageIdFor(
|
||||
url: string | undefined,
|
||||
urlToId: Map<string, string>,
|
||||
fallbackUrl?: string,
|
||||
/** Register or reuse a page observation in the states dictionary. */
|
||||
export function registerObservation(
|
||||
registry: Map<string, StateRegistryEntry>,
|
||||
input: {
|
||||
url: string;
|
||||
title?: string;
|
||||
rawVomText: string;
|
||||
truncated?: boolean;
|
||||
},
|
||||
): string {
|
||||
if (url && urlToId.has(url)) return urlToId.get(url)!;
|
||||
if (fallbackUrl && urlToId.has(fallbackUrl)) return urlToId.get(fallbackUrl)!;
|
||||
return urlToId.values().next().value ?? "p1";
|
||||
const key = stateDedupKey(input.rawVomText, input.url);
|
||||
for (const entry of registry.values()) {
|
||||
if (entry.contentHash === key) return entry.id;
|
||||
}
|
||||
const id = nextStateId();
|
||||
registry.set(id, {
|
||||
id,
|
||||
url: input.url,
|
||||
...(input.title ? { title: input.title } : {}),
|
||||
truncated: input.truncated ?? false,
|
||||
contentHash: key,
|
||||
rawVomText: input.rawVomText,
|
||||
stepsHere: [],
|
||||
});
|
||||
return id;
|
||||
}
|
||||
|
||||
function pageUrlForDraft(step: DraftTraceStep, fallbackUrl?: string): string | undefined {
|
||||
if (step.op === "navigate") return step.page_url ?? step.url;
|
||||
if ("page_url" in step && step.page_url) return step.page_url;
|
||||
return fallbackUrl;
|
||||
}
|
||||
|
||||
function effectForNavigation(
|
||||
navigatedTo: string | undefined,
|
||||
urlToId: Map<string, string>,
|
||||
): Step["effect"] {
|
||||
if (!navigatedTo) return undefined;
|
||||
const pageId = urlToId.get(navigatedTo);
|
||||
if (!pageId) return undefined;
|
||||
return { navigated_to: pageId };
|
||||
}
|
||||
|
||||
function withEffect(step: Step, effect: Step["effect"]): Step {
|
||||
if (!effect) return step;
|
||||
return { ...step, effect };
|
||||
}
|
||||
|
||||
function toSelection(values: string[], labels?: string[]): SelectedOption[] {
|
||||
function toSelection(
|
||||
values: string[],
|
||||
labels?: string[],
|
||||
): Array<{ value: string; label?: string }> {
|
||||
return values.map((value, index) => ({
|
||||
value,
|
||||
...(labels?.[index] ? { label: labels[index] } : {}),
|
||||
}));
|
||||
}
|
||||
|
||||
function toV2Step(
|
||||
step: DraftTraceStep,
|
||||
id: number,
|
||||
urlToId: Map<string, string>,
|
||||
fallbackUrl?: string,
|
||||
): Step | null {
|
||||
if (!shouldIncludeDraft(step)) return null;
|
||||
function navigateCauseForDraft(
|
||||
draft: Extract<DraftTraceStep, { op: "navigate" }>,
|
||||
): NavigationCause {
|
||||
if (draft.cause) return draft.cause;
|
||||
const mapped = mapNavigationCause({
|
||||
transitionType: draft.transitionType,
|
||||
transitionQualifiers: draft.transitionQualifiers,
|
||||
});
|
||||
return mapped ?? "browser";
|
||||
}
|
||||
|
||||
const pageUrl = pageUrlForDraft(step, fallbackUrl);
|
||||
const page = pageIdFor(pageUrl, urlToId, fallbackUrl);
|
||||
function toV3Step(draft: DraftTraceStep, id: number): Step | null {
|
||||
if (!shouldIncludeDraft(draft)) return null;
|
||||
// An action the user really performed must not disappear because one of its
|
||||
// observations is missing — a capture racing a document swap is routine, and
|
||||
// the last step of a recording is the most exposed to it. Fall back to the
|
||||
// endpoint we do know; only a step with no observation at all has nothing to
|
||||
// point at.
|
||||
const preState = draft.preStateId ?? draft.postStateId;
|
||||
const postState = draft.postStateId ?? draft.preStateId;
|
||||
if (!preState || !postState) return null;
|
||||
|
||||
switch (step.op) {
|
||||
const common = { id, state: preState, result: { state: postState } };
|
||||
|
||||
switch (draft.op) {
|
||||
case "navigate":
|
||||
return {
|
||||
op: "navigate",
|
||||
id,
|
||||
page: pageIdFor(step.url, urlToId, fallbackUrl),
|
||||
to: step.url,
|
||||
...common,
|
||||
to: draft.url,
|
||||
cause: navigateCauseForDraft(draft),
|
||||
};
|
||||
case "click":
|
||||
return withEffect(
|
||||
{
|
||||
op: "click",
|
||||
id,
|
||||
page,
|
||||
target: step.target,
|
||||
},
|
||||
effectForNavigation(step.navigated_to, urlToId),
|
||||
);
|
||||
return { op: "click", ...common, target: draft.target };
|
||||
case "hover":
|
||||
return {
|
||||
op: "hover",
|
||||
id,
|
||||
page,
|
||||
target: step.target,
|
||||
};
|
||||
return { op: "hover", ...common, target: draft.target };
|
||||
case "fill":
|
||||
return {
|
||||
op: "fill",
|
||||
id,
|
||||
page,
|
||||
target: step.target,
|
||||
value: step.value,
|
||||
...(step.redacted ? { redacted: true } : {}),
|
||||
...common,
|
||||
target: draft.target,
|
||||
value: draft.value,
|
||||
commit: draft.commit ?? DEFAULT_FILL_COMMIT,
|
||||
...(draft.redacted ? { redacted: true } : {}),
|
||||
};
|
||||
case "press":
|
||||
return withEffect(
|
||||
{
|
||||
op: "press",
|
||||
id,
|
||||
page,
|
||||
key: step.key,
|
||||
...(step.target ? { target: step.target } : {}),
|
||||
...(step.modifiers?.length ? { modifiers: step.modifiers } : {}),
|
||||
},
|
||||
effectForNavigation(step.navigated_to, urlToId),
|
||||
);
|
||||
return {
|
||||
op: "press",
|
||||
...common,
|
||||
key: draft.key,
|
||||
...(draft.target ? { target: draft.target } : {}),
|
||||
...(draft.modifiers?.length ? { modifiers: draft.modifiers } : {}),
|
||||
};
|
||||
case "select":
|
||||
return withEffect(
|
||||
{
|
||||
op: "select",
|
||||
id,
|
||||
page,
|
||||
target: step.target,
|
||||
selection: toSelection(step.values, step.labels),
|
||||
},
|
||||
effectForNavigation(step.navigated_to, urlToId),
|
||||
);
|
||||
return {
|
||||
op: "select",
|
||||
...common,
|
||||
target: draft.target,
|
||||
selection: toSelection(draft.values, draft.labels),
|
||||
};
|
||||
case "scroll":
|
||||
return { op: "scroll", ...common };
|
||||
}
|
||||
}
|
||||
|
||||
export interface ReducedTrace {
|
||||
pages: PageRef[];
|
||||
states: TraceState[];
|
||||
steps: Step[];
|
||||
/**
|
||||
* Capture-order draft id (`draftIndex + 1`) → published step id. Collapsed
|
||||
* and dropped drafts make the two diverge, so anything the recorder tagged
|
||||
* with a draft id (`steps_here`, inline annotations) must be remapped.
|
||||
*/
|
||||
stepIdByDraftId: Map<number, number>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compile capture drafts into record-only trace v2 steps.
|
||||
* Variable inputs are NOT classified here — executing agents infer that at run time.
|
||||
* Compile capture drafts into record-only trace v3 steps.
|
||||
* Drafts must already carry `preStateId` / `postStateId` from the recorder.
|
||||
*/
|
||||
export function reduceTraceSteps(steps: DraftTraceStep[], startUrl?: string): ReducedTrace {
|
||||
export function reduceTraceSteps(
|
||||
steps: DraftTraceStep[],
|
||||
registry: Map<string, StateRegistryEntry>,
|
||||
): ReducedTrace {
|
||||
const collapsed = collapseNavigations(steps);
|
||||
const { pages, urlToId } = buildPageRegistry(collapsed, startUrl);
|
||||
const out: Step[] = [];
|
||||
const stepIdByDraftId = new Map<number, number>();
|
||||
let id = 1;
|
||||
let lastUrl = startUrl;
|
||||
for (const draft of collapsed) {
|
||||
if (draft.op === "navigate") lastUrl = draft.url;
|
||||
else if ("navigated_to" in draft && draft.navigated_to) lastUrl = draft.navigated_to;
|
||||
else if ("page_url" in draft && draft.page_url) lastUrl = draft.page_url;
|
||||
const step = toV2Step(draft, id, urlToId, lastUrl);
|
||||
for (const { draft, draftIds } of collapsed) {
|
||||
const step = toV3Step(draft, id);
|
||||
if (!step) continue;
|
||||
out.push(step);
|
||||
for (const draftId of draftIds) stepIdByDraftId.set(draftId, id);
|
||||
id += 1;
|
||||
}
|
||||
return { pages, steps: out };
|
||||
const states = [...registry.values()].map(
|
||||
({ contentHash: _hash, rawVomText: _body, stepsHere: _steps, ...state }) => state,
|
||||
);
|
||||
return { states, steps: out, stepIdByDraftId };
|
||||
}
|
||||
|
||||
export function resolveTraceStartUrl(
|
||||
drafts: DraftTraceStep[],
|
||||
startUrl?: string,
|
||||
pages?: PageRef[],
|
||||
states?: TraceState[],
|
||||
): string {
|
||||
if (startUrl) return startUrl;
|
||||
const navigate = drafts.find((step): step is Extract<DraftTraceStep, { op: "navigate" }> => {
|
||||
@@ -220,5 +228,9 @@ export function resolveTraceStartUrl(
|
||||
for (const draft of drafts) {
|
||||
if ("page_url" in draft && draft.page_url) return draft.page_url;
|
||||
}
|
||||
return pages?.[0]?.url ?? "about:blank";
|
||||
return states?.[0]?.url ?? "about:blank";
|
||||
}
|
||||
|
||||
export type { FillCommit, NavigationCause };
|
||||
/** Test seam */
|
||||
export { resetStateIdCounterForTests };
|
||||
|
||||
@@ -75,52 +75,6 @@ describe("SessionManager", () => {
|
||||
await expect(sm.start("aa11")).rejects.toThrow(/already exists/);
|
||||
});
|
||||
|
||||
it("removes a newly created Agent Window when startup is aborted", async () => {
|
||||
const aw = fakeAgentWindow();
|
||||
let resolveCreate: (windowId: number) => void = () => {};
|
||||
aw.createMock.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise<number>((resolve) => {
|
||||
resolveCreate = resolve;
|
||||
}),
|
||||
);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const controller = new AbortController();
|
||||
const pending = sm.start("aa11", { signal: controller.signal });
|
||||
|
||||
controller.abort();
|
||||
resolveCreate(777);
|
||||
|
||||
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
|
||||
expect(aw.removeMock).toHaveBeenCalledWith(777);
|
||||
expect(sm.has("aa11")).toBe(false);
|
||||
});
|
||||
|
||||
it("removes an incomplete Agent Window when active-tab setup fails", async () => {
|
||||
const aw = fakeAgentWindow();
|
||||
aw.ensureActiveTabMock.mockRejectedValueOnce(new Error("tab setup failed"));
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
|
||||
await expect(sm.start("aa11")).rejects.toThrow("tab setup failed");
|
||||
|
||||
expect(aw.removeMock).toHaveBeenCalledWith(100);
|
||||
expect(sm.has("aa11")).toBe(false);
|
||||
});
|
||||
|
||||
it("surfaces the orphan Agent Window id when startup cleanup fails", async () => {
|
||||
const aw = fakeAgentWindow();
|
||||
aw.ensureActiveTabMock.mockRejectedValueOnce(new Error("tab setup failed"));
|
||||
aw.removeMock.mockRejectedValueOnce(new Error("window removal denied"));
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
|
||||
await expect(sm.start("aa11")).rejects.toMatchObject({
|
||||
name: "SessionStartCleanupError",
|
||||
windowId: 100,
|
||||
message: expect.stringMatching(/cleanup of Agent Window 100 failed.*window removal denied/),
|
||||
});
|
||||
expect(sm.has("aa11")).toBe(false);
|
||||
});
|
||||
|
||||
it("stop() closes the Agent Window and forgets the session", async () => {
|
||||
const aw = fakeAgentWindow();
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
|
||||
@@ -33,19 +33,4 @@ describe("RefStore", () => {
|
||||
expect(s.resolve("@e11")).toBe(11);
|
||||
expect(s.size()).toBe(2);
|
||||
});
|
||||
|
||||
it("preserves the child CDP session as part of ref identity", () => {
|
||||
const s = new RefStore();
|
||||
s.set("e1", 42, {
|
||||
tabId: 7,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
expect(s.resolveEntry("e1")).toMatchObject({
|
||||
backendNodeId: 42,
|
||||
tabId: 7,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -43,11 +43,12 @@ export const chromeAgentWindowApi: AgentWindowApi = {
|
||||
return win.id;
|
||||
},
|
||||
async remove(windowId: number): Promise<void> {
|
||||
// Callers decide whether a missing/failed removal is benign. In
|
||||
// particular, transactional session-start cleanup must be able to
|
||||
// surface a window it could not remove instead of reporting a false
|
||||
// cancellation success while the Agent Window remains open.
|
||||
await chrome.windows.remove(windowId);
|
||||
try {
|
||||
await chrome.windows.remove(windowId);
|
||||
} catch (err) {
|
||||
// Window may have been closed by the user already; ignore.
|
||||
console.debug("[bh] chrome.windows.remove failed", err);
|
||||
}
|
||||
},
|
||||
async ensureActiveTab(windowId: number, url: string): Promise<void> {
|
||||
const tabs = await chrome.tabs.query({ windowId });
|
||||
|
||||
@@ -31,38 +31,6 @@ export interface SessionStartOptions {
|
||||
size?: { width: number; height: number };
|
||||
/** Defaults to true so existing clients keep visible Agent Windows. */
|
||||
focused?: boolean;
|
||||
/** Cancellation for the transactional Agent Window startup sequence. */
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export class SessionStartCleanupError extends Error {
|
||||
readonly windowId: number;
|
||||
readonly startupError: unknown;
|
||||
readonly cleanupError: unknown;
|
||||
|
||||
constructor(windowId: number, startupError: unknown, cleanupError: unknown) {
|
||||
const startupMessage =
|
||||
startupError instanceof Error ? startupError.message : String(startupError);
|
||||
const cleanupMessage =
|
||||
cleanupError instanceof Error ? cleanupError.message : String(cleanupError);
|
||||
super(
|
||||
`session_start failed (${startupMessage}) and cleanup of Agent Window ${windowId} failed: ${cleanupMessage}`,
|
||||
);
|
||||
this.name = "SessionStartCleanupError";
|
||||
this.windowId = windowId;
|
||||
this.startupError = startupError;
|
||||
this.cleanupError = cleanupError;
|
||||
}
|
||||
}
|
||||
|
||||
function sessionStartAbortError(): Error {
|
||||
const error = new Error("session_start aborted");
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
function throwIfSessionStartAborted(signal: AbortSignal | undefined): void {
|
||||
if (signal?.aborted) throw sessionStartAbortError();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -172,36 +140,18 @@ export class SessionManager {
|
||||
if (this.sessions.has(sessionId)) {
|
||||
throw new Error(`[bh] session ${sessionId} already exists`);
|
||||
}
|
||||
throwIfSessionStartAborted(opts.signal);
|
||||
|
||||
let windowId: number | null = null;
|
||||
try {
|
||||
const { signal: _signal, ...createOptions } = opts;
|
||||
windowId = await this.agentWindow.create(AGENT_WINDOW_HOME, createOptions);
|
||||
throwIfSessionStartAborted(opts.signal);
|
||||
await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
|
||||
throwIfSessionStartAborted(opts.signal);
|
||||
|
||||
const ctx: SessionContext = {
|
||||
sessionId,
|
||||
agentWindowId: windowId,
|
||||
refStore: new RefStore(),
|
||||
borrowedTabs: new Map(),
|
||||
createdAtMs: this.now(),
|
||||
};
|
||||
this.sessions.set(sessionId, ctx);
|
||||
this.windowIndex.set(windowId, sessionId);
|
||||
return ctx;
|
||||
} catch (startupError) {
|
||||
if (windowId !== null) {
|
||||
try {
|
||||
await this.agentWindow.remove(windowId);
|
||||
} catch (cleanupError) {
|
||||
throw new SessionStartCleanupError(windowId, startupError, cleanupError);
|
||||
}
|
||||
}
|
||||
throw startupError;
|
||||
}
|
||||
const windowId = await this.agentWindow.create(AGENT_WINDOW_HOME, opts);
|
||||
await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
|
||||
const ctx: SessionContext = {
|
||||
sessionId,
|
||||
agentWindowId: windowId,
|
||||
refStore: new RefStore(),
|
||||
borrowedTabs: new Map(),
|
||||
createdAtMs: this.now(),
|
||||
};
|
||||
this.sessions.set(sessionId, ctx);
|
||||
this.windowIndex.set(windowId, sessionId);
|
||||
return ctx;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
/**
|
||||
* Per-session map from `@e<N>` snapshot refs to a CDP node address.
|
||||
* Per-session map from `@e<N>` snapshot refs to CDP `backendNodeId`.
|
||||
*
|
||||
* Each fresh `tool.snapshot` resets the store: M6 will call
|
||||
* `replace(...)` with the new ref → node address pairs. A node address
|
||||
* includes the owning flat CDP session/frame when the element lives in
|
||||
* an OOPIF; tools resolve live geometry from that identity at call time.
|
||||
* `replace(...)` with the new ref → backendNodeId pairs. Tools like
|
||||
* `tool.click` consume the store via `resolve("@e1")`.
|
||||
*
|
||||
* Refs are session-scoped (§7): looking up a ref in the wrong session
|
||||
* returns `null`, never silently leaks. Storing values in different
|
||||
@@ -16,19 +15,10 @@ export type BackendNodeId = number;
|
||||
export interface RefEntry {
|
||||
backendNodeId: BackendNodeId;
|
||||
tabId: number | null;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
generation: number;
|
||||
}
|
||||
|
||||
export type RefInput =
|
||||
| BackendNodeId
|
||||
| {
|
||||
backendNodeId: BackendNodeId;
|
||||
tabId: number;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
};
|
||||
type RefInput = BackendNodeId | { backendNodeId: BackendNodeId; tabId: number };
|
||||
|
||||
export class RefStore {
|
||||
private readonly map = new Map<string, RefEntry>();
|
||||
@@ -54,7 +44,7 @@ export class RefStore {
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace the entire store with a new ref → CDP node identity mapping.
|
||||
* Replace the entire store with a new ref → backendNodeId mapping.
|
||||
* Used after every fresh `tool.snapshot`.
|
||||
*/
|
||||
replace(entries: Iterable<readonly [string, RefInput]>): void {
|
||||
@@ -63,20 +53,10 @@ export class RefStore {
|
||||
for (const [ref, input] of entries) this.map.set(normaliseRef(ref), this.entry(input));
|
||||
}
|
||||
|
||||
set(
|
||||
ref: string,
|
||||
id: BackendNodeId,
|
||||
opts: {
|
||||
tabId?: number;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
} = {},
|
||||
): void {
|
||||
set(ref: string, id: BackendNodeId, opts: { tabId?: number } = {}): void {
|
||||
this.map.set(normaliseRef(ref), {
|
||||
backendNodeId: id,
|
||||
tabId: opts.tabId ?? null,
|
||||
...(opts.frameId ? { frameId: opts.frameId } : {}),
|
||||
...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}),
|
||||
generation: this.generation,
|
||||
});
|
||||
}
|
||||
@@ -100,8 +80,6 @@ export class RefStore {
|
||||
return {
|
||||
backendNodeId: input.backendNodeId,
|
||||
tabId: input.tabId,
|
||||
...(input.frameId ? { frameId: input.frameId } : {}),
|
||||
...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
|
||||
generation: this.generation,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -612,14 +612,4 @@ describe("isInjectableContentScriptUrl", () => {
|
||||
).toBe(false);
|
||||
expect(isInjectableContentScriptUrl("https://chromewebstore.google.com/")).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects the Edge Add-ons store", () => {
|
||||
expect(
|
||||
isInjectableContentScriptUrl(
|
||||
"https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg",
|
||||
),
|
||||
).toBe(false);
|
||||
// Non-store paths on the same host stay injectable.
|
||||
expect(isInjectableContentScriptUrl("https://microsoftedge.microsoft.com/")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { RECORD_DOCUMENT_TOKEN_ATTR } from "@/lib/record-document-token";
|
||||
import type { CapturedNode } from "@/tools/vom/capture";
|
||||
import { collectDocumentTokenOwners } from "../capture-vom-observation";
|
||||
|
||||
function htmlNode(
|
||||
backendNodeId: number,
|
||||
token: string,
|
||||
ownerFrameBackendNodeId: number | null = null,
|
||||
): CapturedNode {
|
||||
return {
|
||||
backendNodeId,
|
||||
parentBackendNodeId: null,
|
||||
tag: "html",
|
||||
attrs: { [RECORD_DOCUMENT_TOKEN_ATTR]: token },
|
||||
rect: { x: 0, y: 0, w: 1280, h: 720 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
...(ownerFrameBackendNodeId === null ? {} : { ownerFrameBackendNodeId }),
|
||||
};
|
||||
}
|
||||
|
||||
describe("collectDocumentTokenOwners", () => {
|
||||
it("maps document tokens to their immediate iframe owner backend node", () => {
|
||||
const topToken = "top-doc-token";
|
||||
const childToken = "child-doc-token";
|
||||
const owners = collectDocumentTokenOwners({
|
||||
viewport: { width: 1280, height: 720 },
|
||||
nodes: [htmlNode(1, topToken, null)],
|
||||
iframeNodes: new Map([
|
||||
[
|
||||
13,
|
||||
[
|
||||
htmlNode(101, childToken, 13),
|
||||
{
|
||||
backendNodeId: 102,
|
||||
parentBackendNodeId: 101,
|
||||
tag: "input",
|
||||
attrs: {},
|
||||
rect: { x: 10, y: 20, w: 200, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
ownerFrameBackendNodeId: 13,
|
||||
},
|
||||
],
|
||||
],
|
||||
]),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
});
|
||||
|
||||
expect(owners.get(topToken)).toBeNull();
|
||||
expect(owners.get(childToken)).toBe(13);
|
||||
});
|
||||
|
||||
it("maps nested iframe documents to their direct owner, not the outer iframe", () => {
|
||||
const nestedToken = "nested-doc-token";
|
||||
const owners = collectDocumentTokenOwners({
|
||||
viewport: { width: 1280, height: 720 },
|
||||
nodes: [htmlNode(1, "top-token", null)],
|
||||
iframeNodes: new Map([
|
||||
[
|
||||
13,
|
||||
[
|
||||
{
|
||||
backendNodeId: 23,
|
||||
parentBackendNodeId: 101,
|
||||
tag: "iframe",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 400, h: 300 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
ownerFrameBackendNodeId: 13,
|
||||
},
|
||||
],
|
||||
],
|
||||
[23, [htmlNode(201, nestedToken, 23)]],
|
||||
]),
|
||||
excludedBackendNodeIds: new Set(),
|
||||
});
|
||||
|
||||
expect(owners.get(nestedToken)).toBe(23);
|
||||
});
|
||||
});
|
||||
@@ -203,55 +203,6 @@ describe("ToolDispatcher", () => {
|
||||
expect(sent[0]).toEqual({ id: "r-1", result: {} });
|
||||
});
|
||||
|
||||
it("forwards daemon cancellation into an in-flight session stop", async () => {
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const remove = vi.fn(async () => {});
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 4242),
|
||||
remove,
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
await sessions.start("aa11");
|
||||
let finishDetach: () => void = () => {};
|
||||
const detach = new Promise<void>((resolve) => {
|
||||
finishDetach = resolve;
|
||||
});
|
||||
const cdp = {
|
||||
send: vi.fn(),
|
||||
detachSession: vi.fn(() => detach),
|
||||
ensureNetworkCapture: vi.fn(async () => {}),
|
||||
networkEntriesSince: vi.fn(() => ({
|
||||
tab_id: 7,
|
||||
entries: [],
|
||||
next_since: 0,
|
||||
truncated: false,
|
||||
})),
|
||||
setDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
clearDeviceMetricsOverride: vi.fn(async () => {}),
|
||||
setUserAgentOverride: vi.fn(async () => {}),
|
||||
setTouchEmulationEnabled: vi.fn(async () => {}),
|
||||
};
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions, cdp: cdp as TestDispatcherCdp });
|
||||
dispatcher.start();
|
||||
|
||||
deliver(makeRequest("tool.session_stop", { session_id: "aa11" }));
|
||||
await vi.waitFor(() => expect(cdp.detachSession).toHaveBeenCalledWith("aa11"));
|
||||
deliver({ id: "cancel-stop", method: "cancel", params: { rpc_id: "r-1" } });
|
||||
await flushMicrotasks();
|
||||
finishDetach();
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(sent).toContainEqual({ id: "cancel-stop", result: { cancelled: true } });
|
||||
expect(sent).toContainEqual({
|
||||
id: "r-1",
|
||||
error: expect.objectContaining({ code: "cancelled" }),
|
||||
});
|
||||
expect(sessions.has("aa11")).toBe(true);
|
||||
expect(remove).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns not_found when stopping an unknown session", async () => {
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
@@ -726,31 +677,23 @@ describe("ToolDispatcher", () => {
|
||||
await flushMicrotasks();
|
||||
expect(ac?.signal.aborted).toBe(true);
|
||||
|
||||
// Cancel ack arrives synchronously, but the original RPC must not reply
|
||||
// until the in-progress window creation has completed and been rolled back.
|
||||
// Cancel ack arrived synchronously; the slow tool replies with
|
||||
// `cancelled` once the dispatcher's race observes the abort.
|
||||
const ack = sent.find(
|
||||
(m) =>
|
||||
typeof (m as { id?: string }).id === "string" && (m as { id: string }).id === "cancel-1",
|
||||
);
|
||||
expect(ack).toEqual({ id: "cancel-1", result: { cancelled: true } });
|
||||
|
||||
expect(
|
||||
sent.find(
|
||||
(m) => typeof (m as { id?: string }).id === "string" && (m as { id: string }).id === "r-1",
|
||||
),
|
||||
).toBeUndefined();
|
||||
expect(dispatcher.inflightAbortControllers.has("r-1")).toBe(true);
|
||||
|
||||
resolveCreate(9999);
|
||||
await flushMicrotasks();
|
||||
|
||||
const slow = sent.find(
|
||||
(m) => typeof (m as { id?: string }).id === "string" && (m as { id: string }).id === "r-1",
|
||||
);
|
||||
expect(slow).toMatchObject({ id: "r-1", error: { code: "cancelled" } });
|
||||
expect(dispatcher.inflightAbortControllers.has("r-1")).toBe(false);
|
||||
expect(sessions.has("aa44")).toBe(false);
|
||||
expect(sessions.list()).toEqual([]);
|
||||
|
||||
// Drain the dangling create promise so vitest does not warn.
|
||||
resolveCreate(9999);
|
||||
await flushMicrotasks();
|
||||
});
|
||||
|
||||
it("cancel for an unknown rpc_id replies with cancelled=false", async () => {
|
||||
|
||||
@@ -1,54 +1,24 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { nodeContentRegion } from "../element-geometry";
|
||||
import { nodeBoundingRect, quadBoundingRect } from "../element-geometry";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
describe("nodeContentRegion", () => {
|
||||
it("rejects invalid content quads before using the box model fallback", async () => {
|
||||
const send = async (_tabId: number, method: string) => {
|
||||
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 0, 0, 0, 0, 0, 0]] };
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [5, 10, 25, 10, 25, 30, 5, 30] } };
|
||||
}
|
||||
throw new Error(`unexpected CDP call ${method}`);
|
||||
};
|
||||
|
||||
await expect(nodeContentRegion({ send: send as CdpRunner["send"] }, 7, 555)).resolves.toEqual([
|
||||
[
|
||||
{ x: 5, y: 10 },
|
||||
{ x: 25, y: 10 },
|
||||
{ x: 25, y: 30 },
|
||||
{ x: 5, y: 30 },
|
||||
],
|
||||
]);
|
||||
});
|
||||
|
||||
it("preserves every visible content fragment", async () => {
|
||||
const send = async () => ({
|
||||
quads: [
|
||||
[10, 20, 60, 20, 60, 40, 10, 40],
|
||||
[15, 50, 90, 50, 90, 80, 15, 80],
|
||||
],
|
||||
describe("quadBoundingRect", () => {
|
||||
it("computes axis-aligned bounds from an 8-double quad", () => {
|
||||
expect(quadBoundingRect([10, 20, 110, 20, 110, 60, 10, 60])).toEqual({
|
||||
x: 10,
|
||||
y: 20,
|
||||
width: 100,
|
||||
height: 40,
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as CdpRunner["send"],
|
||||
};
|
||||
|
||||
await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([
|
||||
[
|
||||
{ x: 10, y: 20 },
|
||||
{ x: 60, y: 20 },
|
||||
{ x: 60, y: 40 },
|
||||
{ x: 10, y: 40 },
|
||||
],
|
||||
[
|
||||
{ x: 15, y: 50 },
|
||||
{ x: 90, y: 50 },
|
||||
{ x: 90, y: 80 },
|
||||
{ x: 15, y: 80 },
|
||||
],
|
||||
]);
|
||||
});
|
||||
|
||||
it("returns null for degenerate quads", () => {
|
||||
expect(quadBoundingRect([0, 0, 0, 0, 0, 0, 0, 0])).toBeNull();
|
||||
expect(quadBoundingRect([1, 2, 3])).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("nodeBoundingRect", () => {
|
||||
it("falls back to visible descendant bounds for zero-size containers", async () => {
|
||||
const calls: Array<{ method: string; params?: object }> = [];
|
||||
const send = async (_tabId: number, method: string, params?: object) => {
|
||||
@@ -68,14 +38,12 @@ describe("nodeContentRegion", () => {
|
||||
};
|
||||
const cdp = { send: send as CdpRunner["send"] };
|
||||
|
||||
await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([
|
||||
[
|
||||
{ x: 242, y: 468 },
|
||||
{ x: 1011, y: 468 },
|
||||
{ x: 1011, y: 648 },
|
||||
{ x: 242, y: 648 },
|
||||
],
|
||||
]);
|
||||
await expect(nodeBoundingRect(cdp, 7, 555)).resolves.toEqual({
|
||||
x: 242,
|
||||
y: 468,
|
||||
width: 769,
|
||||
height: 180,
|
||||
});
|
||||
expect(calls.map((c) => c.method)).toEqual([
|
||||
"DOM.getContentQuads",
|
||||
"DOM.getBoxModel",
|
||||
|
||||
@@ -1,286 +0,0 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry";
|
||||
import {
|
||||
clipPolygon,
|
||||
polygonArea,
|
||||
polygonBounds,
|
||||
projectAndClipRegion,
|
||||
projectUnitPoint,
|
||||
type Quad,
|
||||
rectPolygon,
|
||||
regionBounds,
|
||||
} from "../geometry";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
describe("frame geometry projection", () => {
|
||||
it("keeps region bounds separate from polygon area", () => {
|
||||
const region = [
|
||||
rectPolygon({ x: 0, y: 0, w: 10, h: 10 }),
|
||||
rectPolygon({ x: 20, y: 5, w: 5, h: 5 }),
|
||||
];
|
||||
expect(regionBounds(region)).toEqual({ x: 0, y: 0, width: 25, height: 10 });
|
||||
expect(polygonArea(region[0])).toBe(100);
|
||||
});
|
||||
|
||||
it("maps through the iframe content box instead of its border box", () => {
|
||||
const contentQuad: Quad = [
|
||||
{ x: 204, y: 306 },
|
||||
{ x: 604, y: 306 },
|
||||
{ x: 604, y: 506 },
|
||||
{ x: 204, y: 506 },
|
||||
];
|
||||
const projected = projectAndClipRegion(
|
||||
[rectPolygon({ x: 10, y: 20, w: 100, h: 40 })],
|
||||
[{ sourceViewport: { width: 200, height: 100 }, destinationQuad: contentQuad }],
|
||||
{ width: 1000, height: 800 },
|
||||
);
|
||||
expect(polygonBounds(projected.flat())).toEqual({ x: 224, y: 346, width: 200, height: 80 });
|
||||
});
|
||||
|
||||
it("uses a projective mapping for perspective-transformed iframe quads", () => {
|
||||
const quad: Quad = [
|
||||
{ x: 0, y: 0 },
|
||||
{ x: 200, y: 20 },
|
||||
{ x: 180, y: 140 },
|
||||
{ x: 20, y: 100 },
|
||||
];
|
||||
expect(projectUnitPoint({ x: 0, y: 0 }, quad)).toEqual(quad[0]);
|
||||
expect(projectUnitPoint({ x: 1, y: 0 }, quad)).toEqual(quad[1]);
|
||||
expect(projectUnitPoint({ x: 1, y: 1 }, quad)).toEqual(quad[2]);
|
||||
expect(projectUnitPoint({ x: 0, y: 1 }, quad)).toEqual(quad[3]);
|
||||
});
|
||||
|
||||
it("clips against the iframe polygon rather than its bounding box", () => {
|
||||
const frameQuad: Quad = [
|
||||
{ x: 50, y: 0 },
|
||||
{ x: 100, y: 50 },
|
||||
{ x: 50, y: 100 },
|
||||
{ x: 0, y: 50 },
|
||||
];
|
||||
expect(clipPolygon(rectPolygon({ x: 0, y: 0, w: 10, h: 10 }), frameQuad)).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not reapply same-target frame transforms around an OOPIF", async () => {
|
||||
const send = vi.fn(async (_tabId, method, params) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||
}
|
||||
if (method === "DOM.getBoxModel") {
|
||||
const backendNodeId = (params as { backendNodeId?: number }).backendNodeId;
|
||||
return backendNodeId === 20
|
||||
? { model: { content: [110, 70, 310, 70, 310, 170, 110, 170] } }
|
||||
: { model: { content: [100, 50, 400, 50, 400, 250, 100, 250] } };
|
||||
}
|
||||
throw new Error(`unexpected root command ${method}`);
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
throw new Error(`unexpected child command ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
};
|
||||
const projection = await resolveFrameProjection(
|
||||
cdp,
|
||||
{
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "same-process-parent",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4 },
|
||||
},
|
||||
{
|
||||
frameId: "oopif",
|
||||
parentFrameId: "same-process-parent",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 4, sessionId: "oopif-session" },
|
||||
},
|
||||
],
|
||||
},
|
||||
"oopif",
|
||||
);
|
||||
|
||||
expect(projection?.edges).toEqual([
|
||||
{
|
||||
sourceViewport: { width: 200, height: 100 },
|
||||
destinationQuad: [
|
||||
{ x: 110, y: 70 },
|
||||
{ x: 310, y: 70 },
|
||||
{ x: 310, y: 170 },
|
||||
{ x: 110, y: 170 },
|
||||
],
|
||||
destinationClips: [
|
||||
[
|
||||
{ x: 100, y: 50 },
|
||||
{ x: 400, y: 50 },
|
||||
{ x: 400, y: 250 },
|
||||
{ x: 100, y: 250 },
|
||||
],
|
||||
],
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("clips live OOPIF geometry at every frame boundary", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||
}
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||
}
|
||||
throw new Error(`unexpected root command ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[-50, -20, 250, -20, 250, 120, -50, 120]] };
|
||||
}
|
||||
throw new Error(`unexpected child command ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const geometry = await resolveNodeGeometry(cdp, 4, {
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
frameId: "child",
|
||||
backendNodeId: 101,
|
||||
});
|
||||
|
||||
expect(geometry).toMatchObject({
|
||||
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
||||
actionPoint: { x: 200, y: 150 },
|
||||
});
|
||||
});
|
||||
|
||||
it("reuses one frame graph while scrolling and projecting live geometry", async () => {
|
||||
const getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
}));
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||
}
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||
}
|
||||
throw new Error(`unexpected root command ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||
}
|
||||
throw new Error(`unexpected child command ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
getFrameGraph,
|
||||
};
|
||||
|
||||
await expect(
|
||||
resolveNodeGeometry(
|
||||
cdp,
|
||||
4,
|
||||
{
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
frameId: "child",
|
||||
backendNodeId: 101,
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
),
|
||||
).resolves.not.toMatchObject({ code: expect.any(String) });
|
||||
expect(getFrameGraph).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("clips live geometry to same-target iframe ancestors", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||
}
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||
}
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[50, 50, 350, 50, 350, 250, 50, 250]] };
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4 },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const geometry = await resolveNodeGeometry(cdp, 4, {
|
||||
target: { tabId: 4 },
|
||||
frameId: "child",
|
||||
backendNodeId: 101,
|
||||
});
|
||||
|
||||
expect(geometry).toMatchObject({
|
||||
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
||||
actionPoint: { x: 200, y: 150 },
|
||||
});
|
||||
});
|
||||
|
||||
it("fails closed when an OOPIF frame graph is unavailable", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(),
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
};
|
||||
|
||||
await expect(
|
||||
resolveNodeGeometry(cdp, 4, {
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
frameId: "child",
|
||||
backendNodeId: 101,
|
||||
}),
|
||||
).resolves.toMatchObject({ code: "cdp_failed" });
|
||||
});
|
||||
});
|
||||
@@ -1,6 +1,5 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { RefStore } from "@/session-manager/ref-store";
|
||||
import type { RequestHelpParams } from "@/transport/types";
|
||||
import { handleRequestHelp, type RequestHelpDeps, resetHelpLifecycleForTests } from "../human-loop";
|
||||
|
||||
@@ -51,10 +50,11 @@ function installHelpLifecycleChrome() {
|
||||
}
|
||||
|
||||
function fakeManager(sessionId: string, agentWindowId: number, tabId: number) {
|
||||
const refStore = new RefStore();
|
||||
const mgr = {
|
||||
get: (id: string) =>
|
||||
id === sessionId ? { sessionId, agentWindowId, refStore, borrowedTabs: new Map() } : null,
|
||||
id === sessionId
|
||||
? { sessionId, agentWindowId, refStore: { resolve: () => null }, borrowedTabs: new Map() }
|
||||
: null,
|
||||
findByWindowId: (wid: number) => (wid === agentWindowId ? { sessionId } : null),
|
||||
} as unknown as SessionManager;
|
||||
return mgr;
|
||||
@@ -339,23 +339,26 @@ describe("handleRequestHelp", () => {
|
||||
|
||||
ac.abort();
|
||||
await expect(pending).resolves.toMatchObject({ code: "cancelled" });
|
||||
expect(cdpSend).not.toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
"DOM.removeAttribute",
|
||||
expect.anything(),
|
||||
expect(cdpSend).toHaveBeenCalledWith(
|
||||
5,
|
||||
"DOM.querySelectorAll",
|
||||
expect.objectContaining({ selector: "[data-bsk-help]" }),
|
||||
);
|
||||
expect(cdpSend).toHaveBeenCalledWith(
|
||||
6,
|
||||
"DOM.querySelectorAll",
|
||||
expect.objectContaining({ selector: "[data-bsk-help]" }),
|
||||
);
|
||||
});
|
||||
|
||||
it("resolves ref targets to explicit viewport rectangles without mutating the page", async () => {
|
||||
const refStore = new RefStore();
|
||||
refStore.set("e1", 42, { tabId: 5 });
|
||||
it("tags ref targets via CDP and reports them matched", async () => {
|
||||
const mgr = {
|
||||
get: (id: string) =>
|
||||
id === "abcd"
|
||||
? {
|
||||
sessionId: "abcd",
|
||||
agentWindowId: 99,
|
||||
refStore,
|
||||
refStore: { resolve: () => 42 },
|
||||
borrowedTabs: new Map(),
|
||||
}
|
||||
: null,
|
||||
@@ -363,17 +366,7 @@ describe("handleRequestHelp", () => {
|
||||
} as unknown as SessionManager;
|
||||
const deps = baseDeps({
|
||||
cdp: {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}),
|
||||
send: vi.fn(async () => ({ object: { objectId: "obj-1" } })),
|
||||
} as unknown as RequestHelpDeps["cdp"],
|
||||
});
|
||||
const res = await handleRequestHelp(
|
||||
@@ -382,97 +375,14 @@ describe("handleRequestHelp", () => {
|
||||
deps,
|
||||
);
|
||||
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
|
||||
expect(sentMsg.selectors).toEqual([]);
|
||||
expect(sentMsg.rects).toEqual([{ top: 20, left: 10, width: 100, height: 40 }]);
|
||||
expect(sentMsg.selectors).toContain('[data-bsk-help="0"]');
|
||||
expect(res).toMatchObject({
|
||||
outcome: "continued",
|
||||
tab_id: 5,
|
||||
resolved_targets: [
|
||||
{
|
||||
matched: true,
|
||||
ref: "@e1",
|
||||
},
|
||||
],
|
||||
resolved_targets: [{ matched: true, ref: "@e1" }],
|
||||
});
|
||||
});
|
||||
|
||||
it("projects an OOPIF ref through its live content quad for the help overlay", async () => {
|
||||
const mgr = {
|
||||
get: (id: string) =>
|
||||
id === "abcd"
|
||||
? {
|
||||
sessionId: "abcd",
|
||||
agentWindowId: 99,
|
||||
refStore: {
|
||||
resolveEntry: () => ({
|
||||
backendNodeId: 42,
|
||||
tabId: 5,
|
||||
frameId: "child",
|
||||
cdpSessionId: "child-session",
|
||||
generation: 1,
|
||||
}),
|
||||
},
|
||||
borrowedTabs: new Map(),
|
||||
}
|
||||
: null,
|
||||
findByWindowId: (wid: number) => (wid === 99 ? { sessionId: "abcd" } : null),
|
||||
} as unknown as SessionManager;
|
||||
const send = vi.fn(async (_tabId, method) => {
|
||||
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [204, 306, 604, 306, 604, 506, 204, 506] } };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
throw new Error(`unexpected root ${method}`);
|
||||
});
|
||||
const sendToTarget = vi.fn(async (_target, method) => {
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
throw new Error(`unexpected child ${method}`);
|
||||
});
|
||||
const deps = baseDeps({
|
||||
cdp: {
|
||||
send,
|
||||
sendToTarget,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 5 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 99,
|
||||
target: { tabId: 5, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
} as unknown as NonNullable<RequestHelpDeps["cdp"]>,
|
||||
});
|
||||
|
||||
const res = await handleRequestHelp(
|
||||
mgr,
|
||||
baseParams({ tab_id: 5, targets: [{ ref: "@e1" }] }),
|
||||
deps,
|
||||
);
|
||||
|
||||
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
|
||||
expect(sentMsg.rects).toEqual([{ top: 346, left: 224, width: 200, height: 80 }]);
|
||||
expect(res).toMatchObject({ resolved_targets: [{ matched: true, ref: "@e1" }] });
|
||||
expect(sendToTarget).toHaveBeenCalledWith(
|
||||
{ tabId: 5, sessionId: "child-session" },
|
||||
"DOM.getContentQuads",
|
||||
{ backendNodeId: 42 },
|
||||
);
|
||||
});
|
||||
|
||||
it("reports ref target unmatched when ref does not resolve", async () => {
|
||||
const deps = baseDeps();
|
||||
const res = await handleRequestHelp(
|
||||
@@ -488,15 +398,16 @@ describe("handleRequestHelp", () => {
|
||||
});
|
||||
|
||||
it("reports ref target unmatched when ref is for another tab", async () => {
|
||||
const refStore = new RefStore();
|
||||
refStore.set("e1", 42, { tabId: 4 });
|
||||
const mgr = {
|
||||
get: (id: string) =>
|
||||
id === "abcd"
|
||||
? {
|
||||
sessionId: "abcd",
|
||||
agentWindowId: 99,
|
||||
refStore,
|
||||
refStore: {
|
||||
resolve: (ref: string, opts: { tabId?: number }) =>
|
||||
ref === "e1" && opts.tabId === 4 ? 42 : null,
|
||||
},
|
||||
borrowedTabs: new Map(),
|
||||
}
|
||||
: null,
|
||||
|
||||
@@ -30,9 +30,6 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
||||
const sendImpl = async (tabId: number, method: string, params?: object) => {
|
||||
sent.push({ tabId, method, params });
|
||||
const h = handlers[method];
|
||||
if (!h && method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
if (!h) throw new Error(`unexpected CDP call ${method}`);
|
||||
return h(params);
|
||||
};
|
||||
@@ -167,63 +164,6 @@ describe("handleClick", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves frame refs in their CDP session and dispatches input in top coordinates", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
const fake = makeFakeCdp({
|
||||
"Input.dispatchMouseEvent": () => ({}),
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.getBoxModel": () => ({
|
||||
model: { content: [204, 306, 604, 306, 604, 506, 204, 506] },
|
||||
}),
|
||||
});
|
||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child-frame",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 99,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
}));
|
||||
const targetCalls: Array<{ sessionId?: string; method: string }> = [];
|
||||
fake.cdp.sendToTarget = vi.fn(async (target, method) => {
|
||||
targetCalls.push({ sessionId: target.sessionId, method });
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
throw new Error(`unexpected child CDP call ${method}`);
|
||||
}) as CdpRunner["sendToTarget"];
|
||||
|
||||
const res = await handleClick(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e3" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
// The iframe content box starts at (204, 306) after its border and is scaled 2x.
|
||||
expect(res).toMatchObject({ x: 324, y: 386 });
|
||||
expect(targetCalls).toEqual([
|
||||
{ sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" },
|
||||
{ sessionId: "child-session", method: "DOM.getContentQuads" },
|
||||
{ sessionId: "child-session", method: "Page.getLayoutMetrics" },
|
||||
]);
|
||||
expect(fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength(3);
|
||||
});
|
||||
|
||||
it("enables overlay bypass before mouse events when overlay blocks the click point", async () => {
|
||||
const order: string[] = [];
|
||||
const bypassOverlay = vi.fn(async (_tabId: number, enabled: boolean) => {
|
||||
|
||||
@@ -5,10 +5,8 @@ import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "@/lib/overlay-bridg
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import type { CdpRunner } from "@/tools/shared";
|
||||
import {
|
||||
buildFrameVomScene,
|
||||
buildVomScene,
|
||||
type CdpAxNode,
|
||||
captureVomObservation,
|
||||
handleGetHtml,
|
||||
handleObserve,
|
||||
handleScreenshot,
|
||||
@@ -64,9 +62,6 @@ function makeFakeCdp(handlers: Record<string, (params?: object) => unknown>) {
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
sent.push({ method, params });
|
||||
const handler = handlers[method];
|
||||
if (!handler && method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
if (!handler) throw new Error(`unexpected CDP call ${method}`);
|
||||
return handler(params);
|
||||
});
|
||||
@@ -513,292 +508,6 @@ describe("handleScreenshot overlay suppression", () => {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("buildVomScene", () => {
|
||||
it("merges multiple sibling and nested frames without mixing their targets", () => {
|
||||
const node = (
|
||||
backendNodeId: number,
|
||||
parentBackendNodeId: number | null,
|
||||
frameId: string,
|
||||
tag: string,
|
||||
attrs: Record<string, string> = {},
|
||||
): CapturedNode => ({
|
||||
backendNodeId,
|
||||
parentBackendNodeId,
|
||||
frameId,
|
||||
tag,
|
||||
attrs,
|
||||
rect: { x: 0, y: 0, w: 100, h: 30 },
|
||||
localRect: { x: 0, y: 0, w: 100, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
});
|
||||
const mainNodes = [
|
||||
node(1, null, "main", "body"),
|
||||
node(10, 1, "main", "iframe", { title: "First" }),
|
||||
node(20, 1, "main", "iframe", { title: "Second" }),
|
||||
];
|
||||
const firstNodes = [
|
||||
node(100, null, "first", "body"),
|
||||
node(101, 100, "first", "button", { "aria-label": "First action" }),
|
||||
];
|
||||
const secondNodes = [
|
||||
node(200, null, "second", "body"),
|
||||
node(201, 200, "second", "h2"),
|
||||
node(210, 200, "second", "iframe", { title: "Nested" }),
|
||||
];
|
||||
const nestedNodes = [
|
||||
node(300, null, "nested", "body"),
|
||||
node(301, 300, "nested", "input", { placeholder: "Nested field" }),
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
viewport: { width: 1200, height: 800 },
|
||||
nodes: mainNodes,
|
||||
iframeNodes: new Map([
|
||||
[10, firstNodes],
|
||||
[20, secondNodes],
|
||||
[210, nestedNodes],
|
||||
]),
|
||||
frameNodes: new Map([
|
||||
["main", mainNodes],
|
||||
["first", firstNodes],
|
||||
["second", secondNodes],
|
||||
["nested", nestedNodes],
|
||||
]),
|
||||
frameOwnerBackendNodeIds: new Map([
|
||||
["first", 10],
|
||||
["second", 20],
|
||||
["nested", 210],
|
||||
]),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
const axNode = (
|
||||
nodeId: string,
|
||||
backendDOMNodeId: number,
|
||||
role: string,
|
||||
parentId?: string,
|
||||
name?: string,
|
||||
): CdpAxNode => ({
|
||||
nodeId,
|
||||
backendDOMNodeId,
|
||||
role: { type: "role", value: role },
|
||||
...(parentId ? { parentId } : {}),
|
||||
...(name ? { name: { type: "computedString", value: name } } : {}),
|
||||
});
|
||||
const result = buildFrameVomScene(
|
||||
[
|
||||
{
|
||||
frameId: "main",
|
||||
contextScopeId: "main",
|
||||
target: { tabId: 7 },
|
||||
domNodes: mainNodes,
|
||||
axNodes: [
|
||||
axNode("root", 1, "RootWebArea"),
|
||||
axNode("frame-1", 10, "Iframe", "root", "First"),
|
||||
axNode("frame-2", 20, "Iframe", "root", "Second"),
|
||||
],
|
||||
},
|
||||
{
|
||||
frameId: "first",
|
||||
contextScopeId: "first",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 7, sessionId: "session-first" },
|
||||
domNodes: firstNodes,
|
||||
axNodes: [
|
||||
axNode("root", 100, "RootWebArea"),
|
||||
axNode("button", 101, "button", "root", "First action"),
|
||||
],
|
||||
},
|
||||
{
|
||||
frameId: "second",
|
||||
contextScopeId: "second",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 7, sessionId: "session-second" },
|
||||
domNodes: secondNodes,
|
||||
axNodes: [
|
||||
axNode("root", 200, "RootWebArea"),
|
||||
axNode("heading", 201, "heading", "root", "Second heading"),
|
||||
axNode("nested-frame", 210, "Iframe", "root", "Nested"),
|
||||
],
|
||||
},
|
||||
{
|
||||
frameId: "nested",
|
||||
contextScopeId: "nested",
|
||||
parentFrameId: "second",
|
||||
ownerBackendNodeId: 210,
|
||||
target: { tabId: 7, sessionId: "session-nested" },
|
||||
domNodes: nestedNodes,
|
||||
axNodes: [
|
||||
axNode("root", 300, "RootWebArea"),
|
||||
axNode("input", 301, "textbox", "root", "Nested field"),
|
||||
],
|
||||
},
|
||||
],
|
||||
captured,
|
||||
);
|
||||
|
||||
const rendered = renderVom(result);
|
||||
expect(rendered.text).toContain('Iframe "First"');
|
||||
expect(rendered.text).toContain('@e1 button "First action"');
|
||||
expect(rendered.text).toContain('heading "Second heading"');
|
||||
expect(rendered.text).toContain('Iframe "Nested"');
|
||||
expect(rendered.text).toContain('textbox "Nested field"');
|
||||
expect(rendered.refs.find((ref) => ref.backendNodeId === 101)?.frameId).toBe("first");
|
||||
expect(rendered.refs.find((ref) => ref.backendNodeId === 301)?.frameId).toBe("nested");
|
||||
|
||||
const firstAction = result.nodes.find((item) => item.backendNodeId === 101);
|
||||
const nestedInput = result.nodes.find((item) => item.backendNodeId === 301);
|
||||
expect(firstAction).toMatchObject({ frameId: "first", contextScopeId: "first" });
|
||||
expect(nestedInput).toMatchObject({ frameId: "nested", contextScopeId: "nested" });
|
||||
expect(nestedInput?.parentId).toBe(result.nodes.find((item) => item.backendNodeId === 210)?.id);
|
||||
});
|
||||
|
||||
it("does not place a child document at the page root when its frame boundary is unresolved", () => {
|
||||
const childNode: CapturedNode = {
|
||||
backendNodeId: 101,
|
||||
parentBackendNodeId: null,
|
||||
frameId: "child",
|
||||
tag: "button",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 100, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
};
|
||||
const captured: CapturedViewModel = {
|
||||
viewport: { width: 800, height: 600 },
|
||||
nodes: [],
|
||||
iframeNodes: new Map(),
|
||||
frameNodes: new Map([["child", [childNode]]]),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
|
||||
const result = buildFrameVomScene(
|
||||
[
|
||||
{
|
||||
frameId: "main",
|
||||
contextScopeId: "main",
|
||||
target: { tabId: 7 },
|
||||
domNodes: [],
|
||||
axNodes: [],
|
||||
},
|
||||
{
|
||||
frameId: "child",
|
||||
contextScopeId: "child",
|
||||
parentFrameId: "main",
|
||||
target: { tabId: 7 },
|
||||
domNodes: [childNode],
|
||||
axNodes: [
|
||||
{
|
||||
nodeId: "button",
|
||||
frameId: "child",
|
||||
backendDOMNodeId: 101,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Child action" },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
captured,
|
||||
);
|
||||
|
||||
expect(result.nodes).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps AX-only frame content attached to its iframe boundary", () => {
|
||||
const mainNodes: CapturedNode[] = [
|
||||
{
|
||||
backendNodeId: 1,
|
||||
parentBackendNodeId: null,
|
||||
frameId: "main",
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 800, h: 600 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: 1,
|
||||
frameId: "main",
|
||||
tag: "iframe",
|
||||
attrs: { title: "Remote" },
|
||||
rect: { x: 100, y: 100, w: 400, h: 300 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
viewport: { width: 800, height: 600 },
|
||||
nodes: mainNodes,
|
||||
iframeNodes: new Map(),
|
||||
frameNodes: new Map([
|
||||
["main", mainNodes],
|
||||
["child", []],
|
||||
]),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
|
||||
const result = buildFrameVomScene(
|
||||
[
|
||||
{
|
||||
frameId: "main",
|
||||
contextScopeId: "main",
|
||||
target: { tabId: 7 },
|
||||
domNodes: mainNodes,
|
||||
axNodes: [
|
||||
{
|
||||
nodeId: "main-root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
},
|
||||
{
|
||||
nodeId: "frame-owner",
|
||||
parentId: "main-root",
|
||||
backendDOMNodeId: 10,
|
||||
role: { type: "role", value: "Iframe" },
|
||||
name: { type: "computedString", value: "Remote" },
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
frameId: "child",
|
||||
contextScopeId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 7, sessionId: "child-session" },
|
||||
domNodes: [],
|
||||
axNodes: [
|
||||
{
|
||||
nodeId: "child-root",
|
||||
backendDOMNodeId: 100,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
},
|
||||
{
|
||||
nodeId: "child-button",
|
||||
parentId: "child-root",
|
||||
backendDOMNodeId: 101,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "AX fallback action" },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
captured,
|
||||
);
|
||||
|
||||
const childButton = result.nodes.find((node) => node.backendNodeId === 101);
|
||||
expect(childButton).toMatchObject({ frameId: "child", contextScopeId: "child" });
|
||||
expect(childButton?.parentId).toBe(result.nodes.find((node) => node.backendNodeId === 10)?.id);
|
||||
expect(renderVom(result).text).toContain('@e1 button "AX fallback action"');
|
||||
});
|
||||
|
||||
it("joins AX semantics with captured geometry by backendDOMNodeId", () => {
|
||||
const axNodes: CdpAxNode[] = [
|
||||
{
|
||||
@@ -1107,7 +816,6 @@ describe("buildVomScene", () => {
|
||||
parentId: "2",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "x", value: "输入密码" },
|
||||
value: { value: "iframe-secret" },
|
||||
backendDOMNodeId: 202,
|
||||
properties: [{ name: "inputType", value: { value: "password" } }],
|
||||
},
|
||||
@@ -1162,10 +870,6 @@ describe("buildVomScene", () => {
|
||||
|
||||
expect(passwordNodes).toHaveLength(1);
|
||||
expect(passwordNodes[0]).toEqual(expect.objectContaining({ id: 202, sensitive: true }));
|
||||
expect(passwordNodes[0].value).toBeUndefined();
|
||||
const rendered = renderVom(buildVomScene(axNodes, captured)).text;
|
||||
expect(rendered).toContain('textbox "输入密码" [filled] ="•••"');
|
||||
expect(rendered).not.toContain("iframe-secret");
|
||||
});
|
||||
|
||||
it("keeps unnamed iframe controls but skips unnamed iframe links", () => {
|
||||
@@ -2757,64 +2461,6 @@ describe("handleSnapshot", () => {
|
||||
expect(ctx.refStore.resolve("e1")).toBeNull();
|
||||
});
|
||||
|
||||
it("keeps the previous RefStore when cancellation lands during DOM capture", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e1", 999, { tabId: 4 });
|
||||
const controller = new AbortController();
|
||||
let resolveCapture: (value: unknown) => void = () => {};
|
||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") {
|
||||
return {
|
||||
nodes: [
|
||||
{
|
||||
nodeId: "new",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "New" },
|
||||
backendDOMNodeId: 123,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return new Promise((resolve) => {
|
||||
resolveCapture = resolve;
|
||||
});
|
||||
}
|
||||
throw new Error(`unexpected CDP method ${method}`);
|
||||
});
|
||||
const deps = {
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
};
|
||||
|
||||
const pending = handleSnapshot(sm, { session_id: "aa11" }, deps, controller.signal);
|
||||
await vi.waitFor(() =>
|
||||
expect(send).toHaveBeenCalledWith(4, "DOMSnapshot.captureSnapshot", expect.any(Object)),
|
||||
);
|
||||
controller.abort();
|
||||
resolveCapture({ strings: [], documents: [] });
|
||||
|
||||
await expect(pending).resolves.toMatchObject({ code: "cancelled" });
|
||||
expect(ctx.refStore.resolve("e1", { tabId: 4 })).toBe(999);
|
||||
expect(ctx.refStore.resolve("e2")).toBeNull();
|
||||
});
|
||||
|
||||
it("surfaces CDP failures as cdp_failed", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
@@ -2909,360 +2555,10 @@ describe("handleSnapshot", () => {
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function secretsSnapshotReply(secrets: {
|
||||
email: string;
|
||||
password: string;
|
||||
otp: string;
|
||||
card: string;
|
||||
}) {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"input",
|
||||
"position",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"type",
|
||||
"text",
|
||||
"password",
|
||||
"autocomplete",
|
||||
"one-time-code",
|
||||
"cc-number",
|
||||
"value",
|
||||
secrets.email,
|
||||
secrets.password,
|
||||
secrets.otp,
|
||||
secrets.card,
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const style = [i("static"), i("auto"), i("auto")];
|
||||
return {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 1, 1, 1],
|
||||
nodeName: [i("html"), i("body"), i("input"), i("input"), i("input"), i("input")],
|
||||
backendNodeId: [10, 11, 12, 13, 14, 15],
|
||||
attributes: [
|
||||
[],
|
||||
[],
|
||||
[i("type"), i("text"), i("value"), i(secrets.email)],
|
||||
[i("type"), i("password"), i("value"), i(secrets.password)],
|
||||
[
|
||||
i("type"),
|
||||
i("text"),
|
||||
i("autocomplete"),
|
||||
i("one-time-code"),
|
||||
i("value"),
|
||||
i(secrets.otp),
|
||||
],
|
||||
[
|
||||
i("type"),
|
||||
i("text"),
|
||||
i("autocomplete"),
|
||||
i("cc-number"),
|
||||
i("value"),
|
||||
i(secrets.card),
|
||||
],
|
||||
],
|
||||
inputValue: {
|
||||
index: [2, 3, 4, 5],
|
||||
value: [i(secrets.email), i(secrets.password), i(secrets.otp), i(secrets.card)],
|
||||
},
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 3, 4, 5],
|
||||
styles: [style, style, style, style, style],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[10, 10, 200, 32],
|
||||
[10, 50, 200, 32],
|
||||
[10, 90, 200, 32],
|
||||
[10, 130, 200, 32],
|
||||
],
|
||||
paintOrders: [0, 1, 2, 3, 4],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
const VP_METRICS = {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
|
||||
function makeFrameAwareDeps() {
|
||||
const strings = ["html", "body", "iframe", "button", "title", "Remote", "static", "auto"];
|
||||
const i = (value: string) => strings.indexOf(value);
|
||||
const styles = [i("static"), i("auto"), i("auto")];
|
||||
const snapshot = {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
frameId: "main",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("iframe")],
|
||||
backendNodeId: [10, 11, 12],
|
||||
attributes: [[], [], [i("title"), i("Remote")]],
|
||||
contentDocumentIndex: { index: [2], value: [1] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1, 2],
|
||||
styles: [styles, styles, styles],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[0, 0, 1000, 800],
|
||||
[100, 100, 400, 300],
|
||||
],
|
||||
paintOrders: [0, 0, 1],
|
||||
},
|
||||
},
|
||||
{
|
||||
frameId: "child",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("button")],
|
||||
backendNodeId: [20, 21, 22],
|
||||
attributes: [[], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1, 2],
|
||||
styles: [styles, styles, styles],
|
||||
bounds: [
|
||||
[0, 0, 400, 300],
|
||||
[0, 0, 400, 300],
|
||||
[20, 30, 120, 40],
|
||||
],
|
||||
paintOrders: [0, 0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const mainAx: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "main-root",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 11,
|
||||
childIds: ["frame-owner"],
|
||||
},
|
||||
{
|
||||
nodeId: "frame-owner",
|
||||
parentId: "main-root",
|
||||
role: { type: "role", value: "Iframe" },
|
||||
name: { type: "computedString", value: "Remote" },
|
||||
backendDOMNodeId: 12,
|
||||
},
|
||||
];
|
||||
const childAx: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "child-root",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 21,
|
||||
childIds: ["child-button"],
|
||||
},
|
||||
{
|
||||
nodeId: "child-button",
|
||||
parentId: "child-root",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Frame action" },
|
||||
backendDOMNodeId: 22,
|
||||
},
|
||||
];
|
||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "Page.getLayoutMetrics") return VP_METRICS;
|
||||
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: mainAx };
|
||||
throw new Error(`unexpected root CDP method ${method}`);
|
||||
});
|
||||
const sendToTarget = vi.fn(async (_target, method: string) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: childAx };
|
||||
throw new Error(`unexpected child CDP method ${method}`);
|
||||
});
|
||||
const cdp = {
|
||||
send: send as unknown as CdpRunner["send"],
|
||||
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 12,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
trackSessionTab: vi.fn(),
|
||||
} satisfies CdpRunner;
|
||||
return {
|
||||
cdp,
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
it("exposes the frame-aware observation result for recording", async () => {
|
||||
const ax: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "root",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 11,
|
||||
childIds: ["password"],
|
||||
},
|
||||
{
|
||||
nodeId: "password",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Password" },
|
||||
value: { value: "hunter2" },
|
||||
backendDOMNodeId: 13,
|
||||
},
|
||||
];
|
||||
const deps = makeOverlayDeps(ax, loginSnapshotReply(), VP_METRICS);
|
||||
|
||||
const result = await captureVomObservation(deps.cdp, 4, "https://example.com", {
|
||||
redactValues: true,
|
||||
});
|
||||
|
||||
expect(result.text).not.toContain("hunter2");
|
||||
expect(result.text).toContain("•••");
|
||||
expect(result.refs[0]).toMatchObject({
|
||||
backendNodeId: 13,
|
||||
role: "textbox",
|
||||
name: "Password",
|
||||
line: expect.any(Number),
|
||||
});
|
||||
expect(result.rootFrameId).toBe("root");
|
||||
expect(result).not.toHaveProperty("frameDocuments");
|
||||
expect(result.frames).toEqual([{ frameId: "root", target: { tabId: 4 } }]);
|
||||
expect(result.matchNodes.find((node) => node.backendNodeId === 13)).toMatchObject({
|
||||
frameId: "root",
|
||||
backendNodeId: 13,
|
||||
tag: "input",
|
||||
rect: { x: 400, y: 300, w: 200, h: 40 },
|
||||
localRect: { x: 400, y: 300, w: 200, h: 40 },
|
||||
});
|
||||
});
|
||||
|
||||
it("does not leak form secrets through the record-safe observation payload", async () => {
|
||||
const secrets = {
|
||||
email: "user@example.com",
|
||||
password: "hunter2",
|
||||
otp: "847291",
|
||||
card: "4111111111111111",
|
||||
};
|
||||
const ax: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "root",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 11,
|
||||
childIds: ["email", "password", "otp", "card"],
|
||||
},
|
||||
{
|
||||
nodeId: "email",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Email" },
|
||||
value: { value: secrets.email },
|
||||
backendDOMNodeId: 12,
|
||||
},
|
||||
{
|
||||
nodeId: "password",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Password" },
|
||||
value: { value: secrets.password },
|
||||
backendDOMNodeId: 13,
|
||||
},
|
||||
{
|
||||
nodeId: "otp",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Code" },
|
||||
value: { value: secrets.otp },
|
||||
backendDOMNodeId: 14,
|
||||
},
|
||||
{
|
||||
nodeId: "card",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Card" },
|
||||
value: { value: secrets.card },
|
||||
backendDOMNodeId: 15,
|
||||
},
|
||||
];
|
||||
const deps = makeOverlayDeps(ax, secretsSnapshotReply(secrets), VP_METRICS);
|
||||
|
||||
const result = await captureVomObservation(deps.cdp, 4, "https://example.com", {
|
||||
redactValues: true,
|
||||
});
|
||||
const dumped = JSON.stringify(result);
|
||||
|
||||
expect(result).not.toHaveProperty("frameDocuments");
|
||||
expect(dumped).not.toContain(secrets.email);
|
||||
expect(dumped).not.toContain(secrets.password);
|
||||
expect(dumped).not.toContain(secrets.otp);
|
||||
expect(dumped).not.toContain(secrets.card);
|
||||
expect(dumped).not.toContain("formValue");
|
||||
expect(dumped).not.toContain("formDefaultValue");
|
||||
expect(dumped).not.toContain("axNodes");
|
||||
expect(dumped).not.toContain("domNodes");
|
||||
expect(dumped).not.toContain("attrs");
|
||||
expect(result.text).toContain("•••");
|
||||
});
|
||||
|
||||
it("keeps frames and rendered refs on the same frame identity", async () => {
|
||||
const deps = makeFrameAwareDeps();
|
||||
|
||||
const result = await captureVomObservation(deps.cdp, 4, "https://example.com");
|
||||
|
||||
expect(result.rootFrameId).toBe("main");
|
||||
expect(result.frames).toHaveLength(2);
|
||||
expect(result.frames.find((frame) => frame.frameId === "child")).toEqual({
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 12,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
});
|
||||
expect(result.matchNodes.find((node) => node.backendNodeId === 22)).toMatchObject({
|
||||
frameId: "child",
|
||||
tag: "button",
|
||||
localRect: { x: 20, y: 30, w: 120, h: 40 },
|
||||
});
|
||||
expect(result.refs.find((ref) => ref.backendNodeId === 22)).toMatchObject({
|
||||
frameId: "child",
|
||||
name: "Frame action",
|
||||
});
|
||||
});
|
||||
|
||||
it("stores iframe refs with their owning CDP session", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
const deps = makeFrameAwareDeps();
|
||||
|
||||
const result = await handleSnapshot(sm, { session_id: "aa11" }, deps);
|
||||
|
||||
if ("code" in result) throw new Error(`unexpected error: ${JSON.stringify(result)}`);
|
||||
expect(result.text).toContain('@e1 button "Frame action"');
|
||||
const frameRef = [...ctx.refStore.entries()].find(([, entry]) => entry.backendNodeId === 22);
|
||||
expect(frameRef?.[1]).toMatchObject({
|
||||
tabId: 4,
|
||||
frameId: "child",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
});
|
||||
|
||||
it("renders a blocking login overlay as the focused top layer", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
@@ -3773,32 +3069,6 @@ describe("handleGetHtml", () => {
|
||||
expect(deps.send).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("reads a frame ref through its child CDP session", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e7", 4242, {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
const deps = makeDeps({});
|
||||
const sendToTarget = vi.fn(async (target, method, params) => {
|
||||
expect(target).toEqual({ tabId: 4, sessionId: "child-session" });
|
||||
expect(method).toBe("DOM.getOuterHTML");
|
||||
expect(params).toEqual({ backendNodeId: 4242 });
|
||||
return { outerHTML: "<button>frame</button>" };
|
||||
});
|
||||
(deps.cdp as CdpRunner).sendToTarget = sendToTarget as unknown as NonNullable<
|
||||
CdpRunner["sendToTarget"]
|
||||
>;
|
||||
|
||||
const res = await handleGetHtml(sm, { session_id: "aa11", ref: "@e7" }, deps);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.html).toBe("<button>frame</button>");
|
||||
expect(deps.send).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns not_found when a ref belongs to another tab", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -60,58 +60,11 @@ function makeApis(
|
||||
}),
|
||||
getLastFocused: vi.fn(async () => ({ id: 500 }) as chrome.windows.Window),
|
||||
create: vi.fn(async () => ({ id: 999 }) as chrome.windows.Window),
|
||||
remove: vi.fn(async () => {}),
|
||||
};
|
||||
return { tabs, windows };
|
||||
}
|
||||
|
||||
describe("handleSessionStop with auto-return", () => {
|
||||
it("leaves the session untouched when cancellation arrived before teardown", async () => {
|
||||
const aw = fakeAgentWindow([100]);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
await sm.start("aa11");
|
||||
const cdp = { detachSession: vi.fn(async () => {}) };
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
|
||||
const res = await handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ cdp, signal: controller.signal },
|
||||
);
|
||||
|
||||
expect(res).toMatchObject({ code: "cancelled" });
|
||||
expect(sm.has("aa11")).toBe(true);
|
||||
expect(cdp.detachSession).not.toHaveBeenCalled();
|
||||
expect(aw.remove).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("does not close the Agent Window when cancellation lands during CDP detach", async () => {
|
||||
const aw = fakeAgentWindow([100]);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
await sm.start("aa11");
|
||||
const controller = new AbortController();
|
||||
let finishDetach: () => void = () => {};
|
||||
const detach = new Promise<void>((resolve) => {
|
||||
finishDetach = resolve;
|
||||
});
|
||||
const cdp = { detachSession: vi.fn(() => detach) };
|
||||
|
||||
const stopping = handleSessionStop(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ cdp, signal: controller.signal },
|
||||
);
|
||||
await vi.waitFor(() => expect(cdp.detachSession).toHaveBeenCalledWith("aa11"));
|
||||
controller.abort();
|
||||
finishDetach();
|
||||
|
||||
const res = await stopping;
|
||||
expect(res).toMatchObject({ code: "cancelled" });
|
||||
expect(sm.has("aa11")).toBe(true);
|
||||
expect(aw.remove).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns every borrowed tab and closes the Agent Window in the right order", async () => {
|
||||
const aw = fakeAgentWindow([100]);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
|
||||
@@ -46,25 +46,6 @@ describe("lookupSnapshotRef", () => {
|
||||
});
|
||||
|
||||
describe("resolveSnapshotRef", () => {
|
||||
it("preserves the frame and child session needed to route iframe refs", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
|
||||
const expected = {
|
||||
backendNodeId: 1234,
|
||||
refKey: "e3",
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
};
|
||||
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
||||
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
||||
});
|
||||
|
||||
it("returns not_found for unknown ref", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
|
||||
@@ -179,7 +179,6 @@ function makeWindowsApi(
|
||||
get: ReturnType<typeof vi.fn>;
|
||||
lastFocused: ReturnType<typeof vi.fn>;
|
||||
create: ReturnType<typeof vi.fn>;
|
||||
remove: ReturnType<typeof vi.fn>;
|
||||
};
|
||||
} {
|
||||
const get = vi.fn(async (windowId: number) => {
|
||||
@@ -198,13 +197,7 @@ function makeWindowsApi(
|
||||
const id = opts?.createWindowId ?? 999;
|
||||
return { id } as chrome.windows.Window;
|
||||
});
|
||||
const remove = vi.fn(async (windowId: number) => {
|
||||
state.windowsClosed.add(windowId);
|
||||
});
|
||||
return {
|
||||
api: { get, getLastFocused: lastFocused, create, remove },
|
||||
spies: { get, lastFocused, create, remove },
|
||||
};
|
||||
return { api: { get, getLastFocused: lastFocused, create }, spies: { get, lastFocused, create } };
|
||||
}
|
||||
|
||||
describe("handleTabCreate", () => {
|
||||
@@ -608,86 +601,6 @@ describe("handleTabReturn", () => {
|
||||
expect(winSpies.create).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("closes a newly created fallback window when cancellation wins before the move", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
|
||||
const state: FakeTabState = {
|
||||
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
|
||||
nextTabId: 50,
|
||||
windowsClosed: new Set([200]),
|
||||
};
|
||||
const { api, spies } = makeTabMutationApi(state);
|
||||
const { api: windowsApi, spies: winSpies } = makeWindowsApi(state, {
|
||||
lastFocused: 100,
|
||||
createWindowId: 777,
|
||||
});
|
||||
let resolveCreate: (window: chrome.windows.Window) => void = () => {};
|
||||
winSpies.create.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
resolveCreate = resolve;
|
||||
}),
|
||||
);
|
||||
const controller = new AbortController();
|
||||
|
||||
const pending = handleTabReturn(
|
||||
sm,
|
||||
{ session_id: "aa11", tab_id: 7 },
|
||||
{ tabs: api, windows: windowsApi, signal: controller.signal },
|
||||
);
|
||||
await vi.waitFor(() => expect(winSpies.create).toHaveBeenCalledOnce());
|
||||
controller.abort();
|
||||
resolveCreate({ id: 777 } as chrome.windows.Window);
|
||||
|
||||
await expect(pending).resolves.toMatchObject({ code: "cancelled" });
|
||||
expect(winSpies.remove).toHaveBeenCalledWith(777);
|
||||
expect(spies.move).not.toHaveBeenCalled();
|
||||
expect(ctx.borrowedTabs.has(7)).toBe(true);
|
||||
});
|
||||
|
||||
it("surfaces the fallback window id when cancellation cleanup fails", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
|
||||
const state: FakeTabState = {
|
||||
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
|
||||
nextTabId: 50,
|
||||
windowsClosed: new Set([200]),
|
||||
};
|
||||
const { api, spies } = makeTabMutationApi(state);
|
||||
const { api: windowsApi, spies: winSpies } = makeWindowsApi(state, {
|
||||
lastFocused: 100,
|
||||
createWindowId: 777,
|
||||
});
|
||||
let resolveCreate: (window: chrome.windows.Window) => void = () => {};
|
||||
winSpies.create.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
resolveCreate = resolve;
|
||||
}),
|
||||
);
|
||||
winSpies.remove.mockRejectedValueOnce(new Error("window removal denied"));
|
||||
const controller = new AbortController();
|
||||
|
||||
const pending = handleTabReturn(
|
||||
sm,
|
||||
{ session_id: "aa11", tab_id: 7 },
|
||||
{ tabs: api, windows: windowsApi, signal: controller.signal },
|
||||
);
|
||||
await vi.waitFor(() => expect(winSpies.create).toHaveBeenCalledOnce());
|
||||
controller.abort();
|
||||
resolveCreate({ id: 777 } as chrome.windows.Window);
|
||||
|
||||
await expect(pending).resolves.toMatchObject({
|
||||
code: "protocol_error",
|
||||
data: { reason: "cleanup_failed", resource_type: "window", resource_id: 777 },
|
||||
message: expect.stringMatching(/fallback window 777.*window removal denied/),
|
||||
});
|
||||
expect(spies.move).not.toHaveBeenCalled();
|
||||
expect(ctx.borrowedTabs.has(7)).toBe(true);
|
||||
});
|
||||
|
||||
it("falls back to a new window when getLastFocused only returns the Agent Window", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
|
||||
@@ -250,10 +250,9 @@ const DEFAULT_NOTIFICATION_COPY: BorrowNotificationCopy = {
|
||||
// URL injection-eligibility test
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const EXTENSION_STORE_RES = [
|
||||
const CHROME_WEB_STORE_RES = [
|
||||
/^https:\/\/chrome\.google\.com\/webstore/i,
|
||||
/^https:\/\/chromewebstore\.google\.com/i,
|
||||
/^https:\/\/microsoftedge\.microsoft\.com\/addons/i,
|
||||
];
|
||||
|
||||
/**
|
||||
@@ -265,15 +264,14 @@ const EXTENSION_STORE_RES = [
|
||||
* - `ftp://` no longer hosts content scripts reliably in modern Chrome.
|
||||
* - `about:` / `chrome:` / `chrome-extension:` / `edge:` / `devtools:` /
|
||||
* `view-source:` / `data:` / `blob:` are blocked by the platform.
|
||||
* - Extension storefronts are blocked by their own browser: the Chrome Web
|
||||
* Store (legacy and new domains) under Chrome, and Edge Add-ons under Edge.
|
||||
* - The Chrome Web Store (both legacy and new domains) is also blocked.
|
||||
*
|
||||
* Mirrors the implicit scheme list documented in
|
||||
* https://developer.chrome.com/docs/extensions/develop/concepts/match-patterns.
|
||||
*/
|
||||
export function isInjectableContentScriptUrl(url: string | undefined): boolean {
|
||||
if (!url) return false;
|
||||
for (const re of EXTENSION_STORE_RES) {
|
||||
for (const re of CHROME_WEB_STORE_RES) {
|
||||
if (re.test(url)) return false;
|
||||
}
|
||||
return /^https?:\/\//i.test(url);
|
||||
|
||||
@@ -1,9 +1,124 @@
|
||||
export {
|
||||
type CaptureVomObservationOptions,
|
||||
captureVomObservation,
|
||||
} from "./observation";
|
||||
export {
|
||||
type CaptureVomFrame,
|
||||
type CaptureVomMatchNode,
|
||||
type CaptureVomObservationResult,
|
||||
} from "./vom/record-safe-observation";
|
||||
import { type RenderedRef, renderVom, type VomOptions } from "@browser-skill/vom";
|
||||
import { readRecordDocumentTokenFromAttrs } from "@/lib/record-document-token";
|
||||
import type { CdpAxNode } from "./observation";
|
||||
import { buildVomScene } from "./observation";
|
||||
import type { CdpRunner } from "./shared";
|
||||
import {
|
||||
type CapturedNode,
|
||||
type CapturedSurfaceProbe,
|
||||
type CapturedViewModel,
|
||||
captureViewModel,
|
||||
collectOverlayExcludedBackendIds,
|
||||
} from "./vom/capture";
|
||||
|
||||
export interface CaptureVomObservationOptions extends VomOptions {
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface CaptureVomObservationResult {
|
||||
text: string;
|
||||
refs: RenderedRef[];
|
||||
truncated: boolean;
|
||||
captured: CapturedNode[];
|
||||
/** All nodes including iframe sub-documents, for geometric target matching. */
|
||||
matchNodes: CapturedNode[];
|
||||
/** Maps ephemeral record document tokens to immediate iframe owner backend ids. */
|
||||
documentTokenOwners: Map<string, number | null>;
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
}
|
||||
|
||||
export function flattenCapturedNodes(vm: CapturedViewModel): CapturedNode[] {
|
||||
return [...vm.nodes, ...[...vm.iframeNodes.values()].flat()];
|
||||
}
|
||||
|
||||
export function collectDocumentTokenOwners(vm: CapturedViewModel): Map<string, number | null> {
|
||||
const owners = new Map<string, number | null>();
|
||||
for (const node of flattenCapturedNodes(vm)) {
|
||||
if (node.tag !== "html") continue;
|
||||
const token = readRecordDocumentTokenFromAttrs(node.attrs);
|
||||
if (!token) continue;
|
||||
owners.set(token, node.ownerFrameBackendNodeId ?? null);
|
||||
}
|
||||
return owners;
|
||||
}
|
||||
|
||||
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
|
||||
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
|
||||
}
|
||||
|
||||
interface LayoutMetricsViewportReply {
|
||||
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
}
|
||||
|
||||
async function fallbackCapturedViewModel(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
): Promise<CapturedViewModel> {
|
||||
let viewport = { width: 0, height: 0 };
|
||||
try {
|
||||
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
viewport = {
|
||||
width: vpSrc.clientWidth ?? 0,
|
||||
height: vpSrc.clientHeight ?? 0,
|
||||
};
|
||||
} catch {
|
||||
// viewport stays zero-sized
|
||||
}
|
||||
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId);
|
||||
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
async function captureForVom(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
conditionalSurfaceProbe: boolean,
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>,
|
||||
): Promise<CapturedViewModel> {
|
||||
try {
|
||||
return await captureViewModel(cdp, tabId, {
|
||||
conditionalSurfaceProbe,
|
||||
hoverProbeBypassOverlay,
|
||||
});
|
||||
} catch {
|
||||
return fallbackCapturedViewModel(cdp, tabId);
|
||||
}
|
||||
}
|
||||
|
||||
/** Pure VOM capture pipeline shared by snapshot, observe, and recording. */
|
||||
export async function captureVomObservation(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
url: string | undefined,
|
||||
opts: CaptureVomObservationOptions = {},
|
||||
): Promise<CaptureVomObservationResult> {
|
||||
await cdp.ensureAttachedToUrl?.(tabId, url);
|
||||
await cdp.send<unknown>(tabId, "Accessibility.enable", {});
|
||||
const result = await cdp.send<{ nodes: CdpAxNode[] }>(tabId, "Accessibility.getFullAXTree", {});
|
||||
const axNodes = result.nodes ?? [];
|
||||
const captured = await captureForVom(
|
||||
cdp,
|
||||
tabId,
|
||||
opts.conditionalSurfaceProbe ?? false,
|
||||
opts.hoverProbeBypassOverlay,
|
||||
);
|
||||
const scene = buildVomScene(axNodes, captured, { pageUrl: url });
|
||||
const rendered = renderVom(scene, {
|
||||
maxDepth: opts.maxDepth,
|
||||
maxTokens: opts.maxTokens,
|
||||
redactValues: opts.redactValues,
|
||||
activeRegionPolicy: opts.activeRegionPolicy ?? true,
|
||||
});
|
||||
const matchNodes = flattenCapturedNodes(captured);
|
||||
return {
|
||||
text: rendered.text,
|
||||
refs: rendered.refs,
|
||||
truncated: rendered.truncated,
|
||||
captured: captured.nodes,
|
||||
matchNodes,
|
||||
documentTokenOwners: collectDocumentTokenOwners(captured),
|
||||
surfaceProbes: captured.surfaceProbes,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -27,23 +27,19 @@ export async function handleConsole(
|
||||
manager: SessionManager,
|
||||
params: ConsoleParams,
|
||||
deps: ConsoleDeps = defaultConsoleDeps(),
|
||||
signal?: AbortSignal,
|
||||
): Promise<ConsoleResult | RpcError> {
|
||||
if (signal?.aborted) return { code: "cancelled", message: "console aborted" };
|
||||
const ctxOrErr = lookupSession(manager, params, "console");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const bounds = parseBufferedReadBounds(params);
|
||||
if (isRpcError(bounds)) return bounds;
|
||||
const target = await resolveTargetTab(manager, ctxOrErr, params.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
if (signal?.aborted) return { code: "cancelled", message: "console aborted" };
|
||||
if (!deps.cdp.ensureConsoleCapture || !deps.cdp.consoleEntriesSince) {
|
||||
return { code: "cdp_failed", message: "console capture requires CDP console support" };
|
||||
}
|
||||
|
||||
try {
|
||||
await deps.cdp.ensureConsoleCapture(target.tabId);
|
||||
if (signal?.aborted) return { code: "cancelled", message: "console aborted" };
|
||||
deps.cdp.trackSessionTab?.(ctxOrErr.sessionId, target.tabId);
|
||||
return deps.cdp.consoleEntriesSince(
|
||||
target.tabId,
|
||||
|
||||
@@ -125,10 +125,11 @@ export interface DispatcherDeps {
|
||||
* `AbortController` keyed by its wire `id` in
|
||||
* [`inflightAbortControllers`]. When the daemon pushes a `cancel`
|
||||
* request the dispatcher trips the matching controller; tool
|
||||
* handlers observe that signal between awaited operations. The
|
||||
* original RPC remains pending until its handler has stopped or
|
||||
* completed compensation; only the separate cancel acknowledgement
|
||||
* takes the fast path.
|
||||
* handlers that already accept a `signal` (waits, navigation,
|
||||
* interaction, evaluate, tabs) react in line, and the dispatcher
|
||||
* additionally races the in-flight invocation against the abort
|
||||
* promise so handlers without explicit signal plumbing still respond
|
||||
* promptly with `cancelled`.
|
||||
*/
|
||||
export class ToolDispatcher {
|
||||
private readonly transport: Transport;
|
||||
@@ -220,7 +221,7 @@ export class ToolDispatcher {
|
||||
try {
|
||||
const sessionId = sessionIdForBrowserControlMethod(req);
|
||||
if (sessionId) this.onBrowserControlResumed?.(sessionId);
|
||||
const result = await this.invoke(req, ac.signal);
|
||||
const result = await Promise.race([this.invoke(req, ac.signal), abortPromise(ac.signal)]);
|
||||
if (isRpcError(result)) {
|
||||
body = { id: req.id, error: result };
|
||||
} else {
|
||||
@@ -285,45 +286,37 @@ export class ToolDispatcher {
|
||||
private async invoke(req: RequestFrame, signal: AbortSignal): Promise<unknown | RpcError> {
|
||||
switch (req.method) {
|
||||
case "tool.session_start":
|
||||
return handleSessionStart(this.sessions, req.params as SessionStartParams, { signal });
|
||||
return handleSessionStart(this.sessions, req.params as SessionStartParams);
|
||||
case "tool.session_stop": {
|
||||
await this.releaseHoverLatch((req.params as SessionStopParams).session_id);
|
||||
return handleSessionStop(this.sessions, req.params as SessionStopParams, {
|
||||
cdp: this.cdp,
|
||||
signal,
|
||||
});
|
||||
}
|
||||
case "tool.tab_list":
|
||||
return handleTabList(this.sessions, req.params as TabListParams, chromeTabsApi, signal);
|
||||
return handleTabList(this.sessions, req.params as TabListParams);
|
||||
case "tool.tab_create":
|
||||
return handleTabCreate(this.sessions, req.params as TabCreateParams, { signal });
|
||||
return handleTabCreate(this.sessions, req.params as TabCreateParams);
|
||||
case "tool.tab_close":
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as TabCloseParams,
|
||||
() => handleTabClose(this.sessions, req.params as TabCloseParams, { signal }),
|
||||
signal,
|
||||
return this.withHoverReleaseForRequest(req.params as TabCloseParams, () =>
|
||||
handleTabClose(this.sessions, req.params as TabCloseParams),
|
||||
);
|
||||
case "tool.tab_select":
|
||||
return handleTabSelect(this.sessions, req.params as TabSelectParams, { signal });
|
||||
return handleTabSelect(this.sessions, req.params as TabSelectParams);
|
||||
case "tool.tab_borrow":
|
||||
return handleTabBorrow(this.sessions, req.params as TabBorrowParams, {
|
||||
signal,
|
||||
approveBorrow: this.approveBorrow,
|
||||
});
|
||||
case "tool.tab_return":
|
||||
return handleTabReturn(this.sessions, req.params as TabReturnParams, { signal });
|
||||
return handleTabReturn(this.sessions, req.params as TabReturnParams);
|
||||
case "tool.window_resize":
|
||||
return handleWindowResize(
|
||||
this.sessions,
|
||||
req.params as WindowResizeParams,
|
||||
undefined,
|
||||
signal,
|
||||
);
|
||||
return handleWindowResize(this.sessions, req.params as WindowResizeParams);
|
||||
case "tool.emulate":
|
||||
return handleEmulate(
|
||||
this.sessions,
|
||||
req.params as EmulateParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi } : undefined,
|
||||
);
|
||||
case "tool.screenshot":
|
||||
return handleScreenshot(
|
||||
@@ -332,57 +325,43 @@ export class ToolDispatcher {
|
||||
this.cdp
|
||||
? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi, captureApi: chromeTabsCaptureApi }
|
||||
: undefined,
|
||||
signal,
|
||||
);
|
||||
case "tool.console":
|
||||
return handleConsole(
|
||||
this.sessions,
|
||||
req.params as ConsoleParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi } : undefined,
|
||||
signal,
|
||||
);
|
||||
case "tool.network":
|
||||
return handleNetwork(
|
||||
this.sessions,
|
||||
req.params as NetworkParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi } : undefined,
|
||||
signal,
|
||||
);
|
||||
case "tool.snapshot":
|
||||
return this.withHoverReassert(
|
||||
req.params as SnapshotParams,
|
||||
() =>
|
||||
handleSnapshot(
|
||||
this.sessions,
|
||||
req.params as SnapshotParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||
signal,
|
||||
),
|
||||
{},
|
||||
signal,
|
||||
return this.withHoverReassert(req.params as SnapshotParams, () =>
|
||||
handleSnapshot(
|
||||
this.sessions,
|
||||
req.params as SnapshotParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||
),
|
||||
);
|
||||
case "tool.observe": {
|
||||
const params = req.params as ObserveParams;
|
||||
const hoverScope = await this.resolveHoverLatchScope(params);
|
||||
throwIfDispatchAborted(signal);
|
||||
return this.withHoverReassert(
|
||||
params,
|
||||
() =>
|
||||
handleObserve(
|
||||
this.sessions,
|
||||
params,
|
||||
this.cdp
|
||||
? {
|
||||
cdp: this.cdp,
|
||||
tabsApi: chromeTabsCaptureApi,
|
||||
conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope),
|
||||
hoverProbeBypassOverlay: bypassOverlay,
|
||||
}
|
||||
: undefined,
|
||||
signal,
|
||||
),
|
||||
{},
|
||||
signal,
|
||||
return this.withHoverReassert(params, () =>
|
||||
handleObserve(
|
||||
this.sessions,
|
||||
params,
|
||||
this.cdp
|
||||
? {
|
||||
cdp: this.cdp,
|
||||
tabsApi: chromeTabsCaptureApi,
|
||||
conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope),
|
||||
hoverProbeBypassOverlay: bypassOverlay,
|
||||
}
|
||||
: undefined,
|
||||
),
|
||||
);
|
||||
}
|
||||
case "tool.get_html":
|
||||
@@ -390,51 +369,38 @@ export class ToolDispatcher {
|
||||
this.sessions,
|
||||
req.params as GetHtmlParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||
signal,
|
||||
);
|
||||
case "tool.navigate":
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as NavigateParams,
|
||||
() =>
|
||||
handleNavigate(
|
||||
this.sessions,
|
||||
req.params as NavigateParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
signal,
|
||||
return this.withHoverReleaseForRequest(req.params as NavigateParams, () =>
|
||||
handleNavigate(
|
||||
this.sessions,
|
||||
req.params as NavigateParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
);
|
||||
case "tool.navigate_back":
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as NavigateBackParams,
|
||||
() =>
|
||||
handleNavigateBack(
|
||||
this.sessions,
|
||||
req.params as NavigateBackParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
signal,
|
||||
return this.withHoverReleaseForRequest(req.params as NavigateBackParams, () =>
|
||||
handleNavigateBack(
|
||||
this.sessions,
|
||||
req.params as NavigateBackParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
);
|
||||
case "tool.navigate_forward":
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as NavigateForwardParams,
|
||||
() =>
|
||||
handleNavigateForward(
|
||||
this.sessions,
|
||||
req.params as NavigateForwardParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
signal,
|
||||
return this.withHoverReleaseForRequest(req.params as NavigateForwardParams, () =>
|
||||
handleNavigateForward(
|
||||
this.sessions,
|
||||
req.params as NavigateForwardParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
);
|
||||
case "tool.reload":
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as ReloadParams,
|
||||
() =>
|
||||
handleReload(
|
||||
this.sessions,
|
||||
req.params as ReloadParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
signal,
|
||||
return this.withHoverReleaseForRequest(req.params as ReloadParams, () =>
|
||||
handleReload(
|
||||
this.sessions,
|
||||
req.params as ReloadParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
);
|
||||
case "tool.click":
|
||||
return this.withHoverReassert(
|
||||
@@ -453,7 +419,6 @@ export class ToolDispatcher {
|
||||
: undefined,
|
||||
),
|
||||
{ releaseAfter: true },
|
||||
signal,
|
||||
);
|
||||
case "tool.hover": {
|
||||
const result = await handleHover(
|
||||
@@ -473,37 +438,28 @@ export class ToolDispatcher {
|
||||
return this.rememberHover((req.params as HoverParams).session_id, result);
|
||||
}
|
||||
case "tool.fill":
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as FillParams,
|
||||
() =>
|
||||
handleFill(
|
||||
this.sessions,
|
||||
req.params as FillParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
signal,
|
||||
return this.withHoverReleaseForRequest(req.params as FillParams, () =>
|
||||
handleFill(
|
||||
this.sessions,
|
||||
req.params as FillParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
);
|
||||
case "tool.press":
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as PressParams,
|
||||
() =>
|
||||
handlePress(
|
||||
this.sessions,
|
||||
req.params as PressParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
signal,
|
||||
return this.withHoverReleaseForRequest(req.params as PressParams, () =>
|
||||
handlePress(
|
||||
this.sessions,
|
||||
req.params as PressParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
);
|
||||
case "tool.select":
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as SelectParams,
|
||||
() =>
|
||||
handleSelect(
|
||||
this.sessions,
|
||||
req.params as SelectParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
signal,
|
||||
return this.withHoverReleaseForRequest(req.params as SelectParams, () =>
|
||||
handleSelect(
|
||||
this.sessions,
|
||||
req.params as SelectParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
);
|
||||
case "tool.evaluate":
|
||||
return handleEvaluate(
|
||||
@@ -610,13 +566,9 @@ export class ToolDispatcher {
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
options: { releaseAfter?: boolean } = {},
|
||||
signal?: AbortSignal,
|
||||
): Promise<T> {
|
||||
throwIfDispatchAborted(signal);
|
||||
const scope = await this.resolveHoverLatchScope(params);
|
||||
throwIfDispatchAborted(signal);
|
||||
await this.reassertHover(scope);
|
||||
throwIfDispatchAborted(signal);
|
||||
try {
|
||||
return await work();
|
||||
} finally {
|
||||
@@ -629,13 +581,9 @@ export class ToolDispatcher {
|
||||
private async withHoverReleaseForRequest<T>(
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<T> {
|
||||
throwIfDispatchAborted(signal);
|
||||
const scope = await this.resolveHoverLatchScope(params);
|
||||
throwIfDispatchAborted(signal);
|
||||
await this.releaseHoverLatch(scope.session_id, scope.tab_id);
|
||||
throwIfDispatchAborted(signal);
|
||||
return work();
|
||||
}
|
||||
|
||||
@@ -745,14 +693,41 @@ async function bypassOverlay(tabId: number, enabled: boolean): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
function throwIfDispatchAborted(signal: AbortSignal | undefined): void {
|
||||
if (!signal?.aborted) return;
|
||||
const error = new Error("rpc aborted by daemon cancel");
|
||||
error.name = "AbortError";
|
||||
throw error;
|
||||
/**
|
||||
* Resolves never; rejects with `AbortLikeError` as soon as the signal
|
||||
* fires (or immediately if it is already aborted). Used by the
|
||||
* dispatcher to race the tool invocation so handlers without explicit
|
||||
* signal plumbing still surface a `cancelled` reply promptly.
|
||||
*/
|
||||
function abortPromise(signal: AbortSignal): Promise<never> {
|
||||
return new Promise<never>((_, reject) => {
|
||||
if (signal.aborted) {
|
||||
reject(new AbortLikeError());
|
||||
return;
|
||||
}
|
||||
signal.addEventListener(
|
||||
"abort",
|
||||
() => {
|
||||
reject(new AbortLikeError());
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Sentinel error class so [`isAbortLikeError`] can recognise our own
|
||||
* race-rejection without confusing it with a real CDP failure.
|
||||
*/
|
||||
class AbortLikeError extends Error {
|
||||
constructor() {
|
||||
super("rpc aborted by daemon cancel");
|
||||
this.name = "BhAbortError";
|
||||
}
|
||||
}
|
||||
|
||||
function isAbortLikeError(err: unknown): boolean {
|
||||
if (err instanceof AbortLikeError) return true;
|
||||
if (err instanceof DOMException && err.name === "AbortError") return true;
|
||||
if (typeof err === "object" && err !== null && (err as { name?: string }).name === "AbortError") {
|
||||
return true;
|
||||
|
||||
@@ -1,15 +1,9 @@
|
||||
// Resolve and scroll nodes inside one CDP target. Cross-frame projection
|
||||
// belongs to frame-geometry.ts.
|
||||
// Shared CDP helpers for scrolling nodes into view and computing
|
||||
// viewport-space bounding rectangles. Used by interaction tools (click
|
||||
// centre) and observation tools (element screenshot clip).
|
||||
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { rpcError } from "./errors";
|
||||
import {
|
||||
parseCdpQuad,
|
||||
polygonBounds,
|
||||
type Region,
|
||||
rectPolygon,
|
||||
type ViewportRect,
|
||||
} from "./geometry";
|
||||
import { type CdpRunner, isRpcError } from "./shared";
|
||||
|
||||
const ELEMENT_NOT_VISIBLE_MESSAGE =
|
||||
@@ -19,6 +13,45 @@ function elementNotVisibleError(): RpcError {
|
||||
return rpcError("permission_denied", "element_not_visible", ELEMENT_NOT_VISIBLE_MESSAGE);
|
||||
}
|
||||
|
||||
/** Viewport-space axis-aligned bounding box for CDP `Page.captureScreenshot` clip. */
|
||||
export interface ViewportRect {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the axis-aligned bounding box of an 8-double CDP content
|
||||
* quad / box-model polygon. Exported for unit tests.
|
||||
*/
|
||||
export function quadBoundingRect(quad: number[]): ViewportRect | null {
|
||||
if (quad.length !== 8) return null;
|
||||
const xs = [quad[0], quad[2], quad[4], quad[6]];
|
||||
const ys = [quad[1], quad[3], quad[5], quad[7]];
|
||||
const minX = Math.min(...xs);
|
||||
const maxX = Math.max(...xs);
|
||||
const minY = Math.min(...ys);
|
||||
const maxY = Math.max(...ys);
|
||||
const width = maxX - minX;
|
||||
const height = maxY - minY;
|
||||
if (width <= 0 || height <= 0) return null;
|
||||
return { x: minX, y: minY, width, height };
|
||||
}
|
||||
|
||||
function rectToQuad(rect: ViewportRect): number[] {
|
||||
return [
|
||||
rect.x,
|
||||
rect.y,
|
||||
rect.x + rect.width,
|
||||
rect.y,
|
||||
rect.x + rect.width,
|
||||
rect.y + rect.height,
|
||||
rect.x,
|
||||
rect.y + rect.height,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve `backendNodeId` → CDP `objectId` so we can invoke
|
||||
* `Runtime.callFunctionOn` against the live JS object.
|
||||
@@ -79,24 +112,48 @@ export async function scrollNodeIntoView(
|
||||
}
|
||||
}
|
||||
|
||||
function validCdpRegion(quads: number[][] | undefined): Region {
|
||||
return (quads ?? []).flatMap((raw) => {
|
||||
const polygon = parseCdpQuad(raw);
|
||||
return polygon && polygonBounds(polygon) ? [polygon] : [];
|
||||
});
|
||||
}
|
||||
|
||||
async function resolveVisibleContentRegion(
|
||||
/**
|
||||
* Viewport-space bounding box for `backendNodeId`. Prefers
|
||||
* `DOM.getContentQuads`, then `DOM.getBoxModel`, then the union of
|
||||
* visible descendant client rects for zero-size overflow containers.
|
||||
*/
|
||||
export async function nodeBoundingRect(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
backendNodeId: number,
|
||||
): Promise<Region | RpcError> {
|
||||
): Promise<ViewportRect | RpcError> {
|
||||
const polygon = await visibleContentPolygon(cdp, tabId, backendNodeId);
|
||||
if (isRpcError(polygon)) return polygon;
|
||||
// visibleContentPolygon only returns quads with positive area.
|
||||
return quadBoundingRect(polygon)!;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the centroid of a CDP content quad. Quads are reported as
|
||||
* 8 doubles in clockwise order: (x1, y1, x2, y2, x3, y3, x4, y4).
|
||||
*/
|
||||
export function quadCentre(quad: number[]): { x: number; y: number } {
|
||||
const x = (quad[0] + quad[4]) / 2;
|
||||
const y = (quad[1] + quad[5]) / 2;
|
||||
return { x, y };
|
||||
}
|
||||
|
||||
/** Centre of a `DOM.getBoxModel` `content` polygon (also 8 doubles). */
|
||||
export function boxCentre(box: number[]): { x: number; y: number } {
|
||||
return quadCentre(box);
|
||||
}
|
||||
|
||||
async function visibleContentPolygon(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
backendNodeId: number,
|
||||
): Promise<number[] | RpcError> {
|
||||
try {
|
||||
const quads = await cdp.send<{ quads?: number[][] }>(tabId, "DOM.getContentQuads", {
|
||||
backendNodeId,
|
||||
});
|
||||
const visible = validCdpRegion(quads.quads);
|
||||
if (visible.length > 0) return visible;
|
||||
const quad = quads.quads?.find((q) => q.length === 8 && quadBoundingRect(q) !== null);
|
||||
if (quad) return quad;
|
||||
} catch (err) {
|
||||
console.debug("[bsk element-geometry] getContentQuads failed", err);
|
||||
}
|
||||
@@ -105,34 +162,15 @@ async function resolveVisibleContentRegion(
|
||||
backendNodeId,
|
||||
});
|
||||
const content = box.model?.content;
|
||||
const fallback = validCdpRegion(content ? [content] : undefined);
|
||||
if (fallback.length > 0) return fallback;
|
||||
if (content && content.length === 8 && quadBoundingRect(content)) return content;
|
||||
} catch (err) {
|
||||
console.debug("[bsk element-geometry] getBoxModel failed", err);
|
||||
}
|
||||
const descendantRect = await descendantBoundingRect(cdp, tabId, backendNodeId);
|
||||
if (descendantRect) {
|
||||
return [
|
||||
rectPolygon({
|
||||
x: descendantRect.x,
|
||||
y: descendantRect.y,
|
||||
w: descendantRect.width,
|
||||
h: descendantRect.height,
|
||||
}),
|
||||
];
|
||||
}
|
||||
if (descendantRect) return rectToQuad(descendantRect);
|
||||
return elementNotVisibleError();
|
||||
}
|
||||
|
||||
/** Resolve every visible content region in the node's target-local viewport. */
|
||||
export async function nodeContentRegion(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
backendNodeId: number,
|
||||
): Promise<Region | RpcError> {
|
||||
return resolveVisibleContentRegion(cdp, tabId, backendNodeId);
|
||||
}
|
||||
|
||||
async function descendantBoundingRect(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
@@ -203,3 +241,20 @@ function parseViewportRect(value: unknown): ViewportRect | null {
|
||||
if (![x, y, width, height].every(Number.isFinite) || width <= 0 || height <= 0) return null;
|
||||
return { x, y, width, height };
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the click point for `backendNodeId`. Prefers
|
||||
* `DOM.getContentQuads` (richer for rotated / transformed elements);
|
||||
* falls back to `DOM.getBoxModel`, then visible descendant bounds.
|
||||
* Returns `permission_denied` with "element not visible" when every
|
||||
* geometry path comes back empty / fails.
|
||||
*/
|
||||
export async function nodeCentre(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
backendNodeId: number,
|
||||
): Promise<{ x: number; y: number } | RpcError> {
|
||||
const polygon = await visibleContentPolygon(cdp, tabId, backendNodeId);
|
||||
if (isRpcError(polygon)) return polygon;
|
||||
return quadCentre(polygon);
|
||||
}
|
||||
|
||||
@@ -44,7 +44,6 @@ export interface EmulateCdpRunner {
|
||||
export interface EmulateDeps {
|
||||
cdp: EmulateCdpRunner;
|
||||
tabsApi: ChromeTabsApi;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
function defaultEmulateDeps(): EmulateDeps {
|
||||
@@ -211,13 +210,11 @@ export async function handleEmulate(
|
||||
params: EmulateParams,
|
||||
deps: EmulateDeps = defaultEmulateDeps(),
|
||||
): Promise<EmulateResult | RpcError> {
|
||||
if (deps.signal?.aborted) return { code: "cancelled", message: "emulate aborted" };
|
||||
const ctxOrErr = lookupSession(manager, params, "emulate");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
const target = await resolveTargetTab(manager, ctx, params?.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
if (deps.signal?.aborted) return { code: "cancelled", message: "emulate aborted" };
|
||||
const denied = enforceAgentWindow(ctx, target, "emulate");
|
||||
if (denied) return denied;
|
||||
|
||||
@@ -226,16 +223,9 @@ export async function handleEmulate(
|
||||
return invalidParams("emulate off cannot be combined with overrides");
|
||||
}
|
||||
try {
|
||||
if (deps.signal?.aborted) return { code: "cancelled", message: "emulate aborted" };
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await deps.cdp.clearDeviceMetricsOverride(target.tabId);
|
||||
if (deps.signal?.aborted) {
|
||||
return { code: "cancelled", message: "emulate aborted after clearing device metrics" };
|
||||
}
|
||||
await deps.cdp.setTouchEmulationEnabled(target.tabId, false);
|
||||
if (deps.signal?.aborted) {
|
||||
return { code: "cancelled", message: "emulate aborted after clearing touch emulation" };
|
||||
}
|
||||
// An empty userAgent string clears the override (CDP convention).
|
||||
await deps.cdp.setUserAgentOverride(target.tabId, { userAgent: "" });
|
||||
} catch (err) {
|
||||
@@ -256,7 +246,6 @@ export async function handleEmulate(
|
||||
// values; the merged state is what gets applied (and echoed back).
|
||||
const merged = mergeEmulateOverrides(tabEmulationStates.get(target.tabId), overrides);
|
||||
try {
|
||||
if (deps.signal?.aborted) return { code: "cancelled", message: "emulate aborted" };
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
if (merged.width !== undefined && merged.height !== undefined) {
|
||||
await deps.cdp.setDeviceMetricsOverride(target.tabId, {
|
||||
@@ -265,9 +254,6 @@ export async function handleEmulate(
|
||||
deviceScaleFactor: merged.device_scale_factor ?? 0,
|
||||
mobile: merged.mobile ?? false,
|
||||
});
|
||||
if (deps.signal?.aborted) {
|
||||
return { code: "cancelled", message: "emulate aborted after applying device metrics" };
|
||||
}
|
||||
}
|
||||
if (merged.user_agent !== undefined) {
|
||||
await deps.cdp.setUserAgentOverride(target.tabId, {
|
||||
@@ -277,9 +263,6 @@ export async function handleEmulate(
|
||||
? { userAgentMetadata: toCdpUserAgentMetadata(merged.user_agent_metadata) }
|
||||
: {}),
|
||||
});
|
||||
if (deps.signal?.aborted) {
|
||||
return { code: "cancelled", message: "emulate aborted after applying user agent" };
|
||||
}
|
||||
}
|
||||
if (merged.touch !== undefined || merged.max_touch_points !== undefined) {
|
||||
await deps.cdp.setTouchEmulationEnabled(
|
||||
|
||||
@@ -1,290 +0,0 @@
|
||||
import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry";
|
||||
import {
|
||||
clipPolygon,
|
||||
type GeometryProjection,
|
||||
type Point,
|
||||
type Polygon,
|
||||
type ProjectiveEdge,
|
||||
parseCdpQuad,
|
||||
polygonArea,
|
||||
polygonCentroid,
|
||||
projectRegionToViewport,
|
||||
type Quad,
|
||||
type Region,
|
||||
rectPolygon,
|
||||
regionBounds,
|
||||
type Size,
|
||||
type ViewportRect,
|
||||
} from "./geometry";
|
||||
import { type CdpRunner, cdpRunnerForTarget, isRpcError, sendToCdpTarget } from "./shared";
|
||||
|
||||
export interface NodeAddress {
|
||||
target: CdpTarget;
|
||||
backendNodeId: number;
|
||||
frameId?: string;
|
||||
}
|
||||
|
||||
export interface ResolvedNodeGeometry {
|
||||
topVisibleRegions: Region;
|
||||
topBounds: ViewportRect;
|
||||
actionPoint: Point;
|
||||
}
|
||||
|
||||
function geometryError(message: string): RpcError {
|
||||
return { code: "cdp_failed", message };
|
||||
}
|
||||
|
||||
function frameMap(graph: CdpFrameGraph): Map<string, CdpFrame> {
|
||||
return new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
||||
}
|
||||
|
||||
function targetRootFrame(byId: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
|
||||
const seen = new Set<string>();
|
||||
let current = frame;
|
||||
while (current.parentFrameId) {
|
||||
if (seen.has(current.frameId)) return null;
|
||||
seen.add(current.frameId);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent || parent.target.sessionId !== current.target.sessionId) break;
|
||||
current = parent;
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
function frameAncestry(graph: CdpFrameGraph, frameId: string): CdpFrame[] | null {
|
||||
const byId = frameMap(graph);
|
||||
const path: CdpFrame[] = [];
|
||||
const seen = new Set<string>();
|
||||
let current = byId.get(frameId);
|
||||
if (!current) return null;
|
||||
while (current.parentFrameId) {
|
||||
if (seen.has(current.frameId)) return null;
|
||||
seen.add(current.frameId);
|
||||
path.push(current);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent) return null;
|
||||
current = parent;
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise<Size | null> {
|
||||
const metrics = await sendToCdpTarget<{
|
||||
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
}>(cdp, target, "Page.getLayoutMetrics", {});
|
||||
const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
const width = viewport.clientWidth ?? 0;
|
||||
const height = viewport.clientHeight ?? 0;
|
||||
return width > 0 && height > 0 ? { width, height } : null;
|
||||
}
|
||||
|
||||
async function ownerContentQuad(
|
||||
cdp: CdpRunner,
|
||||
parent: CdpFrame,
|
||||
ownerBackendNodeId: number,
|
||||
): Promise<Quad | null> {
|
||||
const result = await sendToCdpTarget<{ model?: { content?: number[] } }>(
|
||||
cdp,
|
||||
parent.target,
|
||||
"DOM.getBoxModel",
|
||||
{ backendNodeId: ownerBackendNodeId },
|
||||
);
|
||||
return parseCdpQuad(result.model?.content);
|
||||
}
|
||||
|
||||
async function sameTargetFrameClips(
|
||||
cdp: CdpRunner,
|
||||
byId: Map<string, CdpFrame>,
|
||||
frame: CdpFrame,
|
||||
targetRoot: CdpFrame,
|
||||
): Promise<Polygon[] | null> {
|
||||
const clips: Polygon[] = [];
|
||||
let current = frame;
|
||||
const seen = new Set<string>();
|
||||
while (current.frameId !== targetRoot.frameId) {
|
||||
if (seen.has(current.frameId) || !current.parentFrameId) return null;
|
||||
seen.add(current.frameId);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent || current.ownerBackendNodeId === undefined) return null;
|
||||
const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId);
|
||||
if (!clip) return null;
|
||||
clips.push(clip);
|
||||
current = parent;
|
||||
}
|
||||
return clips;
|
||||
}
|
||||
|
||||
export async function resolveFrameProjection(
|
||||
cdp: CdpRunner,
|
||||
graph: CdpFrameGraph,
|
||||
frameId: string,
|
||||
): Promise<GeometryProjection | null> {
|
||||
const byId = frameMap(graph);
|
||||
const frame = byId.get(frameId);
|
||||
if (!frame) return null;
|
||||
let targetRoot = targetRootFrame(byId, frame);
|
||||
if (!targetRoot) return null;
|
||||
const sourceViewport = await targetViewport(cdp, targetRoot.target);
|
||||
if (!sourceViewport) return null;
|
||||
const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot);
|
||||
if (!sourceClips) return null;
|
||||
|
||||
const edges: ProjectiveEdge[] = [];
|
||||
while (targetRoot.parentFrameId) {
|
||||
const parent = byId.get(targetRoot.parentFrameId);
|
||||
if (!parent || targetRoot.ownerBackendNodeId === undefined) return null;
|
||||
const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId);
|
||||
if (!destinationQuad) return null;
|
||||
const source =
|
||||
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
||||
if (!source) return null;
|
||||
const parentTargetRoot = targetRootFrame(byId, parent);
|
||||
if (!parentTargetRoot) return null;
|
||||
const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot);
|
||||
if (!destinationClips) return null;
|
||||
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
|
||||
targetRoot = parentTargetRoot;
|
||||
}
|
||||
|
||||
const topViewport =
|
||||
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
||||
return topViewport ? { sourceClips, edges, topViewport } : null;
|
||||
}
|
||||
|
||||
async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
|
||||
if (!cdp.getFrameGraph) return null;
|
||||
try {
|
||||
return await cdp.getFrameGraph(tabId);
|
||||
} catch (error) {
|
||||
console.debug("[bsk frame-geometry] frame graph resolution failed", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function scrollFrameOwners(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
graph: CdpFrameGraph,
|
||||
frameId: string,
|
||||
): Promise<RpcError | null> {
|
||||
const byId = frameMap(graph);
|
||||
const ancestry = frameAncestry(graph, frameId);
|
||||
if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`);
|
||||
for (const child of [...ancestry].reverse()) {
|
||||
const parent = child.parentFrameId ? byId.get(child.parentFrameId) : undefined;
|
||||
if (!parent || child.ownerBackendNodeId === undefined) {
|
||||
return geometryError(`could not resolve frame owner for ${child.frameId}`);
|
||||
}
|
||||
const error = await scrollNodeIntoView(
|
||||
cdpRunnerForTarget(cdp, parent.target),
|
||||
tabId,
|
||||
child.ownerBackendNodeId,
|
||||
);
|
||||
if (error) return error;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function scrollElementWithFrameGraph(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
target: CdpTarget,
|
||||
backendNodeId: number,
|
||||
frameId: string | undefined,
|
||||
graph: CdpFrameGraph | null,
|
||||
): Promise<RpcError | null> {
|
||||
if (frameId) {
|
||||
if (!graph) return geometryError(`could not resolve frame graph for ${frameId}`);
|
||||
const error = await scrollFrameOwners(cdp, tabId, graph, frameId);
|
||||
if (error) return error;
|
||||
} else if (target.sessionId) {
|
||||
return geometryError("an OOPIF node address requires frameId");
|
||||
}
|
||||
return scrollNodeIntoView(cdpRunnerForTarget(cdp, target), tabId, backendNodeId);
|
||||
}
|
||||
|
||||
export async function scrollElementAndFramesIntoView(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
target: CdpTarget,
|
||||
backendNodeId: number,
|
||||
frameId?: string,
|
||||
): Promise<RpcError | null> {
|
||||
const graph = frameId ? await loadFrameGraph(cdp, tabId) : null;
|
||||
return scrollElementWithFrameGraph(cdp, tabId, target, backendNodeId, frameId, graph);
|
||||
}
|
||||
|
||||
function largestRegion(regions: Region): Polygon | null {
|
||||
let largest: { polygon: Polygon; area: number } | null = null;
|
||||
for (const polygon of regions) {
|
||||
const area = polygonArea(polygon);
|
||||
if (area <= 0) continue;
|
||||
if (!largest || area > largest.area) largest = { polygon, area };
|
||||
}
|
||||
return largest?.polygon ?? null;
|
||||
}
|
||||
|
||||
export async function resolveNodeGeometry(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
address: NodeAddress,
|
||||
options: { scrollIntoView?: boolean } = {},
|
||||
): Promise<ResolvedNodeGeometry | RpcError> {
|
||||
try {
|
||||
if (address.target.sessionId && !address.frameId) {
|
||||
return geometryError("an OOPIF node address requires frameId");
|
||||
}
|
||||
const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null;
|
||||
if (address.frameId && !graph) {
|
||||
return geometryError(`could not resolve frame graph for ${address.frameId}`);
|
||||
}
|
||||
|
||||
if (options.scrollIntoView) {
|
||||
const scrollError = await scrollElementWithFrameGraph(
|
||||
cdp,
|
||||
tabId,
|
||||
address.target,
|
||||
address.backendNodeId,
|
||||
address.frameId,
|
||||
graph,
|
||||
);
|
||||
if (scrollError) return scrollError;
|
||||
}
|
||||
|
||||
const localRegion = await nodeContentRegion(
|
||||
cdpRunnerForTarget(cdp, address.target),
|
||||
tabId,
|
||||
address.backendNodeId,
|
||||
);
|
||||
if (isRpcError(localRegion)) return localRegion;
|
||||
|
||||
let topVisibleRegions: Region;
|
||||
if (address.frameId && graph) {
|
||||
const projection = await resolveFrameProjection(cdp, graph, address.frameId);
|
||||
if (!projection)
|
||||
return geometryError(`could not resolve frame geometry for ${address.frameId}`);
|
||||
topVisibleRegions = projectRegionToViewport(localRegion, projection);
|
||||
} else {
|
||||
const viewport = await targetViewport(cdp, address.target);
|
||||
if (!viewport) return geometryError("could not resolve top viewport geometry");
|
||||
topVisibleRegions = localRegion
|
||||
.map((polygon) =>
|
||||
clipPolygon(polygon, rectPolygon({ x: 0, y: 0, w: viewport.width, h: viewport.height })),
|
||||
)
|
||||
.filter((polygon) => polygon.length >= 3);
|
||||
}
|
||||
|
||||
const topBounds = regionBounds(topVisibleRegions);
|
||||
const actionRegion = largestRegion(topVisibleRegions);
|
||||
const actionPoint = actionRegion ? polygonCentroid(actionRegion) : null;
|
||||
if (!topBounds || !actionPoint) {
|
||||
return { code: "permission_denied", message: "element not visible" };
|
||||
}
|
||||
return { topVisibleRegions, topBounds, actionPoint };
|
||||
} catch (error) {
|
||||
return geometryError(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
}
|
||||
@@ -1,266 +0,0 @@
|
||||
export interface Point {
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
export type Polygon = Point[];
|
||||
export type Region = Polygon[];
|
||||
export type Quad = [Point, Point, Point, Point];
|
||||
|
||||
export interface Size {
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface ViewportRect {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface ProjectiveEdge {
|
||||
sourceViewport: Size;
|
||||
destinationQuad: Quad;
|
||||
destinationClips?: Polygon[];
|
||||
}
|
||||
|
||||
export interface GeometryProjection {
|
||||
sourceClips: Polygon[];
|
||||
edges: ProjectiveEdge[];
|
||||
topViewport: Size;
|
||||
}
|
||||
|
||||
export function parseCdpQuad(raw: number[] | undefined): Quad | null {
|
||||
if (!raw || raw.length !== 8 || !raw.every(Number.isFinite)) return null;
|
||||
return [
|
||||
{ x: raw[0], y: raw[1] },
|
||||
{ x: raw[2], y: raw[3] },
|
||||
{ x: raw[4], y: raw[5] },
|
||||
{ x: raw[6], y: raw[7] },
|
||||
];
|
||||
}
|
||||
|
||||
export function rectPolygon(rect: { x: number; y: number; w: number; h: number }): Quad {
|
||||
return [
|
||||
{ x: rect.x, y: rect.y },
|
||||
{ x: rect.x + rect.w, y: rect.y },
|
||||
{ x: rect.x + rect.w, y: rect.y + rect.h },
|
||||
{ x: rect.x, y: rect.y + rect.h },
|
||||
];
|
||||
}
|
||||
|
||||
export function viewportPolygon(viewport: Size): Quad {
|
||||
return rectPolygon({ x: 0, y: 0, w: viewport.width, h: viewport.height });
|
||||
}
|
||||
|
||||
/** Project a normalized unit-square point into an arbitrary content quad. */
|
||||
export function projectUnitPoint(point: Point, quad: Quad): Point {
|
||||
const [p0, p1, p2, p3] = quad;
|
||||
const dx1 = p1.x - p2.x;
|
||||
const dx2 = p3.x - p2.x;
|
||||
const dx3 = p0.x - p1.x + p2.x - p3.x;
|
||||
const dy1 = p1.y - p2.y;
|
||||
const dy2 = p3.y - p2.y;
|
||||
const dy3 = p0.y - p1.y + p2.y - p3.y;
|
||||
|
||||
let a: number;
|
||||
let b: number;
|
||||
let c: number;
|
||||
let d: number;
|
||||
let e: number;
|
||||
let f: number;
|
||||
let g = 0;
|
||||
let h = 0;
|
||||
|
||||
if (Math.abs(dx3) < 1e-9 && Math.abs(dy3) < 1e-9) {
|
||||
a = p1.x - p0.x;
|
||||
b = p3.x - p0.x;
|
||||
c = p0.x;
|
||||
d = p1.y - p0.y;
|
||||
e = p3.y - p0.y;
|
||||
f = p0.y;
|
||||
} else {
|
||||
const denominator = dx1 * dy2 - dx2 * dy1;
|
||||
if (Math.abs(denominator) < 1e-9) {
|
||||
return {
|
||||
x: p0.x + (p1.x - p0.x) * point.x + (p3.x - p0.x) * point.y,
|
||||
y: p0.y + (p1.y - p0.y) * point.x + (p3.y - p0.y) * point.y,
|
||||
};
|
||||
}
|
||||
g = (dx3 * dy2 - dx2 * dy3) / denominator;
|
||||
h = (dx1 * dy3 - dx3 * dy1) / denominator;
|
||||
a = p1.x - p0.x + g * p1.x;
|
||||
b = p3.x - p0.x + h * p3.x;
|
||||
c = p0.x;
|
||||
d = p1.y - p0.y + g * p1.y;
|
||||
e = p3.y - p0.y + h * p3.y;
|
||||
f = p0.y;
|
||||
}
|
||||
|
||||
const scale = g * point.x + h * point.y + 1;
|
||||
return {
|
||||
x: (a * point.x + b * point.y + c) / scale,
|
||||
y: (d * point.x + e * point.y + f) / scale,
|
||||
};
|
||||
}
|
||||
|
||||
export function projectPolygon(polygon: Polygon, edge: ProjectiveEdge): Polygon {
|
||||
const { width, height } = edge.sourceViewport;
|
||||
if (width <= 0 || height <= 0) return [];
|
||||
return polygon.map((point) =>
|
||||
projectUnitPoint({ x: point.x / width, y: point.y / height }, edge.destinationQuad),
|
||||
);
|
||||
}
|
||||
|
||||
function polygonSignedArea(points: Polygon): number {
|
||||
let area = 0;
|
||||
for (let index = 0; index < points.length; index += 1) {
|
||||
const current = points[index];
|
||||
const next = points[(index + 1) % points.length];
|
||||
area += current.x * next.y - next.x * current.y;
|
||||
}
|
||||
return area / 2;
|
||||
}
|
||||
|
||||
function edgeIntersection(start: Point, end: Point, clipStart: Point, clipEnd: Point): Point {
|
||||
const subjectX = end.x - start.x;
|
||||
const subjectY = end.y - start.y;
|
||||
const clipX = clipEnd.x - clipStart.x;
|
||||
const clipY = clipEnd.y - clipStart.y;
|
||||
const denominator = subjectX * clipY - subjectY * clipX;
|
||||
if (Math.abs(denominator) < 1e-9) return end;
|
||||
const offsetX = clipStart.x - start.x;
|
||||
const offsetY = clipStart.y - start.y;
|
||||
const scale = (offsetX * clipY - offsetY * clipX) / denominator;
|
||||
return { x: start.x + scale * subjectX, y: start.y + scale * subjectY };
|
||||
}
|
||||
|
||||
/** Clip a polygon to a convex polygon, preserving every resulting vertex. */
|
||||
export function clipPolygon(subject: Polygon, clip: Polygon): Polygon {
|
||||
if (subject.length < 3 || clip.length < 3) return [];
|
||||
const orientation = polygonSignedArea(clip) >= 0 ? 1 : -1;
|
||||
let output = [...subject];
|
||||
for (let edge = 0; edge < clip.length && output.length > 0; edge += 1) {
|
||||
const clipStart = clip[edge];
|
||||
const clipEnd = clip[(edge + 1) % clip.length];
|
||||
const input = output;
|
||||
output = [];
|
||||
const inside = (point: Point) =>
|
||||
orientation *
|
||||
((clipEnd.x - clipStart.x) * (point.y - clipStart.y) -
|
||||
(clipEnd.y - clipStart.y) * (point.x - clipStart.x)) >=
|
||||
-1e-9;
|
||||
let start = input[input.length - 1];
|
||||
for (const end of input) {
|
||||
const startInside = inside(start);
|
||||
const endInside = inside(end);
|
||||
if (endInside) {
|
||||
if (!startInside) output.push(edgeIntersection(start, end, clipStart, clipEnd));
|
||||
output.push(end);
|
||||
} else if (startInside) {
|
||||
output.push(edgeIntersection(start, end, clipStart, clipEnd));
|
||||
}
|
||||
start = end;
|
||||
}
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
export function polygonBounds(points: Polygon): ViewportRect | null {
|
||||
if (points.length < 3) return null;
|
||||
const xs = points.map((point) => point.x);
|
||||
const ys = points.map((point) => point.y);
|
||||
const x = Math.min(...xs);
|
||||
const y = Math.min(...ys);
|
||||
const right = Math.max(...xs);
|
||||
const bottom = Math.max(...ys);
|
||||
if (right <= x || bottom <= y) return null;
|
||||
return { x, y, width: right - x, height: bottom - y };
|
||||
}
|
||||
|
||||
export function regionBounds(region: Region): ViewportRect | null {
|
||||
return polygonBounds(region.flat());
|
||||
}
|
||||
|
||||
export function polygonArea(points: Polygon): number {
|
||||
return Math.abs(polygonSignedArea(points));
|
||||
}
|
||||
|
||||
export function polygonCentroid(points: Polygon): Point | null {
|
||||
if (points.length < 3) return null;
|
||||
let twiceArea = 0;
|
||||
let x = 0;
|
||||
let y = 0;
|
||||
for (let index = 0; index < points.length; index += 1) {
|
||||
const current = points[index];
|
||||
const next = points[(index + 1) % points.length];
|
||||
const cross = current.x * next.y - next.x * current.y;
|
||||
twiceArea += cross;
|
||||
x += (current.x + next.x) * cross;
|
||||
y += (current.y + next.y) * cross;
|
||||
}
|
||||
if (Math.abs(twiceArea) < 1e-9) return null;
|
||||
return { x: x / (3 * twiceArea), y: y / (3 * twiceArea) };
|
||||
}
|
||||
|
||||
export function projectAndClipRegion(
|
||||
region: Region,
|
||||
edges: ProjectiveEdge[],
|
||||
topViewport: Size,
|
||||
): Region {
|
||||
let projected = region;
|
||||
for (const edge of edges) {
|
||||
const sourceClip = viewportPolygon(edge.sourceViewport);
|
||||
projected = projected
|
||||
.map((polygon) => clipPolygon(polygon, sourceClip))
|
||||
.filter((polygon) => polygon.length >= 3)
|
||||
.map((polygon) =>
|
||||
[edge.destinationQuad, ...(edge.destinationClips ?? [])].reduce(
|
||||
(current, clip) => clipPolygon(current, clip),
|
||||
projectPolygon(polygon, edge),
|
||||
),
|
||||
)
|
||||
.filter((polygon) => polygon.length >= 3);
|
||||
}
|
||||
const topClip = viewportPolygon(topViewport);
|
||||
return projected
|
||||
.map((polygon) => clipPolygon(polygon, topClip))
|
||||
.filter((polygon) => polygon.length >= 3);
|
||||
}
|
||||
|
||||
export function projectRegionToViewport(region: Region, projection: GeometryProjection): Region {
|
||||
const clipped = region
|
||||
.map((polygon) =>
|
||||
projection.sourceClips.reduce((current, clip) => clipPolygon(current, clip), polygon),
|
||||
)
|
||||
.filter((polygon) => polygon.length >= 3);
|
||||
return projectAndClipRegion(clipped, projection.edges, projection.topViewport);
|
||||
}
|
||||
|
||||
export function projectRectToViewport(
|
||||
rect: { x: number; y: number; w: number; h: number } | null,
|
||||
projection: GeometryProjection,
|
||||
): ViewportRect | null {
|
||||
if (!rect) return null;
|
||||
return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection));
|
||||
}
|
||||
|
||||
export function childFrameProjection(
|
||||
parent: GeometryProjection,
|
||||
ownerRectInParent: { x: number; y: number; w: number; h: number },
|
||||
): GeometryProjection {
|
||||
const destinationQuad = rectPolygon(ownerRectInParent);
|
||||
return {
|
||||
sourceClips: [],
|
||||
edges: [
|
||||
{
|
||||
sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h },
|
||||
destinationQuad,
|
||||
destinationClips: parent.sourceClips,
|
||||
},
|
||||
...parent.edges,
|
||||
],
|
||||
topViewport: parent.topViewport,
|
||||
};
|
||||
}
|
||||
@@ -8,7 +8,6 @@ import {
|
||||
HELP_REQUEST,
|
||||
type HelpCancelMessage,
|
||||
type HelpFinishMessage,
|
||||
type HelpHighlightRect,
|
||||
type HelpQueryResponse,
|
||||
type HelpRequestMessage,
|
||||
isHelpAckMessage,
|
||||
@@ -26,7 +25,6 @@ import type {
|
||||
ResolvedTarget,
|
||||
RpcError,
|
||||
} from "@/transport/types";
|
||||
import { resolveNodeGeometry } from "./frame-geometry";
|
||||
import {
|
||||
type CdpRunner,
|
||||
type ChromeTabsApi,
|
||||
@@ -39,6 +37,7 @@ import {
|
||||
import { lookupSnapshotRef } from "./snapshot-ref";
|
||||
|
||||
const DEFAULT_HELP_TIMEOUT_MS = 300_000;
|
||||
const HELP_ATTR = "data-bsk-help";
|
||||
const HELP_SEND_RETRIES = 3;
|
||||
const HELP_SEND_RETRY_DELAY_MS = 350;
|
||||
const HELP_REARM_DEBOUNCE_MS = 150;
|
||||
@@ -75,7 +74,6 @@ interface ActiveHelpRequest {
|
||||
title?: string;
|
||||
targets: HelpTarget[];
|
||||
selectors: string[];
|
||||
rects: HelpHighlightRect[];
|
||||
timeoutMs: number;
|
||||
notificationId: string;
|
||||
resolvedTargets?: ResolvedTarget[];
|
||||
@@ -136,7 +134,6 @@ function helpRequestMessage(help: ActiveHelpRequest, tabId: number): HelpRequest
|
||||
...(help.title ? { title: help.title } : {}),
|
||||
displayMode: isPrimaryTab ? "full" : "compact",
|
||||
selectors: isPrimaryTab ? help.selectors : [],
|
||||
rects: isPrimaryTab ? help.rects : [],
|
||||
timeoutMs: help.timeoutMs,
|
||||
};
|
||||
}
|
||||
@@ -168,6 +165,33 @@ async function sendHelpRequestWithAck(
|
||||
throw lastError ?? new Error("failed to show help overlay");
|
||||
}
|
||||
|
||||
async function tagRefTarget(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
backendNodeId: number,
|
||||
index: number,
|
||||
): Promise<string | null> {
|
||||
let objectId: string | undefined;
|
||||
try {
|
||||
const resolved = await cdp.send<{ object?: { objectId?: string } }>(tabId, "DOM.resolveNode", {
|
||||
backendNodeId,
|
||||
});
|
||||
objectId = resolved.object?.objectId;
|
||||
if (!objectId) return null;
|
||||
await cdp.send(tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function(){ this.setAttribute("${HELP_ATTR}", "${index}"); }`,
|
||||
});
|
||||
return `[${HELP_ATTR}="${index}"]`;
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
if (objectId) {
|
||||
await cdp.send(tabId, "Runtime.releaseObject", { objectId }).catch(() => {});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function selectorExists(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
@@ -185,22 +209,36 @@ async function selectorExists(
|
||||
}
|
||||
}
|
||||
|
||||
async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise<void> {
|
||||
if (!cdp) return;
|
||||
try {
|
||||
const doc = await cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
||||
depth: 0,
|
||||
});
|
||||
const rootId = doc.root?.nodeId;
|
||||
if (rootId === undefined) return;
|
||||
const { nodeIds } = await cdp.send<{ nodeIds: number[] }>(tabId, "DOM.querySelectorAll", {
|
||||
nodeId: rootId,
|
||||
selector: `[${HELP_ATTR}]`,
|
||||
});
|
||||
for (const nodeId of nodeIds ?? []) {
|
||||
await cdp.send(tabId, "DOM.removeAttribute", { nodeId, name: HELP_ATTR }).catch(() => {});
|
||||
}
|
||||
} catch {
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveHelpTargets(
|
||||
ctx: ReturnType<SessionManager["get"]>,
|
||||
tabId: number,
|
||||
targets: HelpTarget[],
|
||||
deps: RequestHelpDeps,
|
||||
options: { scrollIntoView: boolean },
|
||||
): Promise<{
|
||||
selectors: string[];
|
||||
rects: HelpHighlightRect[];
|
||||
resolvedTargets?: ResolvedTarget[];
|
||||
}> {
|
||||
): Promise<{ selectors: string[]; resolvedTargets?: ResolvedTarget[] }> {
|
||||
const selectors: string[] = [];
|
||||
const rects: HelpHighlightRect[] = [];
|
||||
const resolved: ResolvedTarget[] = [];
|
||||
let rootNodeId: number | undefined;
|
||||
if (deps.cdp && targets.some((target) => Boolean(target.selector))) {
|
||||
if (deps.cdp && targets.length > 0) {
|
||||
try {
|
||||
const doc = await deps.cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
||||
depth: 0,
|
||||
@@ -224,55 +262,28 @@ async function resolveHelpTargets(
|
||||
resolved.push({ matched, selector: tgt.selector });
|
||||
} else if (tgt.ref) {
|
||||
const looked = ctx ? lookupSnapshotRef(ctx, tgt.ref, tabId) : null;
|
||||
let rect: HelpHighlightRect | null = null;
|
||||
if (looked && deps.cdp) {
|
||||
const target = {
|
||||
tabId,
|
||||
...(looked.cdpSessionId ? { sessionId: looked.cdpSessionId } : {}),
|
||||
};
|
||||
const geometry = await resolveNodeGeometry(
|
||||
deps.cdp,
|
||||
tabId,
|
||||
{
|
||||
target,
|
||||
backendNodeId: looked.backendNodeId,
|
||||
...(looked.frameId ? { frameId: looked.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: options.scrollIntoView },
|
||||
);
|
||||
if (!isRpcError(geometry)) {
|
||||
rect = {
|
||||
top: geometry.topBounds.y,
|
||||
left: geometry.topBounds.x,
|
||||
width: geometry.topBounds.width,
|
||||
height: geometry.topBounds.height,
|
||||
};
|
||||
}
|
||||
const backendNodeId = looked?.backendNodeId ?? null;
|
||||
let sel: string | null = null;
|
||||
if (backendNodeId !== null && deps.cdp) {
|
||||
sel = await tagRefTarget(deps.cdp, tabId, backendNodeId, i);
|
||||
}
|
||||
if (rect) rects.push(rect);
|
||||
resolved.push({
|
||||
matched: rect !== null,
|
||||
ref: tgt.ref,
|
||||
});
|
||||
if (sel) selectors.push(sel);
|
||||
resolved.push({ matched: sel !== null, ref: tgt.ref });
|
||||
}
|
||||
}
|
||||
|
||||
return { selectors, rects, resolvedTargets: resolved.length > 0 ? resolved : undefined };
|
||||
return { selectors, resolvedTargets: resolved.length > 0 ? resolved : undefined };
|
||||
}
|
||||
|
||||
async function refreshHelpTargets(
|
||||
help: ActiveHelpRequest,
|
||||
options: { scrollIntoView: boolean } = { scrollIntoView: true },
|
||||
): Promise<void> {
|
||||
const { selectors, rects, resolvedTargets } = await resolveHelpTargets(
|
||||
async function refreshHelpTargets(help: ActiveHelpRequest): Promise<void> {
|
||||
await clearRefTags(help.deps.cdp, help.primaryTabId);
|
||||
const { selectors, resolvedTargets } = await resolveHelpTargets(
|
||||
help.ctx,
|
||||
help.primaryTabId,
|
||||
help.targets,
|
||||
help.deps,
|
||||
options,
|
||||
);
|
||||
help.selectors = selectors;
|
||||
help.rects = rects;
|
||||
help.resolvedTargets = resolvedTargets;
|
||||
}
|
||||
|
||||
@@ -283,7 +294,12 @@ async function cleanupHelp(help: ActiveHelpRequest): Promise<void> {
|
||||
const tabsToCancel = new Set([help.primaryTabId, ...help.overlayTabIds]);
|
||||
await Promise.all(
|
||||
[...tabsToCancel].map((tabId) =>
|
||||
help.deps.sendToTab(tabId, { type: HELP_CANCEL, requestId: help.requestId }).catch(() => {}),
|
||||
Promise.all([
|
||||
clearRefTags(help.deps.cdp, tabId),
|
||||
help.deps
|
||||
.sendToTab(tabId, { type: HELP_CANCEL, requestId: help.requestId })
|
||||
.catch(() => {}),
|
||||
]),
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -460,9 +476,7 @@ function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void {
|
||||
sendResponse({ active: false });
|
||||
return;
|
||||
}
|
||||
if (tabId === help.primaryTabId) {
|
||||
await refreshHelpTargets(help, { scrollIntoView: false });
|
||||
}
|
||||
if (tabId === help.primaryTabId) await refreshHelpTargets(help);
|
||||
help.overlayTabIds.add(tabId);
|
||||
sendResponse({
|
||||
active: true,
|
||||
@@ -472,7 +486,6 @@ function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void {
|
||||
...(help.title ? { title: help.title } : {}),
|
||||
displayMode: tabId === help.primaryTabId ? "full" : "compact",
|
||||
selectors: tabId === help.primaryTabId ? help.selectors : [],
|
||||
rects: tabId === help.primaryTabId ? help.rects : [],
|
||||
timeoutMs: help.timeoutMs,
|
||||
},
|
||||
});
|
||||
@@ -673,13 +686,7 @@ export async function handleRequestHelp(
|
||||
if (deps.autoAttachLifecycle !== false) ensureHelpLifecycleListeners(deps);
|
||||
const tabId = target.tabId;
|
||||
const initialTargets = params.targets ?? [];
|
||||
const { selectors, rects, resolvedTargets } = await resolveHelpTargets(
|
||||
ctx,
|
||||
tabId,
|
||||
initialTargets,
|
||||
deps,
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
const { selectors, resolvedTargets } = await resolveHelpTargets(ctx, tabId, initialTargets, deps);
|
||||
|
||||
await deps.windows.update(target.windowId, { focused: true }).catch(() => {});
|
||||
await deps.activateTab(tabId).catch(() => {});
|
||||
@@ -722,7 +729,6 @@ export async function handleRequestHelp(
|
||||
...(params.title ? { title: params.title } : {}),
|
||||
targets: initialTargets,
|
||||
selectors,
|
||||
rects,
|
||||
timeoutMs,
|
||||
notificationId,
|
||||
resolvedTargets,
|
||||
|
||||
@@ -3,15 +3,14 @@
|
||||
//
|
||||
// All interaction tools:
|
||||
// 1. Resolve target tab (sandbox: must be inside Agent Window).
|
||||
// 2. Resolve target element by `ref` (compound frame/session identity)
|
||||
// or `selector` (DOM.querySelector + describeNode).
|
||||
// 3. Scroll the node and its frame owners into view, project its live
|
||||
// content quad into the top viewport, then dispatch `Input.*` events.
|
||||
// 2. Resolve target element by `ref` (RefStore.resolve with tabId
|
||||
// binding) or `selector` (DOM.querySelector + describeNode).
|
||||
// 3. Scroll the node into view, then dispatch the appropriate
|
||||
// `Input.*` CDP events.
|
||||
// 4. Honour `AbortSignal` so canceled calls don't issue follow-up CDP
|
||||
// commands.
|
||||
|
||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||
import type {
|
||||
ClickParams,
|
||||
@@ -29,13 +28,17 @@ import type {
|
||||
SelectResult,
|
||||
} from "@/transport/types";
|
||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||
import { backendNodeToObject } from "./element-geometry";
|
||||
import {
|
||||
backendNodeToObject,
|
||||
boxCentre,
|
||||
nodeCentre,
|
||||
quadCentre,
|
||||
scrollNodeIntoView,
|
||||
} from "./element-geometry";
|
||||
import { rpcError } from "./errors";
|
||||
import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry";
|
||||
import {
|
||||
type CdpRunner,
|
||||
type ChromeTabsApi,
|
||||
cdpRunnerForTarget,
|
||||
chromeTabsApi,
|
||||
enforceAgentWindow,
|
||||
isRpcError,
|
||||
@@ -134,16 +137,7 @@ async function resolveBackendNode(
|
||||
target: { tabId: number },
|
||||
params: { ref?: string; selector?: string },
|
||||
toolName: string,
|
||||
): Promise<
|
||||
| {
|
||||
backendNodeId: number;
|
||||
cdpTarget: CdpTarget;
|
||||
frameId?: string;
|
||||
usedRef?: string;
|
||||
usedSelector?: string;
|
||||
}
|
||||
| RpcError
|
||||
> {
|
||||
): Promise<{ backendNodeId: number; usedRef?: string; usedSelector?: string } | RpcError> {
|
||||
const hasRef = typeof params.ref === "string" && params.ref.length > 0;
|
||||
const hasSelector = typeof params.selector === "string" && params.selector.length > 0;
|
||||
if (hasRef && hasSelector) {
|
||||
@@ -161,15 +155,7 @@ async function resolveBackendNode(
|
||||
if (hasRef) {
|
||||
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId);
|
||||
if (isRpcError(resolved)) return resolved;
|
||||
return {
|
||||
backendNodeId: resolved.backendNodeId,
|
||||
cdpTarget: {
|
||||
tabId: target.tabId,
|
||||
...(resolved.cdpSessionId ? { sessionId: resolved.cdpSessionId } : {}),
|
||||
},
|
||||
...(resolved.frameId ? { frameId: resolved.frameId } : {}),
|
||||
usedRef: resolved.refKey,
|
||||
};
|
||||
return { backendNodeId: resolved.backendNodeId, usedRef: resolved.refKey };
|
||||
}
|
||||
// selector path
|
||||
try {
|
||||
@@ -206,7 +192,7 @@ async function resolveBackendNode(
|
||||
message: "DOM.describeNode returned no backendNodeId",
|
||||
};
|
||||
}
|
||||
return { backendNodeId, cdpTarget: { tabId: target.tabId }, usedSelector: params.selector };
|
||||
return { backendNodeId, usedSelector: params.selector };
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -242,19 +228,19 @@ export async function handleClick(
|
||||
return { code: "cancelled", message: "click aborted" };
|
||||
}
|
||||
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const geometry = await resolveNodeGeometry(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
{
|
||||
target: node.cdpTarget,
|
||||
backendNodeId: node.backendNodeId,
|
||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
const centre = geometry.actionPoint;
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (scrollErr) return scrollErr;
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
|
||||
const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(centre)) return centre;
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "click aborted" };
|
||||
@@ -367,19 +353,19 @@ export async function handleHover(
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "hover");
|
||||
if (isRpcError(node)) return node;
|
||||
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const geometry = await resolveNodeGeometry(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
{
|
||||
target: node.cdpTarget,
|
||||
backendNodeId: node.backendNodeId,
|
||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
const centre = geometry.actionPoint;
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (scrollErr) return scrollErr;
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
|
||||
const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(centre)) return centre;
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "hover aborted" };
|
||||
@@ -599,11 +585,10 @@ export async function handleFill(
|
||||
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill");
|
||||
if (isRpcError(node)) return node;
|
||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const described = await nodeCdp.send<{ node?: DescribedNode }>(
|
||||
const described = await deps.cdp.send<{ node?: DescribedNode }>(
|
||||
target.tabId,
|
||||
"DOM.describeNode",
|
||||
{
|
||||
@@ -617,18 +602,12 @@ export async function handleFill(
|
||||
`element ${described.node?.nodeName ?? "?"} not fillable (need input/textarea/contenteditable)`,
|
||||
);
|
||||
}
|
||||
const scrollErr = await scrollElementAndFramesIntoView(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
node.cdpTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
);
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (scrollErr) return scrollErr;
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -640,7 +619,7 @@ export async function handleFill(
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
|
||||
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId);
|
||||
const objectIdOrErr = await backendNodeToObject(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
||||
const objectId = objectIdOrErr;
|
||||
const clearBefore = params.clear_before ?? true;
|
||||
@@ -649,7 +628,7 @@ export async function handleFill(
|
||||
if (clearBefore) {
|
||||
// Clear input/textarea value or wipe contenteditable innerText,
|
||||
// then fire `input` so frameworks observe the empty state.
|
||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
await deps.cdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
if (this.isContentEditable) { this.textContent = ''; }
|
||||
@@ -676,7 +655,7 @@ export async function handleFill(
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
// Fire `input` + `change` so React / Vue controlled inputs commit.
|
||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
await deps.cdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
@@ -874,21 +853,14 @@ export async function handlePress(
|
||||
if (params.ref || params.selector) {
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press");
|
||||
if (isRpcError(node)) return node;
|
||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const scrollErr = await scrollElementAndFramesIntoView(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
node.cdpTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
);
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (scrollErr) return scrollErr;
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "press aborted" };
|
||||
}
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -993,11 +965,10 @@ export async function handleSelect(
|
||||
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "select");
|
||||
if (isRpcError(node)) return node;
|
||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const described = await nodeCdp.send<{ node?: DescribedNode }>(
|
||||
const described = await deps.cdp.send<{ node?: DescribedNode }>(
|
||||
target.tabId,
|
||||
"DOM.describeNode",
|
||||
{ backendNodeId: node.backendNodeId },
|
||||
@@ -1020,18 +991,12 @@ export async function handleSelect(
|
||||
"single-select <select> requires exactly one value",
|
||||
);
|
||||
}
|
||||
const scrollErr = await scrollElementAndFramesIntoView(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
node.cdpTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
);
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (scrollErr) return scrollErr;
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "select aborted" };
|
||||
}
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -1043,12 +1008,12 @@ export async function handleSelect(
|
||||
return { code: "cancelled", message: "select aborted" };
|
||||
}
|
||||
|
||||
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId);
|
||||
const objectIdOrErr = await backendNodeToObject(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
||||
const objectId = objectIdOrErr;
|
||||
|
||||
try {
|
||||
const evaluated = await nodeCdp.send<{
|
||||
const evaluated = await deps.cdp.send<{
|
||||
result?: { value?: SelectMutationResult | null };
|
||||
}>(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
@@ -1107,6 +1072,9 @@ export async function handleSelect(
|
||||
|
||||
export const __testing__ = {
|
||||
DEFAULT_TIMEOUT_MS,
|
||||
quadCentre,
|
||||
boxCentre,
|
||||
resolveBackendNode,
|
||||
nodeCentre,
|
||||
isFillable,
|
||||
};
|
||||
|
||||
@@ -37,22 +37,17 @@ export async function handleNetwork(
|
||||
manager: SessionManager,
|
||||
params: NetworkParams,
|
||||
deps: NetworkDeps = defaultNetworkDeps(),
|
||||
signal?: AbortSignal,
|
||||
): Promise<NetworkResult | RpcError> {
|
||||
if (signal?.aborted) return { code: "cancelled", message: "network aborted" };
|
||||
const ctxOrErr = lookupSession(manager, params, "network");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const bounds = parseBufferedReadBounds(params);
|
||||
if (isRpcError(bounds)) return bounds;
|
||||
const target = await resolveTargetTab(manager, ctxOrErr, params.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
if (signal?.aborted) return { code: "cancelled", message: "network aborted" };
|
||||
|
||||
try {
|
||||
if (signal?.aborted) return { code: "cancelled", message: "network aborted" };
|
||||
deps.cdp.trackSessionTab?.(ctxOrErr.sessionId, target.tabId);
|
||||
await deps.cdp.ensureNetworkCapture(target.tabId);
|
||||
if (signal?.aborted) return { code: "cancelled", message: "network aborted" };
|
||||
return deps.cdp.networkEntriesSince(
|
||||
target.tabId,
|
||||
bounds.since,
|
||||
|
||||
@@ -8,13 +8,10 @@ import {
|
||||
applyVomInteractionRecovery,
|
||||
type CondSurface,
|
||||
isVomReferenceNode,
|
||||
renderVom,
|
||||
type VomNode,
|
||||
type VomOptions,
|
||||
type VomScene,
|
||||
} from "@browser-skill/vom";
|
||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import {
|
||||
type CaptureSuppressSendToTab,
|
||||
withOverlaysHiddenForCapture,
|
||||
@@ -31,9 +28,10 @@ import type {
|
||||
SnapshotParams,
|
||||
SnapshotResult,
|
||||
} from "@/transport/types";
|
||||
import { captureVomObservation } from "./capture-vom-observation";
|
||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||
import { nodeBoundingRect, scrollNodeIntoView } from "./element-geometry";
|
||||
import { rpcError } from "./errors";
|
||||
import { resolveNodeGeometry } from "./frame-geometry";
|
||||
import {
|
||||
type ChromeTabsApi,
|
||||
enforceToolTargetScope,
|
||||
@@ -42,22 +40,11 @@ import {
|
||||
type ResolvedTargetTab,
|
||||
resolveCdpAccessibleTargetTab,
|
||||
type CdpRunner as SharedCdpRunner,
|
||||
sendToCdpTarget,
|
||||
normaliseRef as sharedNormaliseRef,
|
||||
type ToolEffect,
|
||||
} from "./shared";
|
||||
import { resolveSnapshotRef } from "./snapshot-ref";
|
||||
import {
|
||||
type CapturedNode,
|
||||
type CapturedViewModel,
|
||||
captureViewModel,
|
||||
collectOverlayExcludedBackendIds,
|
||||
} from "./vom/capture";
|
||||
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
|
||||
import {
|
||||
type CaptureVomObservationResult,
|
||||
projectRecordSafeObservation,
|
||||
} from "./vom/record-safe-observation";
|
||||
import { type CapturedNode, type CapturedViewModel } from "./vom/capture";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared helpers (legacy aliases — observation.ts kept exporting these
|
||||
@@ -145,66 +132,35 @@ function defaultScreenshotDeps(): ScreenshotDeps {
|
||||
};
|
||||
}
|
||||
|
||||
function cancelled(tool: string): RpcError {
|
||||
return { code: "cancelled", message: `${tool} aborted` };
|
||||
}
|
||||
|
||||
function abortError(tool: string): Error {
|
||||
const error = new Error(`${tool} aborted`);
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
function isAbortError(error: unknown): boolean {
|
||||
return (
|
||||
typeof error === "object" &&
|
||||
error !== null &&
|
||||
(error as { name?: string }).name === "AbortError"
|
||||
);
|
||||
}
|
||||
|
||||
function throwIfAborted(signal: AbortSignal | undefined, tool: string): void {
|
||||
if (signal?.aborted) throw abortError(tool);
|
||||
}
|
||||
|
||||
async function captureElementScreenshot(
|
||||
cdp: SharedCdpRunner,
|
||||
tabId: number,
|
||||
target: CdpTarget,
|
||||
backendNodeId: number,
|
||||
frameId?: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ image_base64: string; width: number; height: number } | RpcError> {
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const geometry = await resolveNodeGeometry(
|
||||
cdp,
|
||||
tabId,
|
||||
{ target, backendNodeId, ...(frameId ? { frameId } : {}) },
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const rect = geometry.topBounds;
|
||||
const scrollErr = await scrollNodeIntoView(cdp, tabId, backendNodeId);
|
||||
if (scrollErr) return scrollErr;
|
||||
|
||||
const rectOrErr = await nodeBoundingRect(cdp, tabId, backendNodeId);
|
||||
if (isRpcError(rectOrErr)) return rectOrErr;
|
||||
|
||||
try {
|
||||
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
|
||||
format: "png",
|
||||
clip: {
|
||||
x: rect.x,
|
||||
y: rect.y,
|
||||
width: rect.width,
|
||||
height: rect.height,
|
||||
x: rectOrErr.x,
|
||||
y: rectOrErr.y,
|
||||
width: rectOrErr.width,
|
||||
height: rectOrErr.height,
|
||||
scale: 1,
|
||||
},
|
||||
});
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const image_base64 = shot.data ?? "";
|
||||
if (!image_base64) {
|
||||
return { code: "cdp_failed", message: "Page.captureScreenshot returned no data" };
|
||||
}
|
||||
const dims = parsePngDimensions(image_base64) ?? {
|
||||
width: Math.round(rect.width),
|
||||
height: Math.round(rect.height),
|
||||
width: Math.round(rectOrErr.width),
|
||||
height: Math.round(rectOrErr.height),
|
||||
};
|
||||
return { image_base64, width: dims.width, height: dims.height };
|
||||
} catch (err) {
|
||||
@@ -232,15 +188,11 @@ async function captureFullTabPng(
|
||||
deps: ScreenshotDeps,
|
||||
ctx: SessionContext,
|
||||
target: ResolvedTargetTab,
|
||||
signal?: AbortSignal,
|
||||
): Promise<string | RpcError> {
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
try {
|
||||
const dataUrl = await deps.captureApi.captureVisibleTab(target.windowId, { format: "png" });
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
return stripDataUrlPrefix(dataUrl);
|
||||
} catch (primaryErr) {
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const cdp = deps.cdp;
|
||||
if (!cdp) {
|
||||
return {
|
||||
@@ -250,15 +202,12 @@ async function captureFullTabPng(
|
||||
}
|
||||
let fallbackMsg: string;
|
||||
try {
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const shot = await cdp.send<{ data?: string }>(target.tabId, "Page.captureScreenshot", {
|
||||
format: "png",
|
||||
fromSurface: true,
|
||||
});
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
if (shot.data) return shot.data;
|
||||
fallbackMsg = "Page.captureScreenshot returned no data";
|
||||
} catch (fallbackErr) {
|
||||
@@ -277,9 +226,7 @@ export async function handleScreenshot(
|
||||
manager: SessionManager,
|
||||
params: ScreenshotParams,
|
||||
deps: ScreenshotDeps = defaultScreenshotDeps(),
|
||||
signal?: AbortSignal,
|
||||
): Promise<ScreenshotResult | RpcError> {
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const ctxOrErr = lookupSession(manager, params, "screenshot");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
@@ -291,7 +238,6 @@ export async function handleScreenshot(
|
||||
"screenshot",
|
||||
);
|
||||
if (isRpcError(target)) return target;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const dialogCursor = deps.cdp ? markDialogCursor(deps.cdp, target.tabId) : 0;
|
||||
const withShotDialogs = <T extends object>(result: T) =>
|
||||
deps.cdp ? attachDialogs(deps.cdp, target.tabId, dialogCursor, result) : result;
|
||||
@@ -303,30 +249,15 @@ export async function handleScreenshot(
|
||||
}
|
||||
const node = resolveSnapshotRef(ctx, ref, target.tabId);
|
||||
if (isRpcError(node)) return node;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const cdp = deps.cdp;
|
||||
const nodeTarget = {
|
||||
tabId: target.tabId,
|
||||
...(node.cdpSessionId ? { sessionId: node.cdpSessionId } : {}),
|
||||
};
|
||||
const captured = await withOverlaysHiddenForCapture(
|
||||
target.tabId,
|
||||
() =>
|
||||
captureElementScreenshot(
|
||||
cdp,
|
||||
target.tabId,
|
||||
nodeTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
signal,
|
||||
),
|
||||
() => captureElementScreenshot(cdp, target.tabId, node.backendNodeId),
|
||||
deps.sendToTab,
|
||||
);
|
||||
if (isRpcError(captured)) return captured;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
return withShotDialogs({
|
||||
image_base64: captured.image_base64,
|
||||
width: captured.width,
|
||||
@@ -346,11 +277,10 @@ export async function handleScreenshot(
|
||||
|
||||
const captured = await withOverlaysHiddenForCapture(
|
||||
target.tabId,
|
||||
() => captureFullTabPng(deps, ctx, target, signal),
|
||||
() => captureFullTabPng(deps, ctx, target),
|
||||
deps.sendToTab,
|
||||
);
|
||||
if (isRpcError(captured)) return captured;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const image_base64 = captured;
|
||||
const dims = parsePngDimensions(image_base64) ?? { width: 0, height: 0 };
|
||||
return withShotDialogs({
|
||||
@@ -376,7 +306,6 @@ export type CdpRunner = SharedCdpRunner;
|
||||
/** Subset of CDP `AXNode` we care about — see `Accessibility.AXNode`. */
|
||||
export interface CdpAxNode {
|
||||
nodeId: string;
|
||||
frameId?: string;
|
||||
parentId?: string;
|
||||
backendDOMNodeId?: number;
|
||||
ignored?: boolean;
|
||||
@@ -419,12 +348,15 @@ function isSensitive(capturedNode: CapturedNode | undefined): boolean {
|
||||
const type = (attrs.type ?? "").toLowerCase();
|
||||
if (type === "password") return true;
|
||||
const autocomplete = (attrs.autocomplete ?? "").toLowerCase();
|
||||
return (
|
||||
autocomplete.startsWith("cc-") ||
|
||||
if (autocomplete.startsWith("cc-")) return true;
|
||||
if (
|
||||
autocomplete === "one-time-code" ||
|
||||
autocomplete === "current-password" ||
|
||||
autocomplete === "new-password"
|
||||
);
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
interface AxNodeSignals {
|
||||
@@ -623,7 +555,7 @@ function iframeNameFor(node: CapturedNode): string | undefined {
|
||||
function isRenderableIframeControl(node: CapturedNode): boolean {
|
||||
const tag = normalizeTag(node.tag);
|
||||
if (!IFRAME_RENDERABLE_TAGS.has(tag)) return false;
|
||||
if (!node.localRect && !node.rect) return false;
|
||||
if (!node.rect) return false;
|
||||
if (node.pointerEvents === "none") return false;
|
||||
if ((node.attrs.type ?? "").toLowerCase() === "hidden") return false;
|
||||
|
||||
@@ -795,12 +727,7 @@ function applyCapturedSignals(
|
||||
signals: VomNodeDomSignals,
|
||||
): VomNode {
|
||||
if (!capturedNode) return node;
|
||||
const attrs =
|
||||
node.sensitive && Object.prototype.hasOwnProperty.call(capturedNode.attrs, "value")
|
||||
? Object.fromEntries(
|
||||
Object.entries(capturedNode.attrs).filter(([name]) => name.toLowerCase() !== "value"),
|
||||
)
|
||||
: capturedNode.attrs;
|
||||
const attrs = capturedNode.attrs;
|
||||
const text = cleanAttr(capturedNode.textContent);
|
||||
const nearbyText = nearbyTextFor(capturedNode, signals.childrenByParentId);
|
||||
const placeholder = cleanAttr(capturedNode.formPlaceholder) ?? cleanAttr(attrs.placeholder);
|
||||
@@ -867,7 +794,8 @@ function vomNodeFromCaptured(
|
||||
signals: VomNodeDomSignals,
|
||||
): VomNode {
|
||||
const sensitive = isSensitive(capturedNode);
|
||||
const value = sensitive ? undefined : capturedNode.formValue;
|
||||
const value =
|
||||
capturedNode.formValue ?? (sensitive ? (capturedNode.attrs.value ?? "") : undefined);
|
||||
const tag = normalizeTag(capturedNode.tag);
|
||||
const node = applyCapturedSignals(
|
||||
{
|
||||
@@ -903,17 +831,8 @@ function axVomNode(
|
||||
): VomNode {
|
||||
const role = axString(axNode.role);
|
||||
const capturedValue = capturedNode?.formValue;
|
||||
const sourceValue = capturedValue ?? axValue(axNode.value);
|
||||
const value = capturedValue ?? axValue(axNode.value);
|
||||
const signalsForNode = axSignals.get(axNode.nodeId);
|
||||
const sensitive = isSensitive(capturedNode) || signalsForNode?.sensitive === true;
|
||||
const value = sensitive ? undefined : sourceValue;
|
||||
const inputState =
|
||||
inputStateFor(capturedNode, sourceValue) ??
|
||||
(sensitive && ["textbox", "searchbox"].includes(role?.toLowerCase() ?? "")
|
||||
? sourceValue === undefined || sourceValue === ""
|
||||
? "empty"
|
||||
: "filled"
|
||||
: undefined);
|
||||
let name = FORM_CONTROL_TAGS.has(normalizeTag(capturedNode?.tag))
|
||||
? formControlName(capturedNode, axString(axNode.name), axSiblingLabel(axNode, axById), signals)
|
||||
: (axString(axNode.name) ?? signalsForNode?.aggregatedText);
|
||||
@@ -929,8 +848,8 @@ function axVomNode(
|
||||
position: capturedNode?.position || "static",
|
||||
pointerEvents: capturedNode?.pointerEvents || "auto",
|
||||
modal: isModalSignal(axNode, capturedNode),
|
||||
sensitive,
|
||||
inputState,
|
||||
sensitive: isSensitive(capturedNode) || signalsForNode?.sensitive === true,
|
||||
inputState: inputStateFor(capturedNode, value),
|
||||
};
|
||||
if (role) node.role = role;
|
||||
if (name) node.name = name;
|
||||
@@ -1193,8 +1112,6 @@ export interface BuildVomSceneOptions {
|
||||
pageUrl?: string;
|
||||
}
|
||||
|
||||
export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>;
|
||||
|
||||
function externalHrefHost(
|
||||
href: string | undefined,
|
||||
pageUrl: string | undefined,
|
||||
@@ -1288,119 +1205,6 @@ export function buildVomScene(
|
||||
};
|
||||
}
|
||||
|
||||
export function buildFrameVomScene(
|
||||
documents: VomFrameDocument[],
|
||||
captured: CapturedViewModel,
|
||||
options: BuildVomSceneOptions = {},
|
||||
): VomScene {
|
||||
const documentByFrameId = new Map(documents.map((document) => [document.frameId, document]));
|
||||
const rootFrameId =
|
||||
captured.rootFrameId ?? documents.find((document) => !document.parentFrameId)?.frameId;
|
||||
const orderedFrameIds = [
|
||||
...(rootFrameId ? [rootFrameId] : []),
|
||||
...documents.map((document) => document.frameId),
|
||||
].filter((frameId, index, all) => all.indexOf(frameId) === index);
|
||||
|
||||
const localScenes = new Map<string, VomScene>();
|
||||
for (const frameId of orderedFrameIds) {
|
||||
const document = documentByFrameId.get(frameId);
|
||||
if (!document) continue;
|
||||
localScenes.set(
|
||||
frameId,
|
||||
buildVomScene(
|
||||
document.axNodes,
|
||||
{
|
||||
viewport: captured.viewport,
|
||||
nodes: document.domNodes,
|
||||
iframeNodes: new Map(),
|
||||
surfaceProbes: frameId === rootFrameId ? captured.surfaceProbes : [],
|
||||
excludedBackendNodeIds:
|
||||
frameId === rootFrameId ? captured.excludedBackendNodeIds : new Set(),
|
||||
},
|
||||
{ pageUrl: document.url ?? options.pageUrl },
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const sceneIdByFrameAndBackend = new Map<string, number>();
|
||||
let nextSceneId = 1;
|
||||
for (const [frameId, scene] of localScenes) {
|
||||
for (const node of scene.nodes) {
|
||||
sceneIdByFrameAndBackend.set(`${frameId}:${node.id}`, nextSceneId);
|
||||
nextSceneId += 1;
|
||||
}
|
||||
}
|
||||
|
||||
const nodes: VomNode[] = [];
|
||||
const surfaces: CondSurface[] = [];
|
||||
const activeScopeBlocks: ActiveScopeBlock[] = [];
|
||||
|
||||
for (const [frameId, scene] of localScenes) {
|
||||
const document = documentByFrameId.get(frameId);
|
||||
const ownerBackendNodeId = document?.ownerBackendNodeId;
|
||||
const parentFrameId = document?.parentFrameId;
|
||||
const ownerSceneId =
|
||||
ownerBackendNodeId !== undefined && parentFrameId
|
||||
? sceneIdByFrameAndBackend.get(`${parentFrameId}:${ownerBackendNodeId}`)
|
||||
: undefined;
|
||||
if (frameId !== rootFrameId && ownerSceneId === undefined) continue;
|
||||
|
||||
const suppressedRoots = new Set(
|
||||
frameId === rootFrameId
|
||||
? []
|
||||
: scene.nodes
|
||||
.filter(
|
||||
(node) =>
|
||||
node.parentId === null &&
|
||||
["rootwebarea", "webarea"].includes(node.role?.toLowerCase() ?? ""),
|
||||
)
|
||||
.map((node) => node.id),
|
||||
);
|
||||
|
||||
for (const node of scene.nodes) {
|
||||
if (suppressedRoots.has(node.id)) continue;
|
||||
const sceneNodeId = sceneIdByFrameAndBackend.get(`${frameId}:${node.id}`) as number;
|
||||
let parentId =
|
||||
node.parentId !== null
|
||||
? sceneIdByFrameAndBackend.get(`${frameId}:${node.parentId}`)
|
||||
: ownerSceneId;
|
||||
if (node.parentId !== null && suppressedRoots.has(node.parentId)) parentId = ownerSceneId;
|
||||
const remapId = (id: number) => sceneIdByFrameAndBackend.get(`${frameId}:${id}`);
|
||||
nodes.push({
|
||||
...node,
|
||||
id: sceneNodeId,
|
||||
backendNodeId: node.id,
|
||||
frameId,
|
||||
contextScopeId: document?.contextScopeId ?? frameId,
|
||||
parentId: parentId ?? null,
|
||||
...(node.domParentId !== undefined
|
||||
? { domParentId: node.domParentId === null ? null : (remapId(node.domParentId) ?? null) }
|
||||
: {}),
|
||||
...(node.domAncestorIds
|
||||
? { domAncestorIds: node.domAncestorIds.flatMap((id) => remapId(id) ?? []) }
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
for (const surface of scene.surfaces ?? []) {
|
||||
const triggerId = sceneIdByFrameAndBackend.get(`${frameId}:${surface.triggerId}`);
|
||||
if (triggerId !== undefined) surfaces.push({ ...surface, triggerId });
|
||||
}
|
||||
for (const block of scene.activeScopeBlocks ?? []) {
|
||||
const triggerId = sceneIdByFrameAndBackend.get(`${frameId}:${block.triggerId}`);
|
||||
if (triggerId !== undefined) activeScopeBlocks.push({ ...block, triggerId });
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
viewport: captured.viewport,
|
||||
nodes,
|
||||
...(rootFrameId ? { rootFrameId } : {}),
|
||||
...(surfaces.length > 0 ? { surfaces } : {}),
|
||||
...(activeScopeBlocks.length > 0 ? { activeScopeBlocks } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export interface SnapshotDeps {
|
||||
cdp: CdpRunner;
|
||||
tabsApi: {
|
||||
@@ -1460,9 +1264,7 @@ export async function handleGetHtml(
|
||||
manager: SessionManager,
|
||||
params: GetHtmlParams,
|
||||
deps: SnapshotDeps = getDefaultDeps(),
|
||||
signal?: AbortSignal,
|
||||
): Promise<GetHtmlResult | RpcError> {
|
||||
if (signal?.aborted) return cancelled("get_html");
|
||||
const ctxOrErr = lookupSession(manager, params, "get_html");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
@@ -1474,31 +1276,21 @@ export async function handleGetHtml(
|
||||
"get_html",
|
||||
);
|
||||
if (isRpcError(target)) return target;
|
||||
if (signal?.aborted) return cancelled("get_html");
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
|
||||
const maxBytes =
|
||||
params.max_bytes && params.max_bytes > 0 ? params.max_bytes : DEFAULT_GET_HTML_MAX_BYTES;
|
||||
|
||||
try {
|
||||
throwIfAborted(signal, "get_html");
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||
throwIfAborted(signal, "get_html");
|
||||
let html: string;
|
||||
if (params.ref) {
|
||||
const resolved = resolveSnapshotRef(ctx, params.ref, target.tabId);
|
||||
if (isRpcError(resolved)) return resolved;
|
||||
const resp = await sendToCdpTarget<{ outerHTML?: string }>(
|
||||
deps.cdp,
|
||||
{
|
||||
tabId: target.tabId,
|
||||
...(resolved.cdpSessionId ? { sessionId: resolved.cdpSessionId } : {}),
|
||||
},
|
||||
"DOM.getOuterHTML",
|
||||
{ backendNodeId: resolved.backendNodeId },
|
||||
);
|
||||
throwIfAborted(signal, "get_html");
|
||||
const resp = await deps.cdp.send<{ outerHTML?: string }>(target.tabId, "DOM.getOuterHTML", {
|
||||
backendNodeId: resolved.backendNodeId,
|
||||
});
|
||||
html = resp.outerHTML ?? "";
|
||||
} else {
|
||||
const doc = await deps.cdp.send<{ root?: { nodeId?: number } }>(
|
||||
@@ -1506,7 +1298,6 @@ export async function handleGetHtml(
|
||||
"DOM.getDocument",
|
||||
{ depth: 0 },
|
||||
);
|
||||
throwIfAborted(signal, "get_html");
|
||||
const nodeId = doc.root?.nodeId;
|
||||
if (typeof nodeId !== "number") {
|
||||
return {
|
||||
@@ -1517,7 +1308,6 @@ export async function handleGetHtml(
|
||||
const resp = await deps.cdp.send<{ outerHTML?: string }>(target.tabId, "DOM.getOuterHTML", {
|
||||
nodeId,
|
||||
});
|
||||
throwIfAborted(signal, "get_html");
|
||||
html = resp.outerHTML ?? "";
|
||||
}
|
||||
const originalBytes = utf8ByteLength(html);
|
||||
@@ -1529,7 +1319,6 @@ export async function handleGetHtml(
|
||||
tab_id: target.tabId,
|
||||
});
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) return cancelled("get_html");
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
@@ -1537,93 +1326,6 @@ export async function handleGetHtml(
|
||||
}
|
||||
}
|
||||
|
||||
function emptyCapturedViewModel(viewport = { width: 0, height: 0 }): CapturedViewModel {
|
||||
return { viewport, nodes: [], iframeNodes: new Map(), excludedBackendNodeIds: new Set() };
|
||||
}
|
||||
|
||||
interface LayoutMetricsViewportReply {
|
||||
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
}
|
||||
|
||||
async function fallbackCapturedViewModel(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CapturedViewModel> {
|
||||
throwIfAborted(signal, "observation");
|
||||
let viewport = { width: 0, height: 0 };
|
||||
try {
|
||||
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
throwIfAborted(signal, "observation");
|
||||
const source = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
viewport = {
|
||||
width: source.clientWidth ?? 0,
|
||||
height: source.clientHeight ?? 0,
|
||||
};
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
}
|
||||
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId, signal);
|
||||
throwIfAborted(signal, "observation");
|
||||
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
async function captureForVom(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
options: CaptureVomObservationOptions,
|
||||
): Promise<CapturedViewModel> {
|
||||
try {
|
||||
return await captureViewModel(cdp, tabId, {
|
||||
conditionalSurfaceProbe: options.conditionalSurfaceProbe ?? false,
|
||||
hoverProbeBypassOverlay: options.hoverProbeBypassOverlay,
|
||||
signal: options.signal,
|
||||
});
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
return fallbackCapturedViewModel(cdp, tabId, options.signal);
|
||||
}
|
||||
}
|
||||
|
||||
export interface CaptureVomObservationOptions extends VomOptions {
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export async function captureVomObservation(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
url: string | undefined,
|
||||
options: CaptureVomObservationOptions = {},
|
||||
): Promise<CaptureVomObservationResult> {
|
||||
throwIfAborted(options.signal, "observation");
|
||||
await cdp.ensureAttachedToUrl?.(tabId, url);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const captured = await captureForVom(cdp, tabId, options);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const documents = await captureFrameData<CdpAxNode>(cdp, tabId, captured, options.signal);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const scene =
|
||||
captured.rootFrameId && captured.frameNodes?.size
|
||||
? buildFrameVomScene(documents, captured, { pageUrl: url })
|
||||
: buildVomScene(documents[0]?.axNodes ?? [], captured, { pageUrl: url });
|
||||
const rendered = renderVom(scene, {
|
||||
maxDepth: options.maxDepth,
|
||||
maxTokens: options.maxTokens,
|
||||
redactValues: options.redactValues,
|
||||
activeRegionPolicy: options.activeRegionPolicy,
|
||||
});
|
||||
throwIfAborted(options.signal, "observation");
|
||||
return projectRecordSafeObservation({
|
||||
rootFrameId: captured.rootFrameId ?? documents[0]?.frameId ?? "root",
|
||||
frameDocuments: documents,
|
||||
rendered,
|
||||
surfaceProbes: captured.surfaceProbes,
|
||||
});
|
||||
}
|
||||
|
||||
async function handleVomObservation(
|
||||
manager: SessionManager,
|
||||
params: SnapshotParams | ObserveParams,
|
||||
@@ -1631,9 +1333,7 @@ async function handleVomObservation(
|
||||
effect: ToolEffect,
|
||||
conditionalSurfaceProbe: boolean,
|
||||
deps: SnapshotDeps = getDefaultDeps(),
|
||||
signal?: AbortSignal,
|
||||
): Promise<SnapshotResult | ObserveResult | RpcError> {
|
||||
if (signal?.aborted) return cancelled(toolName);
|
||||
const ctxOrErr = lookupSession(manager, params, toolName);
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
@@ -1645,51 +1345,35 @@ async function handleVomObservation(
|
||||
toolName,
|
||||
);
|
||||
if (isRpcError(target)) return target;
|
||||
if (signal?.aborted) return cancelled(toolName);
|
||||
const denied = enforceToolTargetScope(ctx, target, effect, toolName);
|
||||
if (denied) return denied;
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
|
||||
try {
|
||||
throwIfAborted(signal, toolName);
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const effectiveConditionalSurfaceProbe =
|
||||
deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe;
|
||||
const observation = await captureVomObservation(deps.cdp, target.tabId, target.url, {
|
||||
const rendered = await captureVomObservation(deps.cdp, target.tabId, target.url, {
|
||||
maxDepth: params.max_depth,
|
||||
maxTokens: params.max_tokens,
|
||||
activeRegionPolicy: true,
|
||||
conditionalSurfaceProbe: effectiveConditionalSurfaceProbe,
|
||||
hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay,
|
||||
signal,
|
||||
});
|
||||
throwIfAborted(signal, toolName);
|
||||
const targetByFrameId = new Map(
|
||||
observation.frames.map((frame) => [frame.frameId, frame.target]),
|
||||
);
|
||||
ctx.refStore.replace(
|
||||
observation.refs.map((ref) => {
|
||||
const refTarget = ref.frameId ? targetByFrameId.get(ref.frameId) : undefined;
|
||||
return [
|
||||
ref.ref,
|
||||
{
|
||||
backendNodeId: ref.backendNodeId,
|
||||
tabId: target.tabId,
|
||||
...(ref.frameId ? { frameId: ref.frameId } : {}),
|
||||
...(refTarget?.sessionId ? { cdpSessionId: refTarget.sessionId } : {}),
|
||||
},
|
||||
] as const;
|
||||
}),
|
||||
rendered.refs.map(
|
||||
(r) => [r.ref, { backendNodeId: r.backendNodeId, tabId: target.tabId }] as const,
|
||||
),
|
||||
);
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
text: observation.text,
|
||||
ref_count: observation.refs.length,
|
||||
text: rendered.text,
|
||||
ref_count: rendered.refs.length,
|
||||
tab_id: target.tabId,
|
||||
truncated: observation.truncated,
|
||||
truncated: rendered.truncated,
|
||||
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
|
||||
? {
|
||||
debug: {
|
||||
surface_probes: (observation.surfaceProbes ?? []).map((probe) => ({
|
||||
surface_probes: (rendered.surfaceProbes ?? []).map((probe) => ({
|
||||
trigger_backend_node_id: probe.triggerBackendNodeId,
|
||||
...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}),
|
||||
trigger_action: probe.triggerAction,
|
||||
@@ -1701,7 +1385,6 @@ async function handleVomObservation(
|
||||
: {}),
|
||||
});
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) return cancelled(toolName);
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
@@ -1713,16 +1396,14 @@ export async function handleSnapshot(
|
||||
manager: SessionManager,
|
||||
params: SnapshotParams,
|
||||
deps: SnapshotDeps = getDefaultDeps(),
|
||||
signal?: AbortSignal,
|
||||
): Promise<SnapshotResult | RpcError> {
|
||||
return handleVomObservation(manager, params, "snapshot", "passive_read", false, deps, signal);
|
||||
return handleVomObservation(manager, params, "snapshot", "passive_read", false, deps);
|
||||
}
|
||||
|
||||
export async function handleObserve(
|
||||
manager: SessionManager,
|
||||
params: ObserveParams,
|
||||
deps: SnapshotDeps = getDefaultDeps(),
|
||||
signal?: AbortSignal,
|
||||
): Promise<ObserveResult | RpcError> {
|
||||
return handleVomObservation(manager, params, "observe", "transient_input", true, deps, signal);
|
||||
return handleVomObservation(manager, params, "observe", "transient_input", true, deps);
|
||||
}
|
||||
|
||||
@@ -7,21 +7,53 @@ import {
|
||||
isRecordQueryMessage,
|
||||
isRecordStepMessage,
|
||||
RECORD_CANCEL,
|
||||
RECORD_CAPTURE_STATUS,
|
||||
RECORD_START,
|
||||
RECORD_STEP,
|
||||
RECORD_STOP,
|
||||
type RecordCancelMessage,
|
||||
type RecordCaptureStatusMessage,
|
||||
type RecordFinishMessage,
|
||||
type RecordQueryResponse,
|
||||
type RecordStartAck,
|
||||
type RecordStartMessage,
|
||||
type RecordStepPayload,
|
||||
type RecordStopMessage,
|
||||
} from "@/lib/record-bridge";
|
||||
import {
|
||||
armAllDocumentsForTab,
|
||||
armDocumentCapture,
|
||||
cancelDocumentCapture,
|
||||
type ArmedDocument,
|
||||
listInjectableFrames,
|
||||
type RecordFrameSendDeps,
|
||||
stopDocumentCapture,
|
||||
} from "@/lib/record-frame-arming";
|
||||
import { DEFAULT_MAX_PAGE_TOKENS } from "@/lib/record-constants";
|
||||
import {
|
||||
applyTargetMatching,
|
||||
buildTraceV3,
|
||||
cancelPendingSettles,
|
||||
captureAndRegisterObservation,
|
||||
clearPendingRedirectLanding,
|
||||
createObservationState,
|
||||
flushPendingRedirectLanding,
|
||||
flushPendingSettles,
|
||||
inferMissingPostStates,
|
||||
type RecordingObservationState,
|
||||
rememberStepOnPage,
|
||||
scheduleDraftSettle,
|
||||
scheduleRedirectLandingFlush,
|
||||
settleUnsettledDrafts,
|
||||
} from "@/lib/record-observation";
|
||||
import { appendRecordedPayload, observeRecordedNavigation } from "@/lib/recording-step-buffer";
|
||||
import { reduceTraceSteps, resolveTraceStartUrl } from "@/lib/trace-reducer";
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import { EXTENSION_VERSION } from "@/transport/handshake";
|
||||
import type {
|
||||
DraftTraceStep,
|
||||
FrameCaptureFailure,
|
||||
FrameCaptureInfo,
|
||||
FrameCaptureStatus,
|
||||
RecordAwaitParams,
|
||||
RecordAwaitResult,
|
||||
RecordStartParams,
|
||||
@@ -29,6 +61,7 @@ import type {
|
||||
RecordStopParams,
|
||||
RecordStopResult,
|
||||
RpcError,
|
||||
StopReason,
|
||||
Trace,
|
||||
} from "@/transport/types";
|
||||
import { handleNavigate } from "./navigation";
|
||||
@@ -58,6 +91,37 @@ interface ActiveRecording {
|
||||
currentUrl?: string;
|
||||
pendingNavigation: boolean;
|
||||
pendingNavigationDeadline?: number;
|
||||
observation: RecordingObservationState;
|
||||
stoppedBy: StopReason;
|
||||
maxPageTokens: number;
|
||||
redactValues: boolean;
|
||||
/** Documents whose capture acknowledged START and still need STOP. */
|
||||
armedDocuments: Map<string, ArmedDocument>;
|
||||
frameCaptureFailures: FrameCaptureFailure[];
|
||||
frameCaptureStatus: FrameCaptureStatus;
|
||||
/** When false, ignore subframe rearm requests during finish. */
|
||||
acceptingNewFrames: boolean;
|
||||
/** Rearms already sending START when finish begins. */
|
||||
rearmCallbacks: Set<Promise<void>>;
|
||||
/** Navigation callbacks tracked from event receipt through action enqueue. */
|
||||
navigationCallbacks: Set<Promise<void>>;
|
||||
/** Synchronous intake gate closed only after finish drains to stability. */
|
||||
acceptingNavigation: boolean;
|
||||
/**
|
||||
* Serializes step appends with navigation observation so a click is always
|
||||
* in `steps` before a same-turn `webNavigation` tries to annotate it.
|
||||
*/
|
||||
actionQueue: Promise<void>;
|
||||
/**
|
||||
* CDP runner captured at `record_start`. The message listeners are attached
|
||||
* once at service-worker startup, where no runner exists yet, so observation
|
||||
* capture must go through the recording instead of the listener deps.
|
||||
*/
|
||||
cdp?: CdpRunner;
|
||||
}
|
||||
|
||||
function enqueueRecordingAction(recording: ActiveRecording, task: () => Promise<void>): void {
|
||||
recording.actionQueue = recording.actionQueue.then(task, task).catch(() => {});
|
||||
}
|
||||
|
||||
const recordings = new Map<string, ActiveRecording>();
|
||||
@@ -78,6 +142,9 @@ function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/** Recording producer version mirrored into trace.recorder.bsk. */
|
||||
export const BSK_TRACE_VERSION = EXTENSION_VERSION;
|
||||
|
||||
/** Injectable http(s) landing page when `tool.record_start` omits `url`. */
|
||||
export const RECORD_DEFAULT_START_URL = "https://example.com/";
|
||||
|
||||
@@ -155,16 +222,95 @@ async function sendRecordStartWithAck(
|
||||
}
|
||||
|
||||
function buildTrace(recording: ActiveRecording): Trace {
|
||||
const { pages, steps } = reduceTraceSteps(recording.steps, recording.startUrl);
|
||||
const startUrl = resolveTraceStartUrl(recording.steps, recording.startUrl, pages);
|
||||
return {
|
||||
const frameCapture: FrameCaptureInfo | undefined =
|
||||
recording.frameCaptureFailures.length > 0 || recording.frameCaptureStatus === "partial"
|
||||
? {
|
||||
status: recording.frameCaptureStatus,
|
||||
...(recording.frameCaptureFailures.length > 0
|
||||
? { failures: recording.frameCaptureFailures }
|
||||
: {}),
|
||||
}
|
||||
: undefined;
|
||||
return buildTraceV3({
|
||||
obs: recording.observation,
|
||||
steps: recording.steps,
|
||||
startedAt: recording.startedAt,
|
||||
...(recording.purpose ? { purpose: recording.purpose } : {}),
|
||||
recorded_at: new Date().toISOString(),
|
||||
started_at: recording.startedAt,
|
||||
entry: { start_url: startUrl },
|
||||
pages,
|
||||
steps,
|
||||
startUrl: recording.startUrl,
|
||||
stoppedBy: recording.stoppedBy,
|
||||
bskVersion: BSK_TRACE_VERSION,
|
||||
...(frameCapture ? { frameCapture } : {}),
|
||||
});
|
||||
}
|
||||
|
||||
function recordFrameFailure(recording: ActiveRecording, failure: FrameCaptureFailure): void {
|
||||
recording.frameCaptureFailures.push(failure);
|
||||
recording.frameCaptureStatus = "partial";
|
||||
}
|
||||
|
||||
function enrichRecordedStep(
|
||||
recording: ActiveRecording,
|
||||
step: RecordStepPayload,
|
||||
): RecordStepPayload {
|
||||
const enriched: RecordStepPayload = { ...step };
|
||||
if (enriched.geometry && enriched.documentToken) {
|
||||
const owner = recording.observation.documentTokenOwners.get(enriched.documentToken);
|
||||
if (owner !== undefined) {
|
||||
enriched.geometry = { ...enriched.geometry, ownerFrameBackendNodeId: owner };
|
||||
}
|
||||
}
|
||||
if (!enriched.page_url && recording.currentUrl) {
|
||||
enriched.page_url = recording.currentUrl;
|
||||
}
|
||||
return enriched;
|
||||
}
|
||||
|
||||
async function publishCaptureStatus(
|
||||
recording: ActiveRecording,
|
||||
deps: RecordDeps,
|
||||
): Promise<void> {
|
||||
if (!deps.broadcastCaptureStatus) return;
|
||||
const msg: RecordCaptureStatusMessage = {
|
||||
type: RECORD_CAPTURE_STATUS,
|
||||
status: recording.frameCaptureStatus,
|
||||
...(recording.frameCaptureFailures.length > 0
|
||||
? { failures: recording.frameCaptureFailures }
|
||||
: {}),
|
||||
};
|
||||
try {
|
||||
await deps.broadcastCaptureStatus(recording.tabId, msg);
|
||||
} catch {
|
||||
// Overlay is best-effort.
|
||||
}
|
||||
}
|
||||
|
||||
async function processRecordedStep(
|
||||
recording: ActiveRecording,
|
||||
deps: RecordDeps,
|
||||
draftIndex: number,
|
||||
): Promise<void> {
|
||||
const cdp = recording.cdp ?? deps.cdp;
|
||||
if (!cdp) return;
|
||||
const draft = recording.steps[draftIndex];
|
||||
if (!draft) return;
|
||||
const stepId = draftIndex + 1;
|
||||
|
||||
if (draft.op === "navigate") {
|
||||
draft.preStateId = recording.observation.lastSettled?.stateId;
|
||||
} else {
|
||||
applyTargetMatching(recording.observation, draft, stepId);
|
||||
}
|
||||
rememberStepOnPage(recording.observation, draft, stepId);
|
||||
|
||||
scheduleDraftSettle(
|
||||
recording.observation,
|
||||
draftIndex,
|
||||
stepId,
|
||||
cdp,
|
||||
recording.tabId,
|
||||
deps.tabsApi,
|
||||
recording.steps,
|
||||
);
|
||||
}
|
||||
|
||||
export interface RecordDeps {
|
||||
@@ -173,17 +319,67 @@ export interface RecordDeps {
|
||||
tabId: number,
|
||||
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
|
||||
): Promise<unknown>;
|
||||
sendToDocument?(
|
||||
tabId: number,
|
||||
documentId: string,
|
||||
msg: RecordStartMessage | RecordStopMessage | RecordCancelMessage,
|
||||
): Promise<unknown>;
|
||||
getAllFrames?(tabId: number): Promise<
|
||||
Array<{
|
||||
frameId: number;
|
||||
documentId?: string;
|
||||
url?: string;
|
||||
parentFrameId: number;
|
||||
}>
|
||||
>;
|
||||
broadcastCaptureStatus?(
|
||||
tabId: number,
|
||||
msg: RecordCaptureStatusMessage,
|
||||
): Promise<void>;
|
||||
bypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
cdp?: CdpRunner;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
function frameSendDeps(deps: RecordDeps): RecordFrameSendDeps {
|
||||
return {
|
||||
sendToDocument: (tabId, documentId, msg) => {
|
||||
if (deps.sendToDocument) return deps.sendToDocument(tabId, documentId, msg);
|
||||
return deps.sendToTab(tabId, msg);
|
||||
},
|
||||
getAllFrames: async (tabId) => {
|
||||
if (deps.getAllFrames) return deps.getAllFrames(tabId);
|
||||
if (!chrome.webNavigation?.getAllFrames) return [];
|
||||
const frames = await chrome.webNavigation.getAllFrames({ tabId });
|
||||
return (frames ?? []).map((frame) => ({
|
||||
frameId: frame.frameId,
|
||||
documentId: frame.documentId,
|
||||
url: frame.url,
|
||||
parentFrameId: frame.parentFrameId,
|
||||
}));
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
let defaultDeps: RecordDeps | null = null;
|
||||
function getDefaultDeps(): RecordDeps {
|
||||
if (!defaultDeps) {
|
||||
defaultDeps = {
|
||||
tabsApi: chromeTabsApi,
|
||||
sendToTab: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg),
|
||||
sendToDocument: (tabId, documentId, msg) =>
|
||||
chrome.tabs.sendMessage(tabId, msg, { documentId }),
|
||||
getAllFrames: async (tabId) => {
|
||||
if (!chrome.webNavigation?.getAllFrames) return [];
|
||||
const frames = await chrome.webNavigation.getAllFrames({ tabId });
|
||||
return (frames ?? []).map((frame) => ({
|
||||
frameId: frame.frameId,
|
||||
documentId: frame.documentId,
|
||||
url: frame.url,
|
||||
parentFrameId: frame.parentFrameId,
|
||||
}));
|
||||
},
|
||||
broadcastCaptureStatus: (tabId, msg) => chrome.tabs.sendMessage(tabId, msg, { frameId: 0 }),
|
||||
};
|
||||
}
|
||||
return defaultDeps;
|
||||
@@ -241,16 +437,29 @@ export function releaseBrowserObservationListenersIfIdle(): void {
|
||||
detachBrowserObservation = null;
|
||||
}
|
||||
|
||||
export function attachRecordStepListener(): () => void {
|
||||
export function attachRecordStepListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||
const listener = (
|
||||
message: unknown,
|
||||
_sender: chrome.runtime.MessageSender,
|
||||
sender: chrome.runtime.MessageSender,
|
||||
_sendResponse: () => void,
|
||||
) => {
|
||||
if (!isRecordStepMessage(message)) return false;
|
||||
for (const recording of recordings.values()) {
|
||||
if (recording.requestId !== message.requestId) continue;
|
||||
appendRecordedPayload(recording, message.step);
|
||||
enqueueRecordingAction(recording, async () => {
|
||||
const cdp = recording.cdp ?? deps.cdp;
|
||||
await flushPendingRedirectLanding(
|
||||
recording.observation,
|
||||
recording.steps,
|
||||
cdp,
|
||||
recording.tabId,
|
||||
deps.tabsApi,
|
||||
{ cancelled: () => recording.settled },
|
||||
);
|
||||
const enriched = enrichRecordedStep(recording, message.step);
|
||||
const draftIndex = appendRecordedPayload(recording, enriched);
|
||||
if (draftIndex !== null) await processRecordedStep(recording, deps, draftIndex);
|
||||
});
|
||||
return false;
|
||||
}
|
||||
return false;
|
||||
@@ -277,7 +486,11 @@ export function attachRecordFinishListener(deps: RecordDeps = getDefaultDeps()):
|
||||
|
||||
function findRecordingByTabId(tabId: number): ActiveRecording | null {
|
||||
for (const recording of recordings.values()) {
|
||||
if (recording.tabId === tabId && !recording.settled) return recording;
|
||||
if (recording.settled) continue;
|
||||
if (recording.tabId === tabId) return recording;
|
||||
for (const armed of recording.armedDocuments.values()) {
|
||||
if (armed.tabId === tabId) return recording;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -325,71 +538,135 @@ async function clearRearmTimersForRecording(
|
||||
}
|
||||
}
|
||||
|
||||
async function stopRecordingOnAllAgentTabs(
|
||||
async function stopRecordingOnAllDocuments(
|
||||
recording: ActiveRecording,
|
||||
deps: RecordDeps,
|
||||
): Promise<void> {
|
||||
recording.acceptingNewFrames = false;
|
||||
const stopMsg: RecordStopMessage = { type: RECORD_STOP, requestId: recording.requestId };
|
||||
let tabIds = [recording.tabId];
|
||||
try {
|
||||
const tabs = await deps.tabsApi.query({ windowId: recording.agentWindowId });
|
||||
tabIds = [
|
||||
...new Set([
|
||||
recording.tabId,
|
||||
...tabs.flatMap((tab) => (typeof tab.id === "number" ? [tab.id] : [])),
|
||||
]),
|
||||
];
|
||||
} catch {
|
||||
// Fall back to the current recording tab.
|
||||
}
|
||||
|
||||
for (const tabId of tabIds) {
|
||||
try {
|
||||
const response = await deps.sendToTab(tabId, stopMsg);
|
||||
if (tabId === recording.tabId && !isRecordStartAck(response)) {
|
||||
throw new Error("content script did not confirm recorded steps");
|
||||
}
|
||||
} catch {
|
||||
if (tabId === recording.tabId) {
|
||||
throw new Error("failed to flush recorded steps");
|
||||
}
|
||||
const frameDeps = frameSendDeps(deps);
|
||||
const armed = [...recording.armedDocuments.values()];
|
||||
for (const doc of armed) {
|
||||
const result = await stopDocumentCapture(doc, stopMsg, frameDeps);
|
||||
if ("ok" in result) {
|
||||
recording.armedDocuments.delete(doc.documentId);
|
||||
continue;
|
||||
}
|
||||
if (deps.bypassOverlay) {
|
||||
try {
|
||||
await deps.bypassOverlay(tabId, false);
|
||||
} catch {
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
recordFrameFailure(recording, result);
|
||||
}
|
||||
if (recording.armedDocuments.size > 0) {
|
||||
for (const doc of [...recording.armedDocuments.values()]) {
|
||||
recordFrameFailure(
|
||||
recording,
|
||||
{
|
||||
reason: "flush_failed",
|
||||
frame_id: doc.frameId,
|
||||
document_id: doc.documentId,
|
||||
url: doc.url,
|
||||
detail: "document still armed after STOP",
|
||||
},
|
||||
);
|
||||
}
|
||||
recording.armedDocuments.clear();
|
||||
}
|
||||
await publishCaptureStatus(recording, deps);
|
||||
if (deps.bypassOverlay) {
|
||||
try {
|
||||
await deps.bypassOverlay(recording.tabId, false);
|
||||
} catch {
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function rearmRecording(
|
||||
async function cancelRecordingOnAllDocuments(
|
||||
recording: ActiveRecording,
|
||||
deps: RecordDeps,
|
||||
): Promise<void> {
|
||||
const cancelMsg: RecordCancelMessage = { type: RECORD_CANCEL, requestId: recording.requestId };
|
||||
const frameDeps = frameSendDeps(deps);
|
||||
for (const doc of [...recording.armedDocuments.values()]) {
|
||||
await cancelDocumentCapture(doc, cancelMsg, frameDeps);
|
||||
}
|
||||
recording.armedDocuments.clear();
|
||||
}
|
||||
|
||||
async function rearmRecordingDocuments(
|
||||
recording: ActiveRecording,
|
||||
targetTabId: number,
|
||||
deps: RecordDeps,
|
||||
targetDocumentId?: string,
|
||||
): Promise<boolean> {
|
||||
// Do NOT toggle automation-bypass here: each retry used to increment the
|
||||
// content-script counter, and a single stop decrement left the ControlOverlay
|
||||
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
|
||||
// already hides the control chrome while activeRecord is set.
|
||||
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
|
||||
const startMsg: RecordStartMessage = {
|
||||
type: RECORD_START,
|
||||
requestId: recording.requestId,
|
||||
startedAtMs: recording.startedAtMs,
|
||||
};
|
||||
try {
|
||||
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
|
||||
recording.tabId = targetTabId;
|
||||
return true;
|
||||
} catch {
|
||||
let resolveTracked!: () => void;
|
||||
const tracked = new Promise<void>((resolve) => {
|
||||
resolveTracked = resolve;
|
||||
});
|
||||
recording.rearmCallbacks.add(tracked);
|
||||
|
||||
const frameDeps = frameSendDeps(deps);
|
||||
try {
|
||||
const attemptLimit = targetDocumentId ? 1 : RECORD_REARM_MAX_ATTEMPTS;
|
||||
for (let attempt = 0; attempt < attemptLimit; attempt += 1) {
|
||||
if (recording.settled || recording.finishing || !recording.acceptingNewFrames) return false;
|
||||
const { injectable, failures } = await listInjectableFrames(targetTabId, frameDeps);
|
||||
for (const failure of failures) recordFrameFailure(recording, failure);
|
||||
const targets = targetDocumentId
|
||||
? injectable.filter((frame) => frame.documentId === targetDocumentId)
|
||||
: injectable.filter((frame) => !recording.armedDocuments.has(frame.documentId ?? ""));
|
||||
if (targets.length === 0) {
|
||||
if (targetDocumentId) {
|
||||
recordFrameFailure(recording, {
|
||||
reason: "rearm_failed",
|
||||
frame_id: 0,
|
||||
document_id: targetDocumentId,
|
||||
detail: "document not injectable",
|
||||
});
|
||||
await publishCaptureStatus(recording, deps);
|
||||
return false;
|
||||
}
|
||||
recording.tabId = targetTabId;
|
||||
return injectable.some((frame) => recording.armedDocuments.has(frame.documentId ?? ""));
|
||||
}
|
||||
|
||||
let armedAny = false;
|
||||
for (const frame of targets) {
|
||||
const startMsg: RecordStartMessage = {
|
||||
type: RECORD_START,
|
||||
requestId: recording.requestId,
|
||||
startedAtMs: recording.startedAtMs,
|
||||
showOverlay: frame.frameId === 0,
|
||||
frameMode: frame.frameId === 0 ? "top" : "child",
|
||||
};
|
||||
const result = await armDocumentCapture(targetTabId, frame, startMsg, frameDeps);
|
||||
if (result.armed) {
|
||||
recording.armedDocuments.set(result.armed.documentId, result.armed);
|
||||
armedAny = true;
|
||||
}
|
||||
if (result.failure) recordFrameFailure(recording, result.failure);
|
||||
}
|
||||
if (armedAny) {
|
||||
recording.tabId = targetTabId;
|
||||
await publishCaptureStatus(recording, deps);
|
||||
if (deps.cdp && recording.acceptingNewFrames) {
|
||||
void captureAndRegisterObservation(
|
||||
recording.observation,
|
||||
deps.cdp,
|
||||
targetTabId,
|
||||
deps.tabsApi,
|
||||
).catch(() => {});
|
||||
}
|
||||
return true;
|
||||
}
|
||||
if (attempt + 1 < RECORD_REARM_MAX_ATTEMPTS) {
|
||||
await sleep(RECORD_REARM_RETRY_DELAY_MS);
|
||||
}
|
||||
}
|
||||
await publishCaptureStatus(recording, deps);
|
||||
return false;
|
||||
} finally {
|
||||
recording.rearmCallbacks.delete(tracked);
|
||||
resolveTracked();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
|
||||
@@ -401,7 +678,7 @@ function scheduleRearmForTab(tabId: number, deps: RecordDeps): void {
|
||||
rearmTimers.delete(tabId);
|
||||
void (async () => {
|
||||
const current = await findRecordingForTab(tabId, deps);
|
||||
if (current) await rearmRecording(current, tabId, deps);
|
||||
if (current) await rearmRecordingDocuments(current, tabId, deps);
|
||||
})();
|
||||
}, RECORD_REARM_DEBOUNCE_MS),
|
||||
);
|
||||
@@ -432,11 +709,86 @@ export function attachRecordTabListener(deps: RecordDeps = getDefaultDeps()): ()
|
||||
}
|
||||
|
||||
export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||
const observeMainFrame = (tabId: number, url?: string, causedByAction?: boolean) => {
|
||||
const observeMainFrame = (
|
||||
tabId: number,
|
||||
url?: string,
|
||||
causedByAction?: boolean,
|
||||
transitionType?: string,
|
||||
transitionQualifiers?: string[],
|
||||
) => {
|
||||
if (!url) return;
|
||||
const direct = findRecordingByTabId(tabId);
|
||||
const candidates = direct
|
||||
? direct.acceptingNavigation
|
||||
? [direct]
|
||||
: []
|
||||
: [...recordings.values()].filter(
|
||||
(recording) => !recording.settled && recording.acceptingNavigation,
|
||||
);
|
||||
if (candidates.length === 0) return;
|
||||
|
||||
let resolveTracked!: () => void;
|
||||
const tracked = new Promise<void>((resolve) => {
|
||||
resolveTracked = resolve;
|
||||
});
|
||||
for (const candidate of candidates) candidate.navigationCallbacks.add(tracked);
|
||||
|
||||
void (async () => {
|
||||
const recording = await findRecordingForTab(tabId, deps);
|
||||
if (recording && url) {
|
||||
observeRecordedNavigation(recording, url, causedByAction);
|
||||
try {
|
||||
const recording = await findRecordingForTab(tabId, deps);
|
||||
if (!recording || !recording.acceptingNavigation || !candidates.includes(recording)) {
|
||||
return;
|
||||
}
|
||||
enqueueRecordingAction(recording, async () => {
|
||||
const result = observeRecordedNavigation(
|
||||
recording,
|
||||
url,
|
||||
causedByAction,
|
||||
transitionType,
|
||||
transitionQualifiers,
|
||||
);
|
||||
const cdp = recording.cdp ?? deps.cdp;
|
||||
if (!cdp) return;
|
||||
|
||||
if (result.kind === "coalesce_redirect") {
|
||||
scheduleRedirectLandingFlush(
|
||||
recording.observation,
|
||||
recording.steps,
|
||||
cdp,
|
||||
tabId,
|
||||
deps.tabsApi,
|
||||
result.url,
|
||||
{ cancelled: () => recording.settled },
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (result.kind === "noop") return;
|
||||
|
||||
// A concrete navigation supersedes any in-flight redirect coalesce.
|
||||
clearPendingRedirectLanding(recording.observation);
|
||||
|
||||
const draftIndex = result.index;
|
||||
const lastDraft = recording.steps[draftIndex];
|
||||
if (!lastDraft) return;
|
||||
|
||||
if (result.kind === "appended") {
|
||||
lastDraft.preStateId = recording.observation.lastSettled?.stateId;
|
||||
rememberStepOnPage(recording.observation, lastDraft, draftIndex + 1);
|
||||
}
|
||||
scheduleDraftSettle(
|
||||
recording.observation,
|
||||
draftIndex,
|
||||
draftIndex + 1,
|
||||
cdp,
|
||||
tabId,
|
||||
deps.tabsApi,
|
||||
recording.steps,
|
||||
);
|
||||
});
|
||||
} finally {
|
||||
for (const candidate of candidates) candidate.navigationCallbacks.delete(tracked);
|
||||
resolveTracked();
|
||||
}
|
||||
})();
|
||||
};
|
||||
@@ -447,29 +799,36 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
|
||||
})();
|
||||
};
|
||||
|
||||
const onSubFrameComplete = (tabId: number, documentId?: string) => {
|
||||
if (!documentId) return;
|
||||
void (async () => {
|
||||
const recording = await findRecordingForTab(tabId, deps);
|
||||
if (!recording || !recording.acceptingNewFrames) return;
|
||||
await rearmRecordingDocuments(recording, tabId, deps, documentId);
|
||||
})();
|
||||
};
|
||||
|
||||
if (chrome.webNavigation?.onCompleted) {
|
||||
const completedListener = (
|
||||
details: chrome.webNavigation.WebNavigationFramedCallbackDetails,
|
||||
) => {
|
||||
if (details.frameId !== 0) return;
|
||||
onMainFrameComplete(details.tabId, details.url);
|
||||
if (details.frameId === 0) {
|
||||
onMainFrameComplete(details.tabId, details.url);
|
||||
return;
|
||||
}
|
||||
onSubFrameComplete(details.tabId, details.documentId);
|
||||
};
|
||||
const committedListener = (
|
||||
details: chrome.webNavigation.WebNavigationTransitionCallbackDetails,
|
||||
) => {
|
||||
if (details.frameId !== 0) return;
|
||||
const causedByAction =
|
||||
details.transitionType === "link" || details.transitionType === "form_submit"
|
||||
? true
|
||||
: details.transitionType === "typed" ||
|
||||
details.transitionType === "auto_bookmark" ||
|
||||
details.transitionType === "generated" ||
|
||||
details.transitionType === "keyword" ||
|
||||
details.transitionType === "keyword_generated" ||
|
||||
details.transitionType === "reload"
|
||||
? false
|
||||
: undefined;
|
||||
observeMainFrame(details.tabId, details.url, causedByAction);
|
||||
observeMainFrame(
|
||||
details.tabId,
|
||||
details.url,
|
||||
undefined,
|
||||
details.transitionType,
|
||||
details.transitionQualifiers,
|
||||
);
|
||||
};
|
||||
chrome.webNavigation.onCompleted.addListener(completedListener);
|
||||
chrome.webNavigation.onCommitted?.addListener(committedListener);
|
||||
@@ -487,6 +846,46 @@ export function attachRecordNavigationListener(deps: RecordDeps = getDefaultDeps
|
||||
return () => chrome.tabs.onUpdated.removeListener(listener);
|
||||
}
|
||||
|
||||
const MAX_FINISH_DRAIN_ROUNDS = 10;
|
||||
|
||||
async function drainRecordingToStability(
|
||||
recording: ActiveRecording,
|
||||
deps: RecordDeps,
|
||||
): Promise<boolean> {
|
||||
const cdp = recording.cdp ?? deps.cdp;
|
||||
for (let round = 0; round < MAX_FINISH_DRAIN_ROUNDS; round += 1) {
|
||||
await Promise.all([...recording.navigationCallbacks]);
|
||||
const actionTail = recording.actionQueue;
|
||||
await actionTail;
|
||||
await flushPendingRedirectLanding(
|
||||
recording.observation,
|
||||
recording.steps,
|
||||
cdp,
|
||||
recording.tabId,
|
||||
deps.tabsApi,
|
||||
{ cancelled: () => recording.settled },
|
||||
);
|
||||
await flushPendingSettles(recording.observation);
|
||||
const redirectTail = recording.observation.redirectFlushQueue;
|
||||
const settleTail = recording.observation.settleQueue;
|
||||
await Promise.all([redirectTail, settleTail]);
|
||||
|
||||
if (
|
||||
recording.navigationCallbacks.size === 0 &&
|
||||
recording.actionQueue === actionTail &&
|
||||
recording.observation.redirectFlushQueue === redirectTail &&
|
||||
recording.observation.settleQueue === settleTail
|
||||
) {
|
||||
// No event or promise continuation can interleave with this synchronous
|
||||
// check-and-close, so work accepted before the cutoff is fully drained.
|
||||
recording.acceptingNavigation = false;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
console.warn("[bsk record] navigation/action queues did not stabilize at stop");
|
||||
return false;
|
||||
}
|
||||
|
||||
export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()): () => void {
|
||||
const listener = (
|
||||
message: unknown,
|
||||
@@ -505,11 +904,15 @@ export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()):
|
||||
sendResponse({ active: false });
|
||||
return;
|
||||
}
|
||||
await rearmRecording(recording, tabId, deps);
|
||||
await rearmRecordingDocuments(recording, tabId, deps, sender.documentId);
|
||||
sendResponse({
|
||||
active: true,
|
||||
requestId: recording.requestId,
|
||||
startedAtMs: recording.startedAtMs,
|
||||
captureStatus: recording.frameCaptureStatus,
|
||||
...(recording.frameCaptureFailures.length > 0
|
||||
? { captureFailures: recording.frameCaptureFailures }
|
||||
: {}),
|
||||
});
|
||||
})();
|
||||
return true;
|
||||
@@ -527,23 +930,46 @@ async function finishRecordingByRequest(
|
||||
if (recording.requestId !== requestId || recording.settled) continue;
|
||||
const match = await findRecordingForTab(tabId, deps);
|
||||
if (match !== recording) continue;
|
||||
await finishRecording(sessionId, deps);
|
||||
await finishRecording(sessionId, deps, "user_finish");
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
async function finishRecording(sessionId: string, deps: RecordDeps): Promise<Trace | null> {
|
||||
async function finishRecording(
|
||||
sessionId: string,
|
||||
deps: RecordDeps,
|
||||
stoppedBy: StopReason,
|
||||
): Promise<Trace | null> {
|
||||
const recording = recordings.get(sessionId);
|
||||
if (!recording || recording.settled || recording.finishing) return null;
|
||||
if (!recording || recording.settled) return null;
|
||||
if (recording.finishing) return recording.finishPromise;
|
||||
recording.finishing = true;
|
||||
recording.stoppedBy = stoppedBy;
|
||||
|
||||
await clearRearmTimersForRecording(recording, deps);
|
||||
await Promise.all([...recording.rearmCallbacks]);
|
||||
try {
|
||||
await stopRecordingOnAllAgentTabs(recording, deps);
|
||||
await stopRecordingOnAllDocuments(recording, deps);
|
||||
} catch {
|
||||
recording.finishing = false;
|
||||
return null;
|
||||
}
|
||||
if (!(await drainRecordingToStability(recording, deps))) {
|
||||
recording.finishing = false;
|
||||
return null;
|
||||
}
|
||||
cancelPendingSettles(recording.observation);
|
||||
inferMissingPostStates(recording.steps);
|
||||
const cdp = recording.cdp ?? deps.cdp;
|
||||
if (cdp) {
|
||||
await settleUnsettledDrafts(
|
||||
recording.observation,
|
||||
cdp,
|
||||
recording.tabId,
|
||||
deps.tabsApi,
|
||||
recording.steps,
|
||||
);
|
||||
}
|
||||
|
||||
recording.settled = true;
|
||||
recordings.delete(sessionId);
|
||||
@@ -582,6 +1008,8 @@ export async function handleRecordStart(
|
||||
});
|
||||
const navigateUrl = params.url ?? RECORD_DEFAULT_START_URL;
|
||||
const startedAtMs = Date.now();
|
||||
const maxPageTokens = params.max_page_tokens;
|
||||
const redactValues = params.redact_values ?? false;
|
||||
recordings.set(params.session_id, {
|
||||
requestId,
|
||||
tabId: target.tabId,
|
||||
@@ -599,23 +1027,30 @@ export async function handleRecordStart(
|
||||
currentUrl: navigateUrl,
|
||||
pendingNavigation: false,
|
||||
pendingNavigationDeadline: undefined,
|
||||
observation: createObservationState({ maxPageTokens, redactValues }),
|
||||
stoppedBy: "user_finish",
|
||||
maxPageTokens: maxPageTokens ?? DEFAULT_MAX_PAGE_TOKENS,
|
||||
redactValues,
|
||||
armedDocuments: new Map(),
|
||||
frameCaptureFailures: [],
|
||||
frameCaptureStatus: "complete",
|
||||
acceptingNewFrames: true,
|
||||
rearmCallbacks: new Set(),
|
||||
navigationCallbacks: new Set(),
|
||||
acceptingNavigation: true,
|
||||
actionQueue: Promise.resolve(),
|
||||
...(deps.cdp ? { cdp: deps.cdp } : {}),
|
||||
});
|
||||
// Observe navigations for the whole recording lifetime; attach before
|
||||
// optional navigate so the destination load can rearm capture.
|
||||
ensureBrowserObservationListeners(deps);
|
||||
|
||||
const abortPending = async (notifyContent: boolean) => {
|
||||
const pending = recordings.get(params.session_id);
|
||||
recordings.delete(params.session_id);
|
||||
releaseBrowserObservationListenersIfIdle();
|
||||
if (notifyContent) {
|
||||
try {
|
||||
await deps.sendToTab(target.tabId, {
|
||||
type: RECORD_CANCEL,
|
||||
requestId,
|
||||
});
|
||||
} catch {
|
||||
// Content may never have received RECORD_START.
|
||||
}
|
||||
if (notifyContent && pending) {
|
||||
await cancelRecordingOnAllDocuments(pending, deps);
|
||||
}
|
||||
if (deps.bypassOverlay) {
|
||||
try {
|
||||
@@ -718,10 +1153,28 @@ export async function handleRecordStart(
|
||||
if (cancelled) return cancelled;
|
||||
}
|
||||
|
||||
const startMsg: RecordStartMessage = { type: RECORD_START, requestId, startedAtMs };
|
||||
const startMsg: RecordStartMessage = {
|
||||
type: RECORD_START,
|
||||
requestId,
|
||||
startedAtMs,
|
||||
showOverlay: true,
|
||||
frameMode: "top",
|
||||
};
|
||||
|
||||
try {
|
||||
await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
|
||||
const frameDeps = frameSendDeps(deps);
|
||||
const { armed, failures } = await armAllDocumentsForTab(target.tabId, startMsg, frameDeps);
|
||||
for (const failure of failures) recordFrameFailure(active, failure);
|
||||
for (const doc of armed) active.armedDocuments.set(doc.documentId, doc);
|
||||
if (active.armedDocuments.size === 0) {
|
||||
await abortPending(true);
|
||||
return {
|
||||
code: "protocol_error",
|
||||
message:
|
||||
"failed to start recording in any frame — reload the BrowserSkill extension, then retry",
|
||||
};
|
||||
}
|
||||
await publishCaptureStatus(active, deps);
|
||||
} catch {
|
||||
await abortPending(true);
|
||||
return {
|
||||
@@ -731,6 +1184,23 @@ export async function handleRecordStart(
|
||||
};
|
||||
}
|
||||
|
||||
if (deps.cdp) {
|
||||
const activeRecording = recordings.get(params.session_id);
|
||||
if (activeRecording) {
|
||||
try {
|
||||
await captureAndRegisterObservation(
|
||||
activeRecording.observation,
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
deps.tabsApi,
|
||||
startUrl,
|
||||
);
|
||||
} catch {
|
||||
// Proceed without initial observation; steps may be dropped by reducer.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
const cancelled = await abortIfCancelled(true);
|
||||
if (cancelled) return cancelled;
|
||||
@@ -755,7 +1225,7 @@ export async function handleRecordStop(
|
||||
};
|
||||
}
|
||||
|
||||
const trace = await finishRecording(params.session_id, deps);
|
||||
const trace = await finishRecording(params.session_id, deps, "cli_stop");
|
||||
if (!trace) {
|
||||
return {
|
||||
code: "protocol_error",
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import { type SessionManager, SessionStartCleanupError } from "@/session-manager/manager";
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { rpcError } from "./errors";
|
||||
import { clearRecordingForSession } from "./record";
|
||||
import { isRpcError } from "./shared";
|
||||
import { returnBorrowedTab, type TabManagementDeps } from "./tabs";
|
||||
@@ -59,10 +58,6 @@ export interface SessionStartResult {
|
||||
agent_window_id?: number;
|
||||
}
|
||||
|
||||
export interface SessionStartDeps {
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface SessionStopParams {
|
||||
session_id: string;
|
||||
}
|
||||
@@ -76,7 +71,6 @@ export interface SessionStopResult {
|
||||
}
|
||||
|
||||
export interface SessionStopDeps {
|
||||
signal?: AbortSignal;
|
||||
cdp?: {
|
||||
detachSession(sessionId: string): Promise<void>;
|
||||
};
|
||||
@@ -97,7 +91,6 @@ export interface SessionStopDeps {
|
||||
export async function handleSessionStart(
|
||||
manager: SessionManager,
|
||||
params: SessionStartParams,
|
||||
deps: SessionStartDeps = {},
|
||||
): Promise<SessionStartResult | RpcError> {
|
||||
if (!params?.session_id) {
|
||||
return {
|
||||
@@ -111,23 +104,9 @@ export async function handleSessionStart(
|
||||
const ctx = await manager.start(params.session_id, {
|
||||
size: sizeOrErr,
|
||||
focused: params.focused,
|
||||
signal: deps.signal,
|
||||
});
|
||||
return { agent_window_id: ctx.agentWindowId };
|
||||
} catch (err) {
|
||||
if (err instanceof SessionStartCleanupError) {
|
||||
return rpcError("protocol_error", "cleanup_failed", err.message, {
|
||||
resource_type: "agent_window",
|
||||
resource_id: err.windowId,
|
||||
});
|
||||
}
|
||||
if (
|
||||
typeof err === "object" &&
|
||||
err !== null &&
|
||||
(err as { name?: string }).name === "AbortError"
|
||||
) {
|
||||
return { code: "cancelled", message: "session_start aborted" };
|
||||
}
|
||||
// chrome.windows.create / SessionManager failures are not CDP
|
||||
// failures (§4.5 reserves cdp_failed for raw CDP errors). Surface
|
||||
// them as protocol_error so the CLI maps to the right exit code
|
||||
@@ -178,9 +157,6 @@ export async function handleSessionStop(
|
||||
message: `session ${params.session_id} unknown`,
|
||||
};
|
||||
}
|
||||
if (deps.signal?.aborted) {
|
||||
return { code: "cancelled", message: "session_stop aborted before teardown" };
|
||||
}
|
||||
|
||||
// Step 1: auto-return borrowed tabs. Iterate over a snapshot of the
|
||||
// ids so deletions during iteration do not break the Map iterator.
|
||||
@@ -191,7 +167,6 @@ export async function handleSessionStop(
|
||||
try {
|
||||
const tabManagement = {
|
||||
...(deps.tabManagement ?? {}),
|
||||
signal: deps.signal,
|
||||
isAgentWindowId:
|
||||
deps.tabManagement?.isAgentWindowId ??
|
||||
((windowId: number) => manager.findByWindowId(windowId) !== null),
|
||||
@@ -219,17 +194,6 @@ export async function handleSessionStop(
|
||||
}
|
||||
}
|
||||
|
||||
if (deps.signal?.aborted) {
|
||||
const nonCancelFailures = returnFailures?.filter((failure) => failure.code !== "cancelled");
|
||||
if (nonCancelFailures && nonCancelFailures.length > 0) {
|
||||
return {
|
||||
...(returnedTabIds.length > 0 ? { returned_tab_ids: returnedTabIds } : {}),
|
||||
return_failures: nonCancelFailures,
|
||||
};
|
||||
}
|
||||
return { code: "cancelled", message: "session_stop aborted during tab return" };
|
||||
}
|
||||
|
||||
const result: SessionStopResult = {};
|
||||
if (returnedTabIds.length > 0) result.returned_tab_ids = returnedTabIds;
|
||||
if (returnFailures && returnFailures.length > 0) {
|
||||
@@ -249,10 +213,6 @@ export async function handleSessionStop(
|
||||
// Step 3: detach CDP sessions this session opened (no-op if none).
|
||||
await deps.cdp?.detachSession(params.session_id);
|
||||
|
||||
if (deps.signal?.aborted) {
|
||||
return { code: "cancelled", message: "session_stop aborted before window close" };
|
||||
}
|
||||
|
||||
// Step 4: close the Agent Window and drop the context.
|
||||
await manager.stop(params.session_id);
|
||||
|
||||
|
||||
@@ -6,8 +6,7 @@
|
||||
// exactly the same sandbox + visibility rules as the M6 observation
|
||||
// handlers (review parity).
|
||||
|
||||
import type { CdpDebuggee, DialogCursor } from "@/browser-driver/chromium-cdp";
|
||||
import type { CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { DialogCursor } from "@/browser-driver/chromium-cdp";
|
||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||
import { normaliseRef } from "@/session-manager/ref-store";
|
||||
import type { ConsoleResult, JavaScriptDialogInfo, RpcError } from "@/transport/types";
|
||||
@@ -52,11 +51,9 @@ export type { DialogCursor };
|
||||
*/
|
||||
export interface CdpRunner {
|
||||
send<T = unknown>(tabId: number, method: string, params?: object): Promise<T>;
|
||||
sendToTarget?<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T>;
|
||||
getFrameGraph?(tabId: number): Promise<CdpFrameGraph>;
|
||||
ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>;
|
||||
trackSessionTab?(sessionId: string, tabId: number): void;
|
||||
onEvent?(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
|
||||
onEvent?(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): {
|
||||
dispose(): void;
|
||||
};
|
||||
dialogCursor?(tabId: number): DialogCursor;
|
||||
@@ -71,25 +68,6 @@ export interface CdpRunner {
|
||||
): ConsoleResult;
|
||||
}
|
||||
|
||||
export function sendToCdpTarget<T = unknown>(
|
||||
cdp: CdpRunner,
|
||||
target: CdpTarget,
|
||||
method: string,
|
||||
params?: object,
|
||||
): Promise<T> {
|
||||
if (target.sessionId && cdp.sendToTarget) {
|
||||
return cdp.sendToTarget<T>(target, method, params);
|
||||
}
|
||||
return cdp.send<T>(target.tabId, method, params);
|
||||
}
|
||||
|
||||
export function cdpRunnerForTarget(cdp: CdpRunner, target: CdpTarget): CdpRunner {
|
||||
return {
|
||||
send: <T = unknown>(_tabId: number, method: string, params?: object) =>
|
||||
sendToCdpTarget<T>(cdp, target, method, params),
|
||||
};
|
||||
}
|
||||
|
||||
export interface BufferedReadBounds {
|
||||
since: number | undefined;
|
||||
limit: number;
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Snapshot ref resolution — normalise `@eN` / `eN`, look up
|
||||
// compound CDP node identity via the session RefStore, and build stable
|
||||
// `backendNodeId` via the session RefStore, and build stable
|
||||
// `ref_not_found` errors for hard-failure tool paths.
|
||||
|
||||
import type { SessionContext } from "@/session-manager/manager";
|
||||
@@ -10,13 +10,6 @@ import { rpcError } from "./errors";
|
||||
export interface SnapshotRefLookup {
|
||||
backendNodeId: number;
|
||||
refKey: string;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
}
|
||||
|
||||
function refEntryForTab(ctx: SessionContext, refKey: string, tabId: number) {
|
||||
const entry = ctx.refStore.resolveEntry(refKey);
|
||||
return entry?.tabId === tabId ? entry : null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -30,14 +23,9 @@ export function lookupSnapshotRef(
|
||||
tabId: number,
|
||||
): SnapshotRefLookup | null {
|
||||
const refKey = normaliseRef(ref);
|
||||
const entry = refEntryForTab(ctx, refKey, tabId);
|
||||
if (!entry) return null;
|
||||
return {
|
||||
backendNodeId: entry.backendNodeId,
|
||||
refKey,
|
||||
...(entry.frameId ? { frameId: entry.frameId } : {}),
|
||||
...(entry.cdpSessionId ? { cdpSessionId: entry.cdpSessionId } : {}),
|
||||
};
|
||||
const backendNodeId = ctx.refStore.resolve(refKey, { tabId });
|
||||
if (backendNodeId === null) return null;
|
||||
return { backendNodeId, refKey };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -49,7 +37,7 @@ export function resolveSnapshotRef(
|
||||
ctx: SessionContext,
|
||||
ref: string,
|
||||
tabId: number,
|
||||
): SnapshotRefLookup | RpcError {
|
||||
): { backendNodeId: number; refKey: string } | RpcError {
|
||||
const looked = lookupSnapshotRef(ctx, ref, tabId);
|
||||
if (looked === null) {
|
||||
return rpcError(
|
||||
|
||||
@@ -157,7 +157,6 @@ export interface ChromeWindowsApi {
|
||||
get(windowId: number): Promise<chrome.windows.Window>;
|
||||
getLastFocused(filters?: chrome.windows.QueryOptions): Promise<chrome.windows.Window>;
|
||||
create(props: chrome.windows.CreateData): Promise<chrome.windows.Window | undefined>;
|
||||
remove(windowId: number): Promise<void>;
|
||||
}
|
||||
|
||||
export const chromeWindowsApi: ChromeWindowsApi = {
|
||||
@@ -165,7 +164,6 @@ export const chromeWindowsApi: ChromeWindowsApi = {
|
||||
getLastFocused: (filters) =>
|
||||
filters ? chrome.windows.getLastFocused(filters) : chrome.windows.getLastFocused(),
|
||||
create: (p) => chrome.windows.create(p),
|
||||
remove: (id) => chrome.windows.remove(id),
|
||||
};
|
||||
|
||||
export interface AgentOverlayResetApi {
|
||||
@@ -217,9 +215,7 @@ export async function handleTabList(
|
||||
manager: SessionManager,
|
||||
params: TabListParams,
|
||||
api: ChromeTabsApi = chromeTabsApi,
|
||||
signal?: AbortSignal,
|
||||
): Promise<TabListResult | RpcError> {
|
||||
if (signal?.aborted) return { code: "cancelled", message: "tab_list aborted" };
|
||||
if (!params || typeof params.session_id !== "string" || params.session_id.length === 0) {
|
||||
return {
|
||||
code: "invalid_params",
|
||||
@@ -253,7 +249,6 @@ export async function handleTabList(
|
||||
const myAgentWindowId = ctx.agentWindowId;
|
||||
|
||||
const allTabs = await api.query({});
|
||||
if (signal?.aborted) return { code: "cancelled", message: "tab_list aborted" };
|
||||
const tabs: TabInfo[] = [];
|
||||
for (const t of allTabs) {
|
||||
if (typeof t.id !== "number") continue;
|
||||
@@ -387,12 +382,7 @@ async function createTabAndCleanup(
|
||||
try {
|
||||
await getTabsApi(deps).remove(tab.id);
|
||||
} catch (cleanupErr) {
|
||||
return rpcError(
|
||||
"protocol_error",
|
||||
"cleanup_failed",
|
||||
`tab_create aborted but cleanup of tab ${tab.id} failed: ${describeError(cleanupErr)}`,
|
||||
{ resource_type: "tab", resource_id: tab.id },
|
||||
);
|
||||
console.debug("[bsk tab_create] cleanup after abort failed", cleanupErr);
|
||||
}
|
||||
}
|
||||
return { code: "cancelled", message: "tab_create aborted" };
|
||||
@@ -703,17 +693,7 @@ async function moveTabForBorrow(
|
||||
try {
|
||||
await tabsApi.move(tabId, { windowId: originalWindowId, index: originalIndex });
|
||||
} catch (rollbackErr) {
|
||||
return rpcError(
|
||||
"protocol_error",
|
||||
"cleanup_failed",
|
||||
`tab_borrow aborted but rollback of tab ${tabId} to window ${originalWindowId} failed: ${describeError(rollbackErr)}`,
|
||||
{
|
||||
resource_type: "tab",
|
||||
resource_id: tabId,
|
||||
original_window_id: originalWindowId,
|
||||
original_index: originalIndex,
|
||||
},
|
||||
);
|
||||
console.debug("[bsk tab_borrow] rollback move failed", rollbackErr);
|
||||
}
|
||||
return { code: "cancelled", message: "tab_borrow aborted" };
|
||||
}
|
||||
@@ -845,12 +825,6 @@ export interface ReturnOutcome {
|
||||
fallback: boolean;
|
||||
}
|
||||
|
||||
interface FallbackWindowTarget {
|
||||
windowId: number;
|
||||
index: number;
|
||||
created: boolean;
|
||||
}
|
||||
|
||||
function describeError(err: unknown): string {
|
||||
return err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
@@ -859,13 +833,10 @@ async function chooseFallbackWindow(
|
||||
ctx: SessionContext,
|
||||
windowsApi: ChromeWindowsApi,
|
||||
isAgentWindowId: (windowId: number) => boolean,
|
||||
signal?: AbortSignal,
|
||||
): Promise<FallbackWindowTarget | RpcError> {
|
||||
): Promise<{ windowId: number; index: number } | RpcError> {
|
||||
let lastFocusedError: unknown;
|
||||
try {
|
||||
const last = await windowsApi.getLastFocused({ windowTypes: ["normal"] });
|
||||
const cancelled = aborted(signal, "tab_return");
|
||||
if (cancelled) return cancelled;
|
||||
const lastId = typeof last?.id === "number" ? last.id : null;
|
||||
// Never relocate a user tab into *any* session's Agent Window: this
|
||||
// session's is excluded explicitly, and other sessions' are excluded
|
||||
@@ -875,15 +846,12 @@ async function chooseFallbackWindow(
|
||||
// session write to it and, worse, see it destroyed when that session
|
||||
// stops and closes its window.
|
||||
if (lastId !== null && lastId !== ctx.agentWindowId && !isAgentWindowId(lastId)) {
|
||||
return { windowId: lastId, index: -1, created: false };
|
||||
return { windowId: lastId, index: -1 };
|
||||
}
|
||||
} catch (err) {
|
||||
lastFocusedError = err;
|
||||
}
|
||||
|
||||
const cancelled = aborted(signal, "tab_return");
|
||||
if (cancelled) return cancelled;
|
||||
|
||||
try {
|
||||
const created = await windowsApi.create({
|
||||
url: NEW_TAB_DEFAULT_URL,
|
||||
@@ -895,20 +863,7 @@ async function chooseFallbackWindow(
|
||||
message: "tab_return: failed to create fallback window",
|
||||
};
|
||||
}
|
||||
if (aborted(signal, "tab_return")) {
|
||||
try {
|
||||
await windowsApi.remove(created.id);
|
||||
} catch (cleanupErr) {
|
||||
return rpcError(
|
||||
"protocol_error",
|
||||
"cleanup_failed",
|
||||
`tab_return aborted but cleanup of fallback window ${created.id} failed: ${describeError(cleanupErr)}`,
|
||||
{ resource_type: "window", resource_id: created.id },
|
||||
);
|
||||
}
|
||||
return { code: "cancelled", message: "tab_return aborted" };
|
||||
}
|
||||
return { windowId: created.id, index: 0, created: true };
|
||||
return { windowId: created.id, index: 0 };
|
||||
} catch (err) {
|
||||
const suffix = lastFocusedError
|
||||
? ` (after getLastFocused failed: ${describeError(lastFocusedError)})`
|
||||
@@ -920,25 +875,6 @@ async function chooseFallbackWindow(
|
||||
}
|
||||
}
|
||||
|
||||
async function cleanupUnusedFallbackWindow(
|
||||
windowsApi: ChromeWindowsApi,
|
||||
target: FallbackWindowTarget,
|
||||
reason: string,
|
||||
): Promise<RpcError | null> {
|
||||
if (!target.created) return null;
|
||||
try {
|
||||
await windowsApi.remove(target.windowId);
|
||||
return null;
|
||||
} catch (cleanupErr) {
|
||||
return rpcError(
|
||||
"protocol_error",
|
||||
"cleanup_failed",
|
||||
`${reason}; cleanup of fallback window ${target.windowId} failed: ${describeError(cleanupErr)}`,
|
||||
{ resource_type: "window", resource_id: target.windowId },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function resetAgentOverlaysInReturnedTab(
|
||||
ctx: SessionContext,
|
||||
tabId: number,
|
||||
@@ -975,13 +911,10 @@ export async function returnBorrowedTab(
|
||||
const tabsApi = getTabsApi(deps);
|
||||
const windowsApi = getWindowsApi(deps);
|
||||
const isAgentWindowId = getIsAgentWindowId(deps);
|
||||
const alreadyCancelled = aborted(deps.signal, "tab_return");
|
||||
if (alreadyCancelled) return alreadyCancelled;
|
||||
|
||||
let targetWindowId = entry.originalWindowId;
|
||||
let targetIndex = entry.originalIndex;
|
||||
let fallback = false;
|
||||
let fallbackTarget: FallbackWindowTarget | null = null;
|
||||
|
||||
// Check the original window is still around.
|
||||
let originalAlive = true;
|
||||
@@ -991,30 +924,14 @@ export async function returnBorrowedTab(
|
||||
console.debug("[bsk tab_return] original window gone, falling back", err);
|
||||
originalAlive = false;
|
||||
}
|
||||
const cancelledAfterLookup = aborted(deps.signal, "tab_return");
|
||||
if (cancelledAfterLookup) return cancelledAfterLookup;
|
||||
if (!originalAlive) {
|
||||
fallback = true;
|
||||
const target = await chooseFallbackWindow(ctx, windowsApi, isAgentWindowId, deps.signal);
|
||||
const target = await chooseFallbackWindow(ctx, windowsApi, isAgentWindowId);
|
||||
if ("code" in target) return target;
|
||||
fallbackTarget = target;
|
||||
targetWindowId = target.windowId;
|
||||
targetIndex = target.index;
|
||||
}
|
||||
|
||||
const cancelledBeforeMove = aborted(deps.signal, "tab_return");
|
||||
if (cancelledBeforeMove) {
|
||||
if (fallbackTarget) {
|
||||
const cleanupError = await cleanupUnusedFallbackWindow(
|
||||
windowsApi,
|
||||
fallbackTarget,
|
||||
"tab_return aborted before moving the borrowed tab",
|
||||
);
|
||||
if (cleanupError) return cleanupError;
|
||||
}
|
||||
return cancelledBeforeMove;
|
||||
}
|
||||
|
||||
try {
|
||||
const moved = await tabsApi.move(tabId, {
|
||||
windowId: targetWindowId,
|
||||
@@ -1030,33 +947,14 @@ export async function returnBorrowedTab(
|
||||
fallback,
|
||||
};
|
||||
} catch (err) {
|
||||
if (fallbackTarget) {
|
||||
const cleanupError = await cleanupUnusedFallbackWindow(
|
||||
windowsApi,
|
||||
fallbackTarget,
|
||||
`tab_return could not move tab ${tabId}`,
|
||||
);
|
||||
if (cleanupError) return cleanupError;
|
||||
}
|
||||
const cancelledAfterMoveFailure = aborted(deps.signal, "tab_return");
|
||||
if (cancelledAfterMoveFailure) return cancelledAfterMoveFailure;
|
||||
if (!fallback) {
|
||||
const target = await chooseFallbackWindow(ctx, windowsApi, isAgentWindowId, deps.signal);
|
||||
const target = await chooseFallbackWindow(ctx, windowsApi, isAgentWindowId);
|
||||
if ("code" in target) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: `tab_return: chrome.tabs.move failed: ${describeError(err)}; fallback failed: ${target.message}`,
|
||||
};
|
||||
}
|
||||
const cancelledBeforeFallbackMove = aborted(deps.signal, "tab_return");
|
||||
if (cancelledBeforeFallbackMove) {
|
||||
const cleanupError = await cleanupUnusedFallbackWindow(
|
||||
windowsApi,
|
||||
target,
|
||||
"tab_return aborted before fallback move",
|
||||
);
|
||||
return cleanupError ?? cancelledBeforeFallbackMove;
|
||||
}
|
||||
try {
|
||||
const moved = await tabsApi.move(tabId, {
|
||||
windowId: target.windowId,
|
||||
@@ -1072,12 +970,6 @@ export async function returnBorrowedTab(
|
||||
fallback: true,
|
||||
};
|
||||
} catch (fallbackErr) {
|
||||
const cleanupError = await cleanupUnusedFallbackWindow(
|
||||
windowsApi,
|
||||
target,
|
||||
`tab_return fallback move for tab ${tabId} failed`,
|
||||
);
|
||||
if (cleanupError) return cleanupError;
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: `tab_return: chrome.tabs.move failed: ${describeError(err)}; fallback move failed: ${describeError(fallbackErr)}`,
|
||||
|
||||
@@ -18,8 +18,6 @@ function fakeSnapshotReply() {
|
||||
"auto",
|
||||
"type",
|
||||
"password",
|
||||
"value",
|
||||
"hunter2",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
return {
|
||||
@@ -31,8 +29,7 @@ function fakeSnapshotReply() {
|
||||
nodeType: [1, 1, 1, 1],
|
||||
nodeName: [i("html"), i("body"), i("div"), i("input")],
|
||||
backendNodeId: [10, 11, 12, 13],
|
||||
attributes: [[], [], [], [i("type"), i("password"), i("value"), i("hunter2")]],
|
||||
inputValue: { index: [3], value: [i("hunter2")] },
|
||||
attributes: [[], [], [], [i("type"), i("password")]],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 3],
|
||||
@@ -220,8 +217,7 @@ function makeCdp(snapshot: unknown) {
|
||||
{
|
||||
state: "filled",
|
||||
sensitive: true,
|
||||
defaultValue: "hunter2",
|
||||
value: "hunter2",
|
||||
defaultValue: "",
|
||||
placeholder: "secret",
|
||||
},
|
||||
],
|
||||
@@ -266,11 +262,10 @@ describe("captureViewModel", () => {
|
||||
|
||||
expect(input).toMatchObject({
|
||||
formState: "filled",
|
||||
formDefaultValue: "",
|
||||
formPlaceholder: "secret",
|
||||
attrs: { type: "password" },
|
||||
});
|
||||
expect(input?.formValue).toBeUndefined();
|
||||
expect(input?.formDefaultValue).toBeUndefined();
|
||||
expect(cdp.send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
@@ -682,7 +677,7 @@ describe("captureViewModel", () => {
|
||||
expect(nodes.find((n) => n.backendNodeId === 11)?.cursor).toBe("auto");
|
||||
});
|
||||
|
||||
it("subtracts scroll offset to make rects viewport-relative", async () => {
|
||||
it("preserves document rects while deriving viewport rects after scroll", async () => {
|
||||
const cdp = {
|
||||
send: vi.fn(async (_t: number, method: string) => {
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
@@ -696,9 +691,10 @@ describe("captureViewModel", () => {
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
const div = nodes.find((n) => n.backendNodeId === 12);
|
||||
expect(div?.localRect?.y).toBe(-200);
|
||||
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
|
||||
const input = nodes.find((n) => n.backendNodeId === 13);
|
||||
expect(input?.documentRect).toEqual({ x: 400, y: 300, w: 200, h: 40 });
|
||||
expect(input?.localRect).toEqual({ x: 400, y: 100, w: 200, h: 40 });
|
||||
expect(input?.rect).toEqual({ x: 400, y: 100, w: 200, h: 40 });
|
||||
});
|
||||
|
||||
it("normalizes device-pixel bounds by devicePixelRatio", async () => {
|
||||
@@ -842,7 +838,6 @@ describe("captureViewModel", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "main-frame",
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
@@ -870,7 +865,6 @@ describe("captureViewModel", () => {
|
||||
},
|
||||
// documents[1]: the iframe's sub-document with a login input
|
||||
{
|
||||
frameId: "child-frame",
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
@@ -900,14 +894,7 @@ describe("captureViewModel", () => {
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const {
|
||||
nodes,
|
||||
iframeNodes,
|
||||
frameNodes,
|
||||
frameOwnerBackendNodeIds,
|
||||
frameParentIds,
|
||||
rootFrameId,
|
||||
} = await captureViewModel(cdp, 4);
|
||||
const { nodes, iframeNodes } = await captureViewModel(cdp, 4);
|
||||
// Main frame has 4 nodes; iframe is included
|
||||
expect(nodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
|
||||
// iframeNodes keyed by the <iframe> element's backendNodeId=13
|
||||
@@ -920,13 +907,9 @@ describe("captureViewModel", () => {
|
||||
expect(input?.ownerFrameBackendNodeId).toBe(13);
|
||||
expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
|
||||
expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
|
||||
expect(rootFrameId).toBe("main-frame");
|
||||
expect(frameNodes?.get("child-frame")).toBe(subNodes);
|
||||
expect(frameOwnerBackendNodeIds?.get("child-frame")).toBe(13);
|
||||
expect(frameParentIds?.get("child-frame")).toBe("main-frame");
|
||||
});
|
||||
|
||||
it("captures iframe documents without requiring owner-frame geometry", async () => {
|
||||
it("clips iframe sub-document rects to the iframe viewport in top coordinates", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
@@ -954,10 +937,16 @@ describe("captureViewModel", () => {
|
||||
contentDocumentIndex: { index: [2], value: [1] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1],
|
||||
styles: [[i("static"), i("auto")]],
|
||||
bounds: [[0, 0, 1000, 800]],
|
||||
paintOrders: [0],
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[950, 760, 200, 100],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -995,7 +984,7 @@ describe("captureViewModel", () => {
|
||||
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101);
|
||||
|
||||
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
|
||||
expect(input?.rect).toBeNull();
|
||||
expect(input?.rect).toEqual({ x: 970, y: 780, w: 30, h: 20 });
|
||||
});
|
||||
|
||||
it("recursively normalizes nested iframe sub-documents", async () => {
|
||||
@@ -1090,10 +1079,8 @@ describe("captureViewModel", () => {
|
||||
const nestedIframe = iframeNodes.get(13)?.find((n) => n.backendNodeId === 23);
|
||||
const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31);
|
||||
|
||||
expect(nestedIframe?.localRect).toEqual({ x: 5, y: 6, w: 100, h: 80 });
|
||||
expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
|
||||
expect(input?.ownerFrameBackendNodeId).toBe(23);
|
||||
expect(input?.localRect).toEqual({ x: 1, y: 2, w: 40, h: 20 });
|
||||
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
|
||||
});
|
||||
|
||||
|
||||
@@ -1,135 +0,0 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { CdpRunner } from "../../shared";
|
||||
import type { CapturedNode, CapturedViewModel } from "../capture";
|
||||
import { captureFrameData } from "../frame-capture";
|
||||
|
||||
function ownerNode(backendNodeId: number, x: number): CapturedNode {
|
||||
return {
|
||||
backendNodeId,
|
||||
parentBackendNodeId: 1,
|
||||
frameId: "main",
|
||||
ownerFrameBackendNodeId: null,
|
||||
tag: "iframe",
|
||||
attrs: {},
|
||||
rect: { x, y: 100, w: 300, h: 200 },
|
||||
localRect: { x, y: 100, w: 300, h: 200 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
};
|
||||
}
|
||||
|
||||
function childSnapshot(frameId: string, backendNodeId: number) {
|
||||
const strings = [frameId, "body", "button", "static", "auto", "pointer"];
|
||||
return {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
frameId,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [1, 2],
|
||||
backendNodeId: [backendNodeId - 1, backendNodeId],
|
||||
attributes: [[], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[3, 4, 4],
|
||||
[3, 4, 5],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 300, 200],
|
||||
[10, 20, 100, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe("captureFrameData", () => {
|
||||
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
|
||||
const leftOwner = ownerNode(10, 50);
|
||||
const rightOwner = ownerNode(20, 500);
|
||||
const captured: CapturedViewModel = {
|
||||
nodes: [leftOwner, rightOwner],
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
frameNodes: new Map([["main", [leftOwner, rightOwner]]]),
|
||||
frameOwnerBackendNodeIds: new Map(),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
const sendToTarget = vi.fn(async (target, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return target.sessionId === "left-session"
|
||||
? childSnapshot("left", 101)
|
||||
: childSnapshot("right", 201);
|
||||
}
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
throw new Error(`unexpected ${method}`);
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method, params) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
if (method === "DOM.getBoxModel") {
|
||||
const backendNodeId = (params as { backendNodeId?: number })?.backendNodeId;
|
||||
const x = backendNodeId === 10 ? 50 : 500;
|
||||
return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
|
||||
}
|
||||
throw new Error(`unexpected root ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "left",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "left-session" },
|
||||
},
|
||||
{
|
||||
frameId: "right",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 4, sessionId: "right-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const trees = await captureFrameData(cdp, 4, captured);
|
||||
|
||||
expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]);
|
||||
expect(
|
||||
captured.frameNodes?.get("left")?.find((node) => node.backendNodeId === 101)?.rect,
|
||||
).toEqual({ x: 60, y: 120, w: 100, h: 40 });
|
||||
expect(
|
||||
captured.frameNodes?.get("right")?.find((node) => node.backendNodeId === 201)?.rect,
|
||||
).toEqual({ x: 510, y: 120, w: 100, h: 40 });
|
||||
expect(captured.frameOwnerBackendNodeIds).toEqual(
|
||||
new Map([
|
||||
["left", 10],
|
||||
["right", 20],
|
||||
]),
|
||||
);
|
||||
expect(captured.frameParentIds).toEqual(
|
||||
new Map([
|
||||
["left", "main"],
|
||||
["right", "main"],
|
||||
]),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,174 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||
import type { CapturedNode, CapturedViewModel } from "../capture";
|
||||
import { buildFrameDocuments, type FrameAxBatch, type FrameOwnedAxNode } from "../frame-document";
|
||||
|
||||
function domNode(frameId: string, backendNodeId: number): CapturedNode {
|
||||
return {
|
||||
backendNodeId,
|
||||
parentBackendNodeId: null,
|
||||
frameId,
|
||||
tag: "div",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 20, h: 20 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
};
|
||||
}
|
||||
|
||||
function captured(frameNodes: Map<string, CapturedNode[]>): CapturedViewModel {
|
||||
return {
|
||||
nodes: frameNodes.get("main") ?? [],
|
||||
viewport: { width: 800, height: 600 },
|
||||
iframeNodes: new Map(),
|
||||
frameNodes,
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
}
|
||||
|
||||
describe("buildFrameDocuments", () => {
|
||||
it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", () => {
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{ frameId: "child", parentFrameId: "main", target: { tabId: 4 } },
|
||||
],
|
||||
};
|
||||
const batches: FrameAxBatch<FrameOwnedAxNode>[] = [
|
||||
{
|
||||
frame: graph.frames[0],
|
||||
nodes: [
|
||||
{ nodeId: "root", frameId: "main", backendDOMNodeId: 1, childIds: ["h"] },
|
||||
{
|
||||
nodeId: "h",
|
||||
frameId: "child",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 100,
|
||||
childIds: ["virtual"],
|
||||
},
|
||||
{ nodeId: "virtual", parentId: "h" },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const documents = buildFrameDocuments(
|
||||
graph,
|
||||
batches,
|
||||
captured(
|
||||
new Map([
|
||||
["main", [domNode("main", 1)]],
|
||||
["child", [domNode("child", 100)]],
|
||||
]),
|
||||
),
|
||||
);
|
||||
|
||||
expect(documents.find((document) => document.frameId === "main")?.axNodes).toHaveLength(1);
|
||||
const childNodes = documents.find((document) => document.frameId === "child")?.axNodes ?? [];
|
||||
expect(childNodes.map((node) => node.nodeId)).toEqual(["h", "virtual"]);
|
||||
expect(childNodes.find((node) => node.nodeId === "h")?.parentId).toBeUndefined();
|
||||
expect(childNodes.find((node) => node.nodeId === "virtual")?.frameId).toBe("child");
|
||||
});
|
||||
|
||||
it("deduplicates overlapping same-target AX results using the strongest ownership", () => {
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{ frameId: "child", parentFrameId: "main", target: { tabId: 4 } },
|
||||
],
|
||||
};
|
||||
const childNode = { nodeId: "shared", backendDOMNodeId: 100 };
|
||||
const batches: FrameAxBatch<FrameOwnedAxNode>[] = [
|
||||
{
|
||||
frame: graph.frames[0],
|
||||
nodes: [{ ...childNode, frameId: "child" }],
|
||||
},
|
||||
{
|
||||
frame: graph.frames[1],
|
||||
nodes: [childNode],
|
||||
},
|
||||
];
|
||||
|
||||
const documents = buildFrameDocuments(
|
||||
graph,
|
||||
batches,
|
||||
captured(
|
||||
new Map([
|
||||
["main", [domNode("main", 1)]],
|
||||
["child", [domNode("child", 100)]],
|
||||
]),
|
||||
),
|
||||
);
|
||||
|
||||
expect(documents.find((document) => document.frameId === "main")?.axNodes).toEqual([]);
|
||||
expect(documents.find((document) => document.frameId === "child")?.axNodes).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("uses target-scoped backend ownership when OOPIF backend ids collide", () => {
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "left",
|
||||
parentFrameId: "main",
|
||||
target: { tabId: 4, sessionId: "left-session" },
|
||||
},
|
||||
{
|
||||
frameId: "right",
|
||||
parentFrameId: "main",
|
||||
target: { tabId: 4, sessionId: "right-session" },
|
||||
},
|
||||
],
|
||||
};
|
||||
const batches: FrameAxBatch<FrameOwnedAxNode>[] = graph.frames.slice(1).map((frame) => ({
|
||||
frame,
|
||||
nodes: [{ nodeId: `${frame.frameId}-node`, backendDOMNodeId: 7 }],
|
||||
}));
|
||||
|
||||
const documents = buildFrameDocuments(
|
||||
graph,
|
||||
batches,
|
||||
captured(
|
||||
new Map([
|
||||
["main", []],
|
||||
["left", [domNode("left", 7)]],
|
||||
["right", [domNode("right", 7)]],
|
||||
]),
|
||||
),
|
||||
);
|
||||
|
||||
expect(documents.find((document) => document.frameId === "left")?.axNodes[0]?.nodeId).toBe(
|
||||
"left-node",
|
||||
);
|
||||
expect(documents.find((document) => document.frameId === "right")?.axNodes[0]?.nodeId).toBe(
|
||||
"right-node",
|
||||
);
|
||||
});
|
||||
|
||||
it("uses captured frame parent identity instead of guessing from backend ids", () => {
|
||||
const frameNodes = new Map([
|
||||
["main", [domNode("main", 1)]],
|
||||
["left", [domNode("left", 7)]],
|
||||
["right", [domNode("right", 7)]],
|
||||
["child", [domNode("child", 9)]],
|
||||
]);
|
||||
const capture = captured(frameNodes);
|
||||
capture.frameOwnerBackendNodeIds = new Map([["child", 7]]);
|
||||
capture.frameParentIds = new Map([["child", "left"]]);
|
||||
const frames = ["main", "left", "right", "child"].map((frameId) => ({
|
||||
frame: { frameId, target: { tabId: 4 } },
|
||||
nodes: [],
|
||||
}));
|
||||
|
||||
const documents = buildFrameDocuments(null, frames, capture);
|
||||
|
||||
expect(documents.find((document) => document.frameId === "child")).toMatchObject({
|
||||
parentFrameId: "left",
|
||||
ownerBackendNodeId: 7,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -7,7 +7,6 @@
|
||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
||||
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
const REQUESTED_STYLES = ["position", "pointer-events", "cursor"] as const;
|
||||
@@ -18,15 +17,16 @@ const STYLE_COL = Object.fromEntries(
|
||||
export interface CapturedNode {
|
||||
backendNodeId: number;
|
||||
parentBackendNodeId: number | null;
|
||||
frameId?: string;
|
||||
/** Owning iframe backend node id; `null` for the top-level document. */
|
||||
ownerFrameBackendNodeId?: number | null;
|
||||
tag: string;
|
||||
attrs: Record<string, string>;
|
||||
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
|
||||
rect: Rect | null;
|
||||
/** Frame-local viewport-relative CSS px before top-level projection. */
|
||||
/** Frame-local viewport-relative CSS px before top-level translation/clipping. */
|
||||
localRect?: Rect | null;
|
||||
/** Frame-local document-relative CSS px as reported by DOMSnapshot. */
|
||||
documentRect?: Rect | null;
|
||||
paintOrder: number;
|
||||
position: string;
|
||||
pointerEvents: string;
|
||||
@@ -59,11 +59,6 @@ export interface CapturedViewModel {
|
||||
nodes: CapturedNode[];
|
||||
viewport: Viewport;
|
||||
iframeNodes: CapturedIframeNodes;
|
||||
frameNodes?: Map<string, CapturedNode[]>;
|
||||
frameOwnerBackendNodeIds?: Map<string, number>;
|
||||
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
|
||||
frameParentIds?: Map<string, string>;
|
||||
rootFrameId?: string;
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
@@ -72,25 +67,6 @@ export interface CapturedViewModel {
|
||||
export interface CaptureViewModelOptions {
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
function captureAbortError(): Error {
|
||||
const error = new Error("observation aborted");
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
function isAbortError(error: unknown): boolean {
|
||||
return (
|
||||
typeof error === "object" &&
|
||||
error !== null &&
|
||||
(error as { name?: string }).name === "AbortError"
|
||||
);
|
||||
}
|
||||
|
||||
function throwIfAborted(signal: AbortSignal | undefined): void {
|
||||
if (signal?.aborted) throw captureAbortError();
|
||||
}
|
||||
|
||||
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
|
||||
@@ -99,12 +75,7 @@ interface SparseArray {
|
||||
value: number[];
|
||||
}
|
||||
|
||||
interface RareBooleanData {
|
||||
index: number[];
|
||||
}
|
||||
|
||||
interface SnapshotDocument {
|
||||
frameId?: string | number;
|
||||
scrollOffsetX?: number;
|
||||
scrollOffsetY?: number;
|
||||
nodes?: {
|
||||
@@ -119,10 +90,6 @@ interface SnapshotDocument {
|
||||
nodeValue?: number[];
|
||||
/** Maps node array index → index into `documents[]` for frame content. */
|
||||
contentDocumentIndex?: SparseArray;
|
||||
inputValue?: SparseArray;
|
||||
textValue?: SparseArray;
|
||||
inputChecked?: RareBooleanData;
|
||||
optionSelected?: RareBooleanData;
|
||||
};
|
||||
layout?: {
|
||||
nodeIndex?: number[];
|
||||
@@ -132,12 +99,6 @@ interface SnapshotDocument {
|
||||
};
|
||||
}
|
||||
|
||||
function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined {
|
||||
if (typeof document.frameId === "string") return document.frameId || undefined;
|
||||
if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
interface SnapshotReply {
|
||||
strings?: string[];
|
||||
documents?: SnapshotDocument[];
|
||||
@@ -157,25 +118,6 @@ function isFormControlTag(tag: string): boolean {
|
||||
return tag === "input" || tag === "textarea" || tag === "select";
|
||||
}
|
||||
|
||||
function isSensitiveFormControl(node: Pick<CapturedNode, "tag" | "attrs">): boolean {
|
||||
return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password";
|
||||
}
|
||||
|
||||
function snapshotFormState(
|
||||
value: string | undefined,
|
||||
defaultValue: string,
|
||||
hasDefaultValue: boolean,
|
||||
sensitive: boolean,
|
||||
): CapturedNode["formState"] {
|
||||
if (sensitive) {
|
||||
if (value === undefined && !hasDefaultValue) return undefined;
|
||||
return (value ?? defaultValue) === "" ? "empty" : "filled";
|
||||
}
|
||||
if (value === undefined) return undefined;
|
||||
if (value === "") return "empty";
|
||||
return value === defaultValue ? "default" : "filled";
|
||||
}
|
||||
|
||||
const MAX_FORM_ENRICH_CONTROLS = 250;
|
||||
|
||||
interface CapturedFormState {
|
||||
@@ -207,8 +149,9 @@ function formStateBatchExpression(maxControls: number): string {
|
||||
return {
|
||||
state,
|
||||
sensitive,
|
||||
defaultValue,
|
||||
placeholder,
|
||||
...(sensitive ? {} : { value: rawValue, defaultValue }),
|
||||
...(sensitive ? {} : { value: rawValue }),
|
||||
};
|
||||
};
|
||||
const collect = (doc) => {
|
||||
@@ -250,14 +193,10 @@ function applyFormStates(nodes: CapturedNode[], states: CapturedFormState[]): vo
|
||||
index += 1;
|
||||
if (!state) continue;
|
||||
node.formState = state.state;
|
||||
node.formDefaultValue = state.defaultValue ?? "";
|
||||
node.formPlaceholder = state.placeholder ?? "";
|
||||
if (state.sensitive || isSensitiveFormControl(node)) {
|
||||
delete node.formValue;
|
||||
delete node.formDefaultValue;
|
||||
delete node.attrs.value;
|
||||
} else {
|
||||
node.formDefaultValue = state.defaultValue ?? "";
|
||||
if (state.value !== undefined) node.formValue = state.value;
|
||||
if (!state.sensitive && state.value !== undefined) {
|
||||
node.formValue = state.value;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -266,25 +205,21 @@ async function enrichFormControlStates(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
frameNodeGroups: CapturedNode[][],
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
const hasControls = frameNodeGroups.some((nodes) =>
|
||||
nodes.some((node) => isFormControlTag(node.tag)),
|
||||
);
|
||||
if (!hasControls) return;
|
||||
try {
|
||||
throwIfAborted(signal);
|
||||
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(signal);
|
||||
const frameStates = flattenFrameFormStates(runtimeValue<CapturedFrameFormState>(result));
|
||||
for (let i = 0; i < frameNodeGroups.length; i += 1) {
|
||||
applyFormStates(frameNodeGroups[i], frameStates[i] ?? []);
|
||||
}
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
} catch {
|
||||
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
|
||||
}
|
||||
}
|
||||
@@ -316,9 +251,9 @@ interface ParseDocumentResult {
|
||||
}
|
||||
|
||||
interface FrameContext {
|
||||
frameId?: string;
|
||||
ownerFrameBackendNodeId: number | null;
|
||||
projection: GeometryProjection | null;
|
||||
originInTop: { x: number; y: number };
|
||||
clipRectInTop: Rect;
|
||||
scrollX: number;
|
||||
scrollY: number;
|
||||
}
|
||||
@@ -337,6 +272,38 @@ function devicePixelRatio(metrics: LayoutMetricsReply): number {
|
||||
return dpr >= 1 ? dpr : 1;
|
||||
}
|
||||
|
||||
function viewportRect(viewport: Viewport): Rect {
|
||||
return { x: 0, y: 0, w: Math.max(0, viewport.width), h: Math.max(0, viewport.height) };
|
||||
}
|
||||
|
||||
function intersectRects(a: Rect, b: Rect): Rect | null {
|
||||
const x1 = Math.max(a.x, b.x);
|
||||
const y1 = Math.max(a.y, b.y);
|
||||
const x2 = Math.min(a.x + a.w, b.x + b.w);
|
||||
const y2 = Math.min(a.y + a.h, b.y + b.h);
|
||||
if (x2 <= x1 || y2 <= y1) return null;
|
||||
return { x: x1, y: y1, w: x2 - x1, h: y2 - y1 };
|
||||
}
|
||||
|
||||
function rectInTop(localRect: Rect, context: FrameContext): Rect | null {
|
||||
const translated = {
|
||||
x: context.originInTop.x + localRect.x,
|
||||
y: context.originInTop.y + localRect.y,
|
||||
w: localRect.w,
|
||||
h: localRect.h,
|
||||
};
|
||||
return intersectRects(translated, context.clipRectInTop);
|
||||
}
|
||||
|
||||
function unclipRectInTop(localRect: Rect, context: FrameContext): Rect {
|
||||
return {
|
||||
x: context.originInTop.x + localRect.x,
|
||||
y: context.originInTop.y + localRect.y,
|
||||
w: localRect.w,
|
||||
h: localRect.h,
|
||||
};
|
||||
}
|
||||
|
||||
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
|
||||
const out = new Map<number, number>();
|
||||
if (!sparse?.index || !sparse.value) return out;
|
||||
@@ -460,20 +427,8 @@ function hoverStateExpression(): string {
|
||||
})()`;
|
||||
}
|
||||
|
||||
async function wait(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
throwIfAborted(signal);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
resolve();
|
||||
}, ms);
|
||||
const onAbort = () => {
|
||||
clearTimeout(timer);
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
reject(captureAbortError());
|
||||
};
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
});
|
||||
async function wait(ms: number): Promise<void> {
|
||||
await new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
|
||||
@@ -645,51 +600,41 @@ async function probeHoverSurfaces(
|
||||
): Promise<CapturedSurfaceProbe[]> {
|
||||
const started = Date.now();
|
||||
try {
|
||||
throwIfAborted(options.signal);
|
||||
const cssScan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverCssTriggerScanExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
const cssHoverPoints = runtimeValue<Array<{ x: number; y: number }>>(cssScan) ?? [];
|
||||
const candidates = buildHoverCandidates(nodes, cssHoverPoints);
|
||||
if (candidates.length === 0) return [];
|
||||
|
||||
const results: CapturedSurfaceProbe[] = [];
|
||||
const seen = new Set<number>();
|
||||
throwIfAborted(options.signal);
|
||||
await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
|
||||
try {
|
||||
throwIfAborted(options.signal);
|
||||
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
|
||||
throwIfAborted(options.signal);
|
||||
if (Date.now() - started > MAX_HOVER_PROBE_MS) break;
|
||||
if (results.length >= MAX_HOVER_SURFACES) break;
|
||||
if (seen.has(candidate.backendNodeId)) continue;
|
||||
try {
|
||||
await clearHover(cdp, tabId);
|
||||
throwIfAborted(options.signal);
|
||||
await wait(HOVER_SETTLE_MS, options.signal);
|
||||
await wait(HOVER_SETTLE_MS);
|
||||
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
const baselineItems = runtimeValue<HoverRuntimeItem[]>(baselineReply) ?? [];
|
||||
|
||||
throwIfAborted(options.signal);
|
||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: candidate.x,
|
||||
y: candidate.y,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
await wait(HOVER_SETTLE_MS, options.signal);
|
||||
await wait(HOVER_SETTLE_MS);
|
||||
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
const subItems = diffHoverItems(
|
||||
baselineItems,
|
||||
runtimeValue<HoverRuntimeItem[]>(collected) ?? [],
|
||||
@@ -703,8 +648,7 @@ async function probeHoverSurfaces(
|
||||
subItems,
|
||||
confidence: confidenceForHover(candidate, subItems),
|
||||
});
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
} catch {
|
||||
continue;
|
||||
} finally {
|
||||
await clearHover(cdp, tabId);
|
||||
@@ -715,7 +659,6 @@ async function probeHoverSurfaces(
|
||||
}
|
||||
return results;
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
console.debug("[bsk capture] hover surface probe failed", err);
|
||||
return [];
|
||||
}
|
||||
@@ -728,16 +671,13 @@ async function probeHoverSurfaces(
|
||||
export async function collectOverlayExcludedBackendIds(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Set<number>> {
|
||||
const excluded = new Set<number>();
|
||||
try {
|
||||
throwIfAborted(signal);
|
||||
const doc = await cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
||||
depth: 0,
|
||||
pierce: true,
|
||||
});
|
||||
throwIfAborted(signal);
|
||||
const rootNodeId = doc.root?.nodeId;
|
||||
if (typeof rootNodeId !== "number") return excluded;
|
||||
|
||||
@@ -745,7 +685,6 @@ export async function collectOverlayExcludedBackendIds(
|
||||
nodeId: rootNodeId,
|
||||
selector: OVERLAY_HOST_SELECTOR,
|
||||
});
|
||||
throwIfAborted(signal);
|
||||
if (typeof found.nodeId !== "number" || found.nodeId === 0) return excluded;
|
||||
|
||||
const described = await cdp.send<{ node?: CdpDomNode }>(tabId, "DOM.describeNode", {
|
||||
@@ -753,10 +692,8 @@ export async function collectOverlayExcludedBackendIds(
|
||||
depth: -1,
|
||||
pierce: true,
|
||||
});
|
||||
throwIfAborted(signal);
|
||||
collectBackendIdsFromDomNode(described.node, excluded);
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
console.debug("[bsk capture] overlay exclusion fallback failed", err);
|
||||
}
|
||||
return excluded;
|
||||
@@ -817,10 +754,6 @@ function parseDocumentNodes(
|
||||
const posCol = STYLE_COL.position;
|
||||
const peCol = STYLE_COL["pointer-events"];
|
||||
const cursorCol = STYLE_COL.cursor;
|
||||
const inputValues = sparseIndexMap(dn.inputValue);
|
||||
const textValues = sparseIndexMap(dn.textValue);
|
||||
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
|
||||
const selectedOptions = new Set(dn.optionSelected?.index ?? []);
|
||||
|
||||
// Collect visible text from #text child nodes so element CapturedNodes
|
||||
// carry a textContent value usable as a button/link label fallback.
|
||||
@@ -857,6 +790,7 @@ function parseDocumentNodes(
|
||||
|
||||
let rect: Rect | null = null;
|
||||
let localRect: Rect | null = null;
|
||||
let documentRect: Rect | null = null;
|
||||
let paintOrder = 0;
|
||||
let position = "static";
|
||||
let pointerEvents = "auto";
|
||||
@@ -865,16 +799,19 @@ function parseDocumentNodes(
|
||||
if (li !== undefined) {
|
||||
const b = dl?.bounds?.[li];
|
||||
if (b && b.length >= 4 && b[2] > 0 && b[3] > 0) {
|
||||
localRect = {
|
||||
x: b[0] / dpr - context.scrollX,
|
||||
y: b[1] / dpr - context.scrollY,
|
||||
documentRect = {
|
||||
x: b[0] / dpr,
|
||||
y: b[1] / dpr,
|
||||
w: b[2] / dpr,
|
||||
h: b[3] / dpr,
|
||||
};
|
||||
const bounds = context.projection
|
||||
? projectRectToViewport(localRect, context.projection)
|
||||
: null;
|
||||
rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||
localRect = {
|
||||
x: documentRect.x - context.scrollX,
|
||||
y: documentRect.y - context.scrollY,
|
||||
w: documentRect.w,
|
||||
h: documentRect.h,
|
||||
};
|
||||
rect = rectInTop(localRect, context);
|
||||
}
|
||||
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
||||
const styleRow = dl?.styles?.[li] ?? [];
|
||||
@@ -889,44 +826,20 @@ function parseDocumentNodes(
|
||||
|
||||
const textContent = nodeTextContent.get(n);
|
||||
|
||||
const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n);
|
||||
const rawFormValue =
|
||||
rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined;
|
||||
const sensitive = isSensitiveFormControl({ tag, attrs });
|
||||
const formDefaultValue = attrs.value ?? "";
|
||||
const formValue = sensitive ? undefined : rawFormValue;
|
||||
const formState = snapshotFormState(
|
||||
rawFormValue,
|
||||
formDefaultValue,
|
||||
Object.prototype.hasOwnProperty.call(attrs, "value"),
|
||||
sensitive,
|
||||
);
|
||||
if (sensitive) delete attrs.value;
|
||||
|
||||
nodes.push({
|
||||
backendNodeId,
|
||||
parentBackendNodeId,
|
||||
...(context.frameId ? { frameId: context.frameId } : {}),
|
||||
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
|
||||
tag,
|
||||
attrs,
|
||||
rect,
|
||||
localRect,
|
||||
documentRect,
|
||||
paintOrder,
|
||||
position,
|
||||
pointerEvents,
|
||||
cursor,
|
||||
textContent,
|
||||
...(formValue !== undefined ? { formValue } : {}),
|
||||
...(tag === "input" || tag === "textarea"
|
||||
? {
|
||||
formPlaceholder: attrs.placeholder ?? "",
|
||||
...(!sensitive ? { formDefaultValue } : {}),
|
||||
...(formState ? { formState } : {}),
|
||||
}
|
||||
: {}),
|
||||
...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}),
|
||||
...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}),
|
||||
});
|
||||
}
|
||||
return { nodes, excludedBackendNodeIds };
|
||||
@@ -934,8 +847,6 @@ function parseDocumentNodes(
|
||||
|
||||
interface ParseFrameDocumentsResult {
|
||||
iframeNodes: CapturedIframeNodes;
|
||||
frameOwnerBackendNodeIds: Map<string, number>;
|
||||
frameParentIds: Map<string, string>;
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
@@ -949,14 +860,10 @@ function parseChildFrameDocuments(
|
||||
visited = new Set<number>(),
|
||||
): ParseFrameDocumentsResult {
|
||||
const iframeNodes: CapturedIframeNodes = new Map();
|
||||
const frameOwnerBackendNodeIds = new Map<string, number>();
|
||||
const frameParentIds = new Map<string, string>();
|
||||
const excludedBackendNodeIds = new Set<number>();
|
||||
const parentDoc = documents[parentDocIndex];
|
||||
const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex);
|
||||
if (cdi.size === 0) {
|
||||
return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds };
|
||||
}
|
||||
if (cdi.size === 0) return { iframeNodes, excludedBackendNodeIds };
|
||||
|
||||
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
|
||||
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
|
||||
@@ -968,15 +875,19 @@ function parseChildFrameDocuments(
|
||||
const iframeBackendId = parentBackendIds[nodeArrayIdx];
|
||||
if (iframeBackendId === undefined) continue;
|
||||
const iframeNode = parentNodeByBackendId.get(iframeBackendId);
|
||||
const projection =
|
||||
parentContext.projection && iframeNode?.localRect
|
||||
? childFrameProjection(parentContext.projection, iframeNode.localRect)
|
||||
: null;
|
||||
if (!iframeNode?.localRect) continue;
|
||||
|
||||
const iframeRectInTop = unclipRectInTop(iframeNode.localRect, parentContext);
|
||||
const childClip = intersectRects(iframeRectInTop, parentContext.clipRectInTop);
|
||||
if (!childClip) {
|
||||
iframeNodes.set(iframeBackendId, []);
|
||||
continue;
|
||||
}
|
||||
|
||||
const childContext: FrameContext = {
|
||||
frameId: snapshotFrameId(childDoc, strings),
|
||||
ownerFrameBackendNodeId: iframeBackendId,
|
||||
projection,
|
||||
originInTop: { x: iframeRectInTop.x, y: iframeRectInTop.y },
|
||||
clipRectInTop: childClip,
|
||||
scrollX: childDoc.scrollOffsetX ?? 0,
|
||||
scrollY: childDoc.scrollOffsetY ?? 0,
|
||||
};
|
||||
@@ -984,10 +895,6 @@ function parseChildFrameDocuments(
|
||||
nextVisited.add(childDocIndex);
|
||||
const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext);
|
||||
iframeNodes.set(iframeBackendId, parsed.nodes);
|
||||
if (childContext.frameId) {
|
||||
frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId);
|
||||
if (parentContext.frameId) frameParentIds.set(childContext.frameId, parentContext.frameId);
|
||||
}
|
||||
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||
|
||||
const nested = parseChildFrameDocuments(
|
||||
@@ -1002,16 +909,10 @@ function parseChildFrameDocuments(
|
||||
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
|
||||
iframeNodes.set(nestedFrameId, nestedNodes);
|
||||
}
|
||||
for (const [frameId, ownerBackendNodeId] of nested.frameOwnerBackendNodeIds) {
|
||||
frameOwnerBackendNodeIds.set(frameId, ownerBackendNodeId);
|
||||
}
|
||||
for (const [frameId, parentFrameId] of nested.frameParentIds) {
|
||||
frameParentIds.set(frameId, parentFrameId);
|
||||
}
|
||||
for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||
}
|
||||
|
||||
return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds };
|
||||
return { iframeNodes, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
export async function captureViewModel(
|
||||
@@ -1019,9 +920,7 @@ export async function captureViewModel(
|
||||
tabId: number,
|
||||
options: CaptureViewModelOptions = {},
|
||||
): Promise<CapturedViewModel> {
|
||||
throwIfAborted(options.signal);
|
||||
const metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
throwIfAborted(options.signal);
|
||||
const dpr = devicePixelRatio(metrics);
|
||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
const viewport: Viewport = {
|
||||
@@ -1032,13 +931,11 @@ export async function captureViewModel(
|
||||
const scrollY = vpSrc.pageY ?? 0;
|
||||
|
||||
await cdp.send(tabId, "DOMSnapshot.enable", {});
|
||||
throwIfAborted(options.signal);
|
||||
const snap = await cdp.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
|
||||
computedStyles: REQUESTED_STYLES,
|
||||
includePaintOrder: true,
|
||||
includeDOMRects: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
|
||||
const strings = snap.strings ?? [];
|
||||
const documents = snap.documents ?? [];
|
||||
@@ -1054,13 +951,9 @@ export async function captureViewModel(
|
||||
}
|
||||
|
||||
const topContext: FrameContext = {
|
||||
frameId: snapshotFrameId(doc0, strings),
|
||||
ownerFrameBackendNodeId: null,
|
||||
projection: {
|
||||
sourceClips: [],
|
||||
edges: [],
|
||||
topViewport: viewport,
|
||||
},
|
||||
originInTop: { x: 0, y: 0 },
|
||||
clipRectInTop: viewportRect(viewport),
|
||||
scrollX,
|
||||
scrollY,
|
||||
};
|
||||
@@ -1074,35 +967,11 @@ export async function captureViewModel(
|
||||
excludedBackendNodeIds.add(id);
|
||||
}
|
||||
|
||||
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()], options.signal);
|
||||
throwIfAborted(options.signal);
|
||||
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()]);
|
||||
|
||||
const surfaceProbes = options.conditionalSurfaceProbe
|
||||
? await probeHoverSurfaces(cdp, tabId, nodes, options)
|
||||
: [];
|
||||
throwIfAborted(options.signal);
|
||||
|
||||
const frameNodes = new Map<string, CapturedNode[]>();
|
||||
if (topContext.frameId) frameNodes.set(topContext.frameId, nodes);
|
||||
for (const iframeFrameNodes of iframeNodes.values()) {
|
||||
const frameId = iframeFrameNodes.find((node) => node.frameId)?.frameId;
|
||||
if (frameId) frameNodes.set(frameId, iframeFrameNodes);
|
||||
}
|
||||
for (const [frameId, ownerBackendNodeId] of frameParsed.frameOwnerBackendNodeIds) {
|
||||
if (!frameNodes.has(frameId)) {
|
||||
frameNodes.set(frameId, iframeNodes.get(ownerBackendNodeId) ?? []);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
nodes,
|
||||
viewport,
|
||||
iframeNodes,
|
||||
frameNodes,
|
||||
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
|
||||
frameParentIds: frameParsed.frameParentIds,
|
||||
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
|
||||
surfaceProbes,
|
||||
excludedBackendNodeIds,
|
||||
};
|
||||
return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
@@ -1,179 +0,0 @@
|
||||
import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||
import { resolveFrameProjection } from "../frame-geometry";
|
||||
import { type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
|
||||
import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
|
||||
import {
|
||||
buildFrameDocuments,
|
||||
type FrameAxBatch,
|
||||
type FrameDocument,
|
||||
type FrameOwnedAxNode,
|
||||
} from "./frame-document";
|
||||
|
||||
export type FrameAxNode = FrameOwnedAxNode;
|
||||
export type CapturedFrameDocument<T extends FrameAxNode> = FrameDocument<T>;
|
||||
|
||||
function throwIfAborted(signal?: AbortSignal): void {
|
||||
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
|
||||
}
|
||||
|
||||
function isAbortError(err: unknown): boolean {
|
||||
return err instanceof Error && err.name === "AbortError";
|
||||
}
|
||||
|
||||
async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
|
||||
if (!cdp.getFrameGraph) return null;
|
||||
try {
|
||||
return await cdp.getFrameGraph(tabId);
|
||||
} catch (err) {
|
||||
console.debug("[bsk observation] frame graph capture failed", err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function transformFrameNodes(
|
||||
nodes: CapturedNode[],
|
||||
projection: GeometryProjection,
|
||||
): CapturedNode[] {
|
||||
return nodes.map((node) => ({
|
||||
...node,
|
||||
rect: (() => {
|
||||
const bounds = projectRectToViewport(node.rect, projection);
|
||||
return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||
})(),
|
||||
}));
|
||||
}
|
||||
|
||||
function frameOwnerNode(captured: CapturedViewModel, frame: CdpFrame): CapturedNode | undefined {
|
||||
if (frame.ownerBackendNodeId === undefined) return undefined;
|
||||
const parentNodes = frame.parentFrameId
|
||||
? captured.frameNodes?.get(frame.parentFrameId)
|
||||
: captured.nodes;
|
||||
return parentNodes?.find((node) => node.backendNodeId === frame.ownerBackendNodeId);
|
||||
}
|
||||
|
||||
async function captureMissingFrameDocuments(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
graph: CdpFrameGraph,
|
||||
captured: CapturedViewModel,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
if (!captured.frameNodes) captured.frameNodes = new Map();
|
||||
if (!captured.frameOwnerBackendNodeIds) captured.frameOwnerBackendNodeIds = new Map();
|
||||
if (!captured.frameParentIds) captured.frameParentIds = new Map();
|
||||
|
||||
for (const frame of graph.frames) {
|
||||
if (!frame.target.sessionId || captured.frameNodes.has(frame.frameId)) continue;
|
||||
const owner = frameOwnerNode(captured, frame);
|
||||
if (!owner?.rect || frame.ownerBackendNodeId === undefined) continue;
|
||||
try {
|
||||
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
|
||||
signal,
|
||||
});
|
||||
const projection = await resolveFrameProjection(cdp, graph, frame.frameId);
|
||||
if (!projection) continue;
|
||||
const rootFrameId = child.rootFrameId ?? frame.frameId;
|
||||
const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
|
||||
if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
|
||||
if (rootFrameId !== frame.frameId && !childFrames.has(frame.frameId)) {
|
||||
childFrames.set(
|
||||
frame.frameId,
|
||||
child.nodes.map((node) => ({ ...node, frameId: frame.frameId })),
|
||||
);
|
||||
}
|
||||
for (const [childFrameId, nodes] of childFrames) {
|
||||
captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
|
||||
}
|
||||
captured.iframeNodes.set(
|
||||
frame.ownerBackendNodeId,
|
||||
captured.frameNodes.get(frame.frameId) ?? [],
|
||||
);
|
||||
captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
|
||||
if (frame.parentFrameId) captured.frameParentIds.set(frame.frameId, frame.parentFrameId);
|
||||
for (const [childFrameId, ownerBackendNodeId] of child.frameOwnerBackendNodeIds ?? []) {
|
||||
captured.frameOwnerBackendNodeIds.set(childFrameId, ownerBackendNodeId);
|
||||
}
|
||||
for (const [childFrameId, parentFrameId] of child.frameParentIds ?? []) {
|
||||
captured.frameParentIds.set(childFrameId, parentFrameId);
|
||||
}
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
console.debug("[bsk observation] child frame DOM capture failed", {
|
||||
frameId: frame.frameId,
|
||||
err,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function captureAxTrees<T extends FrameAxNode>(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
discoveredFrames: CdpFrame[],
|
||||
captured: CapturedViewModel,
|
||||
signal?: AbortSignal,
|
||||
): Promise<FrameAxBatch<T>[]> {
|
||||
let frames = [...discoveredFrames];
|
||||
const capturedFrameIds = [...(captured.frameNodes?.keys() ?? [])];
|
||||
if (frames.length === 0) {
|
||||
const rootFrameId = captured.rootFrameId ?? capturedFrameIds[0] ?? "root";
|
||||
frames = [{ frameId: rootFrameId, target: { tabId } }];
|
||||
}
|
||||
for (const frameId of capturedFrameIds) {
|
||||
if (!frames.some((frame) => frame.frameId === frameId)) {
|
||||
frames.push({ frameId, target: { tabId } });
|
||||
}
|
||||
}
|
||||
|
||||
let firstFailure: unknown;
|
||||
const trees = await Promise.all(
|
||||
frames.map(async (frame): Promise<FrameAxBatch<T>> => {
|
||||
try {
|
||||
throwIfAborted(signal);
|
||||
await sendToCdpTarget(cdp, frame.target, "Accessibility.enable", {});
|
||||
const result = await sendToCdpTarget<{ nodes?: T[] }>(
|
||||
cdp,
|
||||
frame.target,
|
||||
"Accessibility.getFullAXTree",
|
||||
frame.frameId === "root" ? {} : { frameId: frame.frameId },
|
||||
);
|
||||
throwIfAborted(signal);
|
||||
return {
|
||||
frame,
|
||||
nodes: result.nodes ?? [],
|
||||
};
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
firstFailure ??= err;
|
||||
console.debug("[bsk observation] frame AX capture failed", {
|
||||
frameId: frame.frameId,
|
||||
err,
|
||||
});
|
||||
return { frame, nodes: [] };
|
||||
}
|
||||
}),
|
||||
);
|
||||
const capturedNodeCount = [...captured.iframeNodes.values()].reduce(
|
||||
(count, nodes) => count + nodes.length,
|
||||
captured.nodes.length,
|
||||
);
|
||||
if (firstFailure && capturedNodeCount === 0 && trees.every((tree) => tree.nodes.length === 0)) {
|
||||
throw firstFailure;
|
||||
}
|
||||
return trees;
|
||||
}
|
||||
|
||||
export async function captureFrameData<T extends FrameAxNode>(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
captured: CapturedViewModel,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CapturedFrameDocument<T>[]> {
|
||||
const graph = await discoverFrameGraph(cdp, tabId);
|
||||
const frames = graph?.frames ?? [];
|
||||
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal);
|
||||
throwIfAborted(signal);
|
||||
const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
|
||||
return buildFrameDocuments(graph, batches, captured);
|
||||
}
|
||||
@@ -1,173 +0,0 @@
|
||||
import {
|
||||
type CdpFrame,
|
||||
type CdpFrameGraph,
|
||||
type CdpTarget,
|
||||
cdpTargetKey,
|
||||
} from "@/browser-driver/frame-graph";
|
||||
import type { CapturedNode, CapturedViewModel } from "./capture";
|
||||
|
||||
export interface FrameOwnedAxNode {
|
||||
nodeId: string;
|
||||
frameId?: string;
|
||||
parentId?: string;
|
||||
childIds?: string[];
|
||||
backendDOMNodeId?: number;
|
||||
}
|
||||
|
||||
export interface FrameAxBatch<T extends FrameOwnedAxNode> {
|
||||
frame: CdpFrame;
|
||||
nodes: T[];
|
||||
}
|
||||
|
||||
export interface FrameDocument<T extends FrameOwnedAxNode> extends CdpFrame {
|
||||
contextScopeId: string;
|
||||
axNodes: T[];
|
||||
domNodes: CapturedNode[];
|
||||
}
|
||||
|
||||
interface Ownership {
|
||||
frameId: string;
|
||||
strength: number;
|
||||
}
|
||||
|
||||
interface OwnedCandidate<T> {
|
||||
node: T;
|
||||
target: CdpTarget;
|
||||
ownership: Ownership;
|
||||
}
|
||||
|
||||
function targetNodeKey(target: CdpTarget, nodeId: string): string {
|
||||
return `${cdpTargetKey(target)}:${nodeId}`;
|
||||
}
|
||||
|
||||
function targetBackendKey(target: CdpTarget, backendNodeId: number): string {
|
||||
return `${cdpTargetKey(target)}:${backendNodeId}`;
|
||||
}
|
||||
|
||||
function frameList<T extends FrameOwnedAxNode>(
|
||||
graph: CdpFrameGraph | null,
|
||||
batches: FrameAxBatch<T>[],
|
||||
captured: CapturedViewModel,
|
||||
): CdpFrame[] {
|
||||
const frames = new Map<string, CdpFrame>();
|
||||
for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame);
|
||||
for (const batch of batches) {
|
||||
if (!frames.has(batch.frame.frameId)) frames.set(batch.frame.frameId, batch.frame);
|
||||
}
|
||||
for (const frameId of captured.frameNodes?.keys() ?? []) {
|
||||
if (!frames.has(frameId))
|
||||
frames.set(frameId, { frameId, target: { tabId: batches[0]?.frame.target.tabId ?? 0 } });
|
||||
}
|
||||
return [...frames.values()].map((frame) => {
|
||||
const ownerBackendNodeId =
|
||||
frame.ownerBackendNodeId ?? captured.frameOwnerBackendNodeIds?.get(frame.frameId);
|
||||
const parentFrameId = frame.parentFrameId ?? captured.frameParentIds?.get(frame.frameId);
|
||||
return {
|
||||
...frame,
|
||||
...(ownerBackendNodeId !== undefined ? { ownerBackendNodeId } : {}),
|
||||
...(parentFrameId ? { parentFrameId } : {}),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
graph: CdpFrameGraph | null,
|
||||
batches: FrameAxBatch<T>[],
|
||||
captured: CapturedViewModel,
|
||||
): FrameDocument<T>[] {
|
||||
const frames = frameList(graph, batches, captured);
|
||||
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
|
||||
const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId;
|
||||
const domNodesForFrame = (frameId: string): CapturedNode[] =>
|
||||
captured.frameNodes?.get(frameId) ?? (frameId === rootFrameId ? captured.nodes : []);
|
||||
const backendOwner = new Map<string, string>();
|
||||
for (const frame of frames) {
|
||||
for (const node of domNodesForFrame(frame.frameId)) {
|
||||
backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId);
|
||||
}
|
||||
}
|
||||
|
||||
const candidates = new Map<string, OwnedCandidate<T>>();
|
||||
for (const batch of batches) {
|
||||
const nodeById = new Map(batch.nodes.map((node) => [node.nodeId, node]));
|
||||
const ownershipByNodeId = new Map<string, Ownership>();
|
||||
const resolving = new Set<string>();
|
||||
const resolveOwnership = (node: T): Ownership => {
|
||||
const cached = ownershipByNodeId.get(node.nodeId);
|
||||
if (cached) return cached;
|
||||
if (node.frameId && frameById.has(node.frameId)) {
|
||||
const ownership = { frameId: node.frameId, strength: 4 };
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
}
|
||||
if (typeof node.backendDOMNodeId === "number") {
|
||||
const frameId = backendOwner.get(
|
||||
targetBackendKey(batch.frame.target, node.backendDOMNodeId),
|
||||
);
|
||||
if (frameId) {
|
||||
const ownership = { frameId, strength: 3 };
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
}
|
||||
}
|
||||
if (node.parentId && !resolving.has(node.nodeId)) {
|
||||
const parent = nodeById.get(node.parentId);
|
||||
if (parent) {
|
||||
resolving.add(node.nodeId);
|
||||
const inherited = resolveOwnership(parent);
|
||||
resolving.delete(node.nodeId);
|
||||
const ownership = {
|
||||
frameId: inherited.frameId,
|
||||
strength: Math.min(2, inherited.strength),
|
||||
};
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
}
|
||||
}
|
||||
const ownership = { frameId: batch.frame.frameId, strength: 1 };
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
};
|
||||
|
||||
for (const node of batch.nodes) {
|
||||
const ownership = resolveOwnership(node);
|
||||
const key = targetNodeKey(batch.frame.target, node.nodeId);
|
||||
const existing = candidates.get(key);
|
||||
if (!existing || ownership.strength > existing.ownership.strength) {
|
||||
candidates.set(key, { node, target: batch.frame.target, ownership });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const ownershipByTargetNode = new Map(
|
||||
[...candidates].map(([key, candidate]) => [key, candidate.ownership.frameId]),
|
||||
);
|
||||
const axNodesByFrame = new Map<string, T[]>();
|
||||
for (const candidate of candidates.values()) {
|
||||
const { node, target, ownership } = candidate;
|
||||
const parentFrameId = node.parentId
|
||||
? ownershipByTargetNode.get(targetNodeKey(target, node.parentId))
|
||||
: undefined;
|
||||
const childIds = node.childIds?.filter(
|
||||
(childId) => ownershipByTargetNode.get(targetNodeKey(target, childId)) === ownership.frameId,
|
||||
);
|
||||
const ownedNode = {
|
||||
...node,
|
||||
frameId: ownership.frameId,
|
||||
...(parentFrameId === ownership.frameId
|
||||
? { parentId: node.parentId }
|
||||
: { parentId: undefined }),
|
||||
...(childIds ? { childIds } : {}),
|
||||
};
|
||||
const nodes = axNodesByFrame.get(ownership.frameId) ?? [];
|
||||
nodes.push(ownedNode);
|
||||
axNodesByFrame.set(ownership.frameId, nodes);
|
||||
}
|
||||
|
||||
return frames.map((frame) => ({
|
||||
...frame,
|
||||
contextScopeId: frame.frameId,
|
||||
axNodes: axNodesByFrame.get(frame.frameId) ?? [],
|
||||
domNodes: domNodesForFrame(frame.frameId),
|
||||
}));
|
||||
}
|
||||
@@ -1,77 +0,0 @@
|
||||
import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { CapturedSurfaceProbe } from "./capture";
|
||||
import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture";
|
||||
|
||||
/** Frame identity needed to resolve an `@eN` that lives in an iframe. */
|
||||
export interface CaptureVomFrame {
|
||||
frameId: string;
|
||||
target: CdpTarget;
|
||||
parentFrameId?: string;
|
||||
ownerBackendNodeId?: number;
|
||||
}
|
||||
|
||||
/** Allowlisted geometry used to match a recorded action to a rendered ref. */
|
||||
export interface CaptureVomMatchNode {
|
||||
frameId: string;
|
||||
backendNodeId: number;
|
||||
tag: string;
|
||||
/** Top-level viewport-relative geometry. */
|
||||
rect: Rect | null;
|
||||
/** Frame-local viewport-relative geometry, when available. */
|
||||
localRect?: Rect | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Raw AX/DOM trees are never returned. When `redactValues` is enabled by the
|
||||
* caller, form values in `text` are masked as well.
|
||||
*/
|
||||
export interface CaptureVomObservationResult {
|
||||
text: string;
|
||||
refs: RenderedRef[];
|
||||
truncated: boolean;
|
||||
rootFrameId: string;
|
||||
frames: CaptureVomFrame[];
|
||||
matchNodes: CaptureVomMatchNode[];
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
}
|
||||
|
||||
function projectFrames(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomFrame[] {
|
||||
return documents.map((document) => ({
|
||||
frameId: document.frameId,
|
||||
target: document.target,
|
||||
...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}),
|
||||
...(document.ownerBackendNodeId !== undefined
|
||||
? { ownerBackendNodeId: document.ownerBackendNodeId }
|
||||
: {}),
|
||||
}));
|
||||
}
|
||||
|
||||
function projectMatchNodes(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomMatchNode[] {
|
||||
return documents.flatMap((document) =>
|
||||
document.domNodes.map((node) => ({
|
||||
frameId: document.frameId,
|
||||
backendNodeId: node.backendNodeId,
|
||||
tag: node.tag,
|
||||
rect: node.rect,
|
||||
...(node.localRect !== undefined ? { localRect: node.localRect } : {}),
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
export function projectRecordSafeObservation(input: {
|
||||
rootFrameId: string;
|
||||
frameDocuments: CapturedFrameDocument<FrameAxNode>[];
|
||||
rendered: VomResult;
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
}): CaptureVomObservationResult {
|
||||
return {
|
||||
text: input.rendered.text,
|
||||
refs: input.rendered.refs,
|
||||
truncated: input.rendered.truncated,
|
||||
rootFrameId: input.rootFrameId,
|
||||
frames: projectFrames(input.frameDocuments),
|
||||
matchNodes: projectMatchNodes(input.frameDocuments),
|
||||
surfaceProbes: input.surfaceProbes,
|
||||
};
|
||||
}
|
||||
@@ -47,9 +47,7 @@ export async function handleWindowResize(
|
||||
manager: SessionManager,
|
||||
params: WindowResizeParams,
|
||||
api: WindowResizeApi = chromeWindowResizeApi,
|
||||
signal?: AbortSignal,
|
||||
): Promise<WindowResizeResult | RpcError> {
|
||||
if (signal?.aborted) return { code: "cancelled", message: "window_resize aborted" };
|
||||
const ctxOrErr = lookupSession(manager, params, "window_resize");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
@@ -66,7 +64,6 @@ export async function handleWindowResize(
|
||||
}
|
||||
|
||||
try {
|
||||
if (signal?.aborted) return { code: "cancelled", message: "window_resize aborted" };
|
||||
await api.update(ctx.agentWindowId, {
|
||||
width: sizeOrErr.width,
|
||||
height: sizeOrErr.height,
|
||||
|
||||
@@ -65,6 +65,11 @@ function deferredFakeTransport(): { transport: Transport; emit: (frame: Protocol
|
||||
}
|
||||
|
||||
describe("performHandshake", () => {
|
||||
it("advertises the Trace v3 compatibility boundary", () => {
|
||||
expect(PROTOCOL_VERSION).toBe("1.1");
|
||||
expect(MIN_COMPATIBLE_PROTOCOL).toBe("1.1");
|
||||
});
|
||||
|
||||
it("sends system.handshake with identity and both compat fields", async () => {
|
||||
let sentFrame: ProtocolFrame | null = null;
|
||||
const transport = fakeTransport((req) => {
|
||||
@@ -74,9 +79,9 @@ describe("performHandshake", () => {
|
||||
result: {
|
||||
server: "browser-skill-daemon",
|
||||
version: "0.1.0",
|
||||
protocol_version: "1.0",
|
||||
protocol_version: "1.1",
|
||||
min_compatible_peer: "0.0.0",
|
||||
min_compatible_protocol: "1.0",
|
||||
min_compatible_protocol: "1.1",
|
||||
},
|
||||
};
|
||||
});
|
||||
@@ -102,7 +107,7 @@ describe("performHandshake", () => {
|
||||
},
|
||||
});
|
||||
expect(outcome.result.min_compatible_peer).toBe("0.0.0");
|
||||
expect(outcome.result.min_compatible_protocol).toBe("1.0");
|
||||
expect(outcome.result.min_compatible_protocol).toBe("1.1");
|
||||
});
|
||||
|
||||
it("accepts legacy daemon reply with only min_compatible_peer", async () => {
|
||||
@@ -129,8 +134,8 @@ describe("performHandshake", () => {
|
||||
const response = {
|
||||
server: "browser-skill-daemon",
|
||||
version: "0.1.0",
|
||||
protocol_version: "1.0",
|
||||
min_compatible_protocol: "1.0",
|
||||
protocol_version: "1.1",
|
||||
min_compatible_protocol: "1.1",
|
||||
} satisfies HandshakeResult;
|
||||
const transport = fakeTransport((req) => ({
|
||||
id: (req as { id: string }).id,
|
||||
@@ -145,7 +150,7 @@ describe("performHandshake", () => {
|
||||
});
|
||||
|
||||
expect(outcome.result.min_compatible_peer).toBeUndefined();
|
||||
expect(outcome.result.min_compatible_protocol).toBe("1.0");
|
||||
expect(outcome.result.min_compatible_protocol).toBe("1.1");
|
||||
});
|
||||
|
||||
it("rejects when the daemon responds with an error", async () => {
|
||||
@@ -178,9 +183,9 @@ describe("performHandshake", () => {
|
||||
result: {
|
||||
server: "browser-skill-daemon",
|
||||
version: "0.1.0",
|
||||
protocol_version: "1.0",
|
||||
protocol_version: "1.1",
|
||||
min_compatible_peer: "0.0.0",
|
||||
min_compatible_protocol: "1.0",
|
||||
min_compatible_protocol: "1.1",
|
||||
},
|
||||
});
|
||||
emit({
|
||||
@@ -188,14 +193,14 @@ describe("performHandshake", () => {
|
||||
result: {
|
||||
server: "browser-skill-daemon",
|
||||
version: "0.1.0",
|
||||
protocol_version: "1.0",
|
||||
protocol_version: "1.1",
|
||||
min_compatible_peer: "0.0.0",
|
||||
min_compatible_protocol: "1.0",
|
||||
min_compatible_protocol: "1.1",
|
||||
},
|
||||
});
|
||||
|
||||
await expect(pending).resolves.toMatchObject({
|
||||
result: { server: "browser-skill-daemon", protocol_version: "1.0" },
|
||||
result: { server: "browser-skill-daemon", protocol_version: "1.1" },
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ import type {
|
||||
ResponseFrame,
|
||||
} from "./types";
|
||||
|
||||
export const PROTOCOL_VERSION = "1.0";
|
||||
export const PROTOCOL_VERSION = "1.1";
|
||||
/**
|
||||
* Extension semver, injected at build time from `package.json` via
|
||||
* Vite's `define` (see `wxt.config.ts` and `vitest.config.ts`).
|
||||
@@ -18,7 +18,7 @@ export const EXTENSION_VERSION: string =
|
||||
* Lowest **protocol** version this extension accepts (e.g. `"1.0"`).
|
||||
* Must stay in sync with daemon `MIN_COMPATIBLE_PROTOCOL`.
|
||||
*/
|
||||
export const MIN_COMPATIBLE_PROTOCOL = "1.0";
|
||||
export const MIN_COMPATIBLE_PROTOCOL = "1.1";
|
||||
/**
|
||||
* **Deprecated** — legacy app-semver floor for wire compat with old
|
||||
* daemons. New code sends `"0.0.0"`; compat decisions ignore this.
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
// stay structural (interface, not class) so the same JSON parses on both
|
||||
// sides without extra adapters.
|
||||
|
||||
import type { CaptureTargetDescriptor } from "@/lib/describe-target";
|
||||
|
||||
export type RpcId = string;
|
||||
|
||||
export type ErrorCode =
|
||||
@@ -32,8 +34,7 @@ export type RpcErrorReason =
|
||||
| "tab_not_active"
|
||||
| "restricted_tab_url"
|
||||
| "borrow_conflict"
|
||||
| "screenshot_capture_failed"
|
||||
| "cleanup_failed";
|
||||
| "screenshot_capture_failed";
|
||||
|
||||
export interface RpcErrorData {
|
||||
reason?: RpcErrorReason;
|
||||
@@ -657,26 +658,72 @@ export interface EmulateResult {
|
||||
}
|
||||
|
||||
// --------------------------------------------------------------------------
|
||||
// Semantic record payloads — mirror bsk-protocol record.rs (Trace v2)
|
||||
// Semantic record payloads — mirror bsk-protocol record.rs (Trace v3)
|
||||
// --------------------------------------------------------------------------
|
||||
|
||||
export const TRACE_VERSION = 3;
|
||||
export const VOM_FORMAT_VERSION = 1;
|
||||
|
||||
export interface TargetDescriptor {
|
||||
ref?: string;
|
||||
role?: string;
|
||||
name?: string;
|
||||
tag: string;
|
||||
name_attr?: string;
|
||||
placeholder?: string;
|
||||
nearby_label?: string;
|
||||
ctx?: string;
|
||||
unmatched?: boolean;
|
||||
}
|
||||
|
||||
export interface TraceEntry {
|
||||
start_url: string;
|
||||
}
|
||||
|
||||
export interface PageRef {
|
||||
export interface RecorderInfo {
|
||||
bsk: string;
|
||||
vom: number;
|
||||
}
|
||||
|
||||
export type FrameCaptureStatus = "complete" | "partial";
|
||||
|
||||
export type FrameCaptureFailureReason =
|
||||
| "not_injectable"
|
||||
| "arm_failed"
|
||||
| "rearm_failed"
|
||||
| "drain_failed"
|
||||
| "flush_failed";
|
||||
|
||||
export interface FrameCaptureFailure {
|
||||
reason: FrameCaptureFailureReason;
|
||||
frame_id: number;
|
||||
document_id?: string;
|
||||
url?: string;
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
export interface FrameCaptureInfo {
|
||||
status: FrameCaptureStatus;
|
||||
failures?: FrameCaptureFailure[];
|
||||
}
|
||||
|
||||
export type StopReason = "user_finish" | "cli_stop";
|
||||
|
||||
export interface TraceState {
|
||||
id: string;
|
||||
url: string;
|
||||
title?: string;
|
||||
/** Wire-only: full page observation text. */
|
||||
body?: string;
|
||||
/** Disk-only: filename under the bundle `pages/` directory. */
|
||||
page?: string;
|
||||
truncated?: boolean;
|
||||
}
|
||||
|
||||
export interface StepResult {
|
||||
state: string;
|
||||
}
|
||||
|
||||
export interface StepCommon {
|
||||
id: number;
|
||||
state: string;
|
||||
result: StepResult;
|
||||
}
|
||||
|
||||
export interface SelectedOption {
|
||||
@@ -684,66 +731,140 @@ export interface SelectedOption {
|
||||
label?: string;
|
||||
}
|
||||
|
||||
export interface StepEffect {
|
||||
navigated_to: string;
|
||||
export type NavigationCause =
|
||||
| "user_typed"
|
||||
| "link"
|
||||
| "form_submit"
|
||||
| "reload"
|
||||
| "history"
|
||||
| "script"
|
||||
| "browser";
|
||||
|
||||
export type FillCommit = "enter" | "suggestion" | "blur";
|
||||
|
||||
/** Geometry reported by the content script at capture time. */
|
||||
export interface CaptureGeometry {
|
||||
rect: { x: number; y: number; w: number; h: number };
|
||||
scrollX: number;
|
||||
scrollY: number;
|
||||
position: string;
|
||||
tag: string;
|
||||
ownerFrameBackendNodeId?: number | null;
|
||||
}
|
||||
|
||||
export interface StepCommon {
|
||||
id: number;
|
||||
page: string;
|
||||
effect?: StepEffect;
|
||||
}
|
||||
|
||||
/** Capture/buffer draft before v2 reduction. */
|
||||
/** Capture/buffer draft before v3 reduction. */
|
||||
export type DraftTraceStep =
|
||||
| {
|
||||
op: "click";
|
||||
target: TargetDescriptor;
|
||||
captureTarget?: CaptureTargetDescriptor;
|
||||
geometry?: CaptureGeometry;
|
||||
navigated_to?: string;
|
||||
page_url?: string;
|
||||
title?: string;
|
||||
preStateId?: string;
|
||||
postStateId?: string;
|
||||
observationBody?: string;
|
||||
observationHash?: string;
|
||||
truncated?: boolean;
|
||||
}
|
||||
| {
|
||||
op: "hover";
|
||||
target: TargetDescriptor;
|
||||
captureTarget?: CaptureTargetDescriptor;
|
||||
geometry?: CaptureGeometry;
|
||||
page_url?: string;
|
||||
title?: string;
|
||||
preStateId?: string;
|
||||
postStateId?: string;
|
||||
observationBody?: string;
|
||||
observationHash?: string;
|
||||
truncated?: boolean;
|
||||
}
|
||||
| {
|
||||
op: "fill";
|
||||
target: TargetDescriptor;
|
||||
captureTarget?: CaptureTargetDescriptor;
|
||||
geometry?: CaptureGeometry;
|
||||
value: string;
|
||||
commit?: FillCommit;
|
||||
redacted?: boolean;
|
||||
navigated_to?: string;
|
||||
page_url?: string;
|
||||
title?: string;
|
||||
preStateId?: string;
|
||||
postStateId?: string;
|
||||
observationBody?: string;
|
||||
observationHash?: string;
|
||||
truncated?: boolean;
|
||||
}
|
||||
| {
|
||||
op: "press";
|
||||
key: string;
|
||||
target?: TargetDescriptor;
|
||||
captureTarget?: CaptureTargetDescriptor;
|
||||
geometry?: CaptureGeometry;
|
||||
modifiers?: KeyModifier[];
|
||||
navigated_to?: string;
|
||||
page_url?: string;
|
||||
title?: string;
|
||||
preStateId?: string;
|
||||
postStateId?: string;
|
||||
observationBody?: string;
|
||||
observationHash?: string;
|
||||
truncated?: boolean;
|
||||
}
|
||||
| {
|
||||
op: "select";
|
||||
target: TargetDescriptor;
|
||||
captureTarget?: CaptureTargetDescriptor;
|
||||
geometry?: CaptureGeometry;
|
||||
values: string[];
|
||||
labels?: string[];
|
||||
navigated_to?: string;
|
||||
page_url?: string;
|
||||
title?: string;
|
||||
preStateId?: string;
|
||||
postStateId?: string;
|
||||
observationBody?: string;
|
||||
observationHash?: string;
|
||||
truncated?: boolean;
|
||||
}
|
||||
| {
|
||||
op: "scroll";
|
||||
page_url?: string;
|
||||
title?: string;
|
||||
preStateId?: string;
|
||||
postStateId?: string;
|
||||
observationBody?: string;
|
||||
observationHash?: string;
|
||||
truncated?: boolean;
|
||||
}
|
||||
| {
|
||||
op: "navigate";
|
||||
url: string;
|
||||
cause?: NavigationCause;
|
||||
page_url?: string;
|
||||
title?: string;
|
||||
preStateId?: string;
|
||||
postStateId?: string;
|
||||
observationBody?: string;
|
||||
observationHash?: string;
|
||||
truncated?: boolean;
|
||||
/** Raw webNavigation transition metadata for cause mapping. */
|
||||
transitionType?: string;
|
||||
transitionQualifiers?: string[];
|
||||
};
|
||||
|
||||
/** Exported record-only step (trace v2). */
|
||||
/** Exported record-only step (trace v3). */
|
||||
export type Step =
|
||||
| ({ op: "navigate" } & StepCommon & { to: string })
|
||||
| ({ op: "navigate" } & StepCommon & { to: string; cause: NavigationCause })
|
||||
| ({ op: "click" } & StepCommon & { target: TargetDescriptor })
|
||||
| ({ op: "hover" } & StepCommon & { target: TargetDescriptor })
|
||||
| ({ op: "fill" } & StepCommon & {
|
||||
target: TargetDescriptor;
|
||||
value: string;
|
||||
commit: FillCommit;
|
||||
redacted?: boolean;
|
||||
})
|
||||
| ({ op: "select" } & StepCommon & {
|
||||
@@ -754,14 +875,20 @@ export type Step =
|
||||
key: string;
|
||||
modifiers?: KeyModifier[];
|
||||
target?: TargetDescriptor;
|
||||
});
|
||||
})
|
||||
| ({ op: "scroll" } & StepCommon);
|
||||
|
||||
export interface Trace {
|
||||
version: number;
|
||||
recorded_at: string;
|
||||
started_at?: string;
|
||||
purpose?: string;
|
||||
stopped_by: StopReason;
|
||||
entry: TraceEntry;
|
||||
pages: PageRef[];
|
||||
recorder: RecorderInfo;
|
||||
/** Present when one or more frames could not be armed, rearmed, or flushed. */
|
||||
frame_capture?: FrameCaptureInfo;
|
||||
states: TraceState[];
|
||||
steps: Step[];
|
||||
}
|
||||
|
||||
@@ -770,6 +897,8 @@ export interface RecordStartParams {
|
||||
tab_id?: number;
|
||||
url?: string;
|
||||
purpose?: string;
|
||||
max_page_tokens?: number;
|
||||
redact_values?: boolean;
|
||||
}
|
||||
|
||||
export interface RecordStartResult {
|
||||
|
||||
@@ -25,8 +25,10 @@ export default defineConfig({
|
||||
name: "BrowserSkill",
|
||||
description:
|
||||
"Let AI agents use your logged-in browser in a separate Agent Window—without interrupting your work. Powered by the bsk CLI.",
|
||||
// Flat debugger sessions are required to address out-of-process iframes.
|
||||
minimum_chrome_version: "125",
|
||||
// Chrome 116 is the first version where WebSocket activity resets the
|
||||
// service-worker idle timer, which the 20s heartbeat relies on to
|
||||
// keep the daemon link alive.
|
||||
minimum_chrome_version: "116",
|
||||
permissions: [
|
||||
"alarms",
|
||||
"debugger",
|
||||
|
||||
@@ -39,11 +39,6 @@
|
||||
"**",
|
||||
"!**/node_modules/**",
|
||||
"!**/dist/**",
|
||||
"!packages/dsh-plugin-browserskill/lib/**",
|
||||
"!packages/dsh-plugin-browserskill/src/skill-content.generated.ts",
|
||||
"!packages/dsh-plugin-browserskill/src/client/bsk-ui.nomodule.css",
|
||||
"!packages/dsh-plugin-browserskill/.bsk-ui.raw.css",
|
||||
"!packages/dsh-plugin-browserskill/.bsk-ui.input.css",
|
||||
"!**/target/**",
|
||||
"!**/.wxt/**",
|
||||
"!**/.output/**",
|
||||
|
||||
@@ -265,19 +265,27 @@ retry — complete the task autonomously or stop gracefully.
|
||||
|
||||
### Recording — `bsk record`
|
||||
|
||||
Capture the user's own actions in the Agent Window to a `trace.json`, for later LLM-driven automation:
|
||||
Capture the user's own actions in the Agent Window to a **trace v3 bundle**, for later LLM-driven automation:
|
||||
|
||||
```bash
|
||||
bsk record start --browser <instance-id-or-label> [--url https://…] [--purpose "publish a wiki doc"] [--output trace.json]
|
||||
bsk record start --browser <instance-id-or-label> \
|
||||
[--url https://…] [--purpose "publish a wiki doc"] \
|
||||
[--max-page-tokens 3000] [--redact-values] \
|
||||
[--output trace]
|
||||
# `--url` is optional; default https://example.com/ when omitted (must be http(s)).
|
||||
# Blocks until the user clicks Finish in the recording panel, then writes ./trace.json and closes the window.
|
||||
# Blocks until the user clicks Finish in the recording panel, then writes:
|
||||
# trace/trace.json — action chain + state index (version 3)
|
||||
# trace/pages/ — one `.vom.txt` observe snapshot per settled page state
|
||||
|
||||
bsk record stop [--output trace.json] # terminal fallback if the browser panel is unavailable
|
||||
bsk record stop [--output trace] # terminal fallback if the browser panel is unavailable
|
||||
```
|
||||
|
||||
- The trace is a **record-only action log** (a `pages[]` dictionary + `navigate`/`click`/`fill`/`select`/`press` steps with `target` descriptors). It records *what the user did*; deciding which inputs are variable is left to the executing agent.
|
||||
- **Bundle layout:** `--output` is a directory (default `./trace`) containing `trace.json` and `pages/`. `trace.json` lists `states[]` (page observation ids) and `steps[]` (each step binds `state` = observe snapshot *before* the action and `result.state` = snapshot *after* settle). Page bodies live in `pages/sN.vom.txt` using the same VOM format as `bsk observe`.
|
||||
- Each `states[]` entry is one **settled page observation** (captured after recording start, navigation landing, or action settle — not on a timer).
|
||||
- `target.ref` values like `@e12` exist **only inside** the bundle for disambiguation; do **not** copy `@eN` refs into SKILL.md or agent runbooks — use visible names from the observation text instead.
|
||||
- `--purpose` is optional context metadata; it does **not** change what gets captured.
|
||||
- There is **no** `bsk replay` — to redo a flow, read the trace and reuse the existing `session` / `snapshot` / `@eN` / `click` / `fill` tools. Follow **Stop when the goal is met**.
|
||||
- `--redact-values` masks all form values in page files as `[filled]` / `[empty]`.
|
||||
- Recording captures interactions in the **top frame and injectable child frames** (same-origin and cross-origin where the extension can attach). Embedded frames that cannot be injected are omitted; the trace may include `frame_capture.status: "partial"` with failure details.
|
||||
- Do **not** record on banking/SSO/password-manager pages; passwords are redacted but traces may still contain sensitive text.
|
||||
|
||||
## Error handling
|
||||
|
||||
@@ -18,12 +18,6 @@ use crate::daemon::state::PROTOCOL_VERSION;
|
||||
const EXTENSION_STORE_URL: &str =
|
||||
"https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi";
|
||||
|
||||
/// Edge Add-ons listing for the browser-skill extension.
|
||||
const EXTENSION_STORE_URL_EDGE: &str = "https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg";
|
||||
|
||||
/// Store listings highlighted in repair hints, in the order they appear.
|
||||
const EXTENSION_STORE_URLS: [&str; 2] = [EXTENSION_STORE_URL, EXTENSION_STORE_URL_EDGE];
|
||||
|
||||
/// Status of a single doctor check. `Ok` / `Fail` are the legacy two
|
||||
/// states; `NotApplicable` (review M2) is reported as "N/A" in human
|
||||
/// output and as `"status": "na"` in `--json` output, so a check that
|
||||
@@ -426,10 +420,7 @@ fn check_extension_connected(status: Option<&StatusResult>) -> CheckResult {
|
||||
CheckResult::fail(
|
||||
name,
|
||||
"0 browsers connected",
|
||||
format!(
|
||||
"install the extension from {EXTENSION_STORE_URL} (Chrome) \
|
||||
or {EXTENSION_STORE_URL_EDGE} (Edge) and load it in the browser"
|
||||
),
|
||||
format!("install the extension from {EXTENSION_STORE_URL} and load it in Chromium"),
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -437,14 +428,17 @@ fn check_extension_connected(status: Option<&StatusResult>) -> CheckResult {
|
||||
/// Highlight known URLs in repair hints for terminal output. Plain
|
||||
/// text is preserved in `--json` and in stored [`CheckResult::hint`].
|
||||
fn style_hint(hint: &str) -> String {
|
||||
let mut styled = hint.to_string();
|
||||
for url in EXTENSION_STORE_URLS {
|
||||
if !styled.contains(url) {
|
||||
continue;
|
||||
}
|
||||
styled = styled.replace(url, &style(url).cyan().bold().underlined().to_string());
|
||||
if !hint.contains(EXTENSION_STORE_URL) {
|
||||
return hint.to_string();
|
||||
}
|
||||
styled
|
||||
hint.replace(
|
||||
EXTENSION_STORE_URL,
|
||||
&style(EXTENSION_STORE_URL)
|
||||
.cyan()
|
||||
.bold()
|
||||
.underlined()
|
||||
.to_string(),
|
||||
)
|
||||
}
|
||||
|
||||
fn render_human(checks: &[CheckResult]) {
|
||||
@@ -486,7 +480,7 @@ mod m2_tests {
|
||||
fn fake_status(browsers: Vec<BrowserStatusEntry>, skew: Vec<VersionSkewEntry>) -> StatusResult {
|
||||
StatusResult {
|
||||
daemon_version: env!("CARGO_PKG_VERSION").into(),
|
||||
protocol_version: "1.0".into(),
|
||||
protocol_version: "1.1".into(),
|
||||
pid: 1,
|
||||
uptime_secs: 0,
|
||||
ws_port: 0,
|
||||
@@ -510,26 +504,6 @@ mod m2_tests {
|
||||
hint.contains(EXTENSION_STORE_URL),
|
||||
"hint should include Chrome Web Store URL: {hint}"
|
||||
);
|
||||
assert!(
|
||||
hint.contains(EXTENSION_STORE_URL_EDGE),
|
||||
"hint should include Edge Add-ons URL: {hint}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn style_hint_preserves_every_store_url() {
|
||||
let hint = check_extension_connected(Some(&fake_status(Vec::new(), Vec::new())))
|
||||
.hint
|
||||
.expect("extension disconnected should include a hint");
|
||||
let styled = style_hint(&hint);
|
||||
// Whether or not the terminal accepts colors, styling must never drop or
|
||||
// mangle a URL the user has to click.
|
||||
for url in EXTENSION_STORE_URLS {
|
||||
assert!(
|
||||
styled.contains(url),
|
||||
"styled hint should still contain {url}: {styled}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -571,7 +545,7 @@ mod m2_tests {
|
||||
session_count: 0,
|
||||
connected_at_ms: 1,
|
||||
version_skew: false,
|
||||
extension_protocol_version: "1.0".into(),
|
||||
extension_protocol_version: "1.1".into(),
|
||||
}],
|
||||
Vec::new(),
|
||||
);
|
||||
@@ -592,7 +566,7 @@ mod m2_tests {
|
||||
session_count: 0,
|
||||
connected_at_ms: 1,
|
||||
version_skew: true,
|
||||
extension_protocol_version: "1.1".into(),
|
||||
extension_protocol_version: "1.2".into(),
|
||||
}],
|
||||
vec![VersionSkewEntry {
|
||||
instance_id: "alpha".into(),
|
||||
@@ -600,8 +574,8 @@ mod m2_tests {
|
||||
label: "Personal".into(),
|
||||
server_version: env!("CARGO_PKG_VERSION").into(),
|
||||
client_version: "0.0.9".into(),
|
||||
server_protocol_version: "1.0".into(),
|
||||
client_protocol_version: "1.1".into(),
|
||||
server_protocol_version: "1.1".into(),
|
||||
client_protocol_version: "1.2".into(),
|
||||
}],
|
||||
);
|
||||
let check = check_browsers_protocol_compatible(Some(&status));
|
||||
|
||||
@@ -1,15 +1,17 @@
|
||||
//! `bsk record start|stop` — capture user actions in the Agent Window.
|
||||
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
use std::collections::HashSet;
|
||||
use std::fs::{self, OpenOptions};
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Duration;
|
||||
|
||||
use anyhow::Context;
|
||||
use bsk_protocol::Method;
|
||||
use bsk_protocol::tools::{
|
||||
RecordAwaitParams, RecordAwaitResult, RecordStartParams, RecordStartResult, RecordStopParams,
|
||||
RecordStopResult, Trace,
|
||||
FrameCaptureStatus, RecordAwaitParams, RecordAwaitResult, RecordStartParams, RecordStartResult,
|
||||
RecordStopParams, RecordStopResult, TRACE_VERSION, Trace, TraceState,
|
||||
};
|
||||
use bsk_protocol::{ErrorCode, Method};
|
||||
use clap::{Args, Subcommand};
|
||||
|
||||
use crate::cli::TOOL_IPC_TIMEOUT;
|
||||
@@ -18,10 +20,34 @@ use crate::cli::ensure_daemon::ensure_daemon;
|
||||
use crate::cli::error::{CliError, Format};
|
||||
use crate::cli::record_state;
|
||||
use crate::cli::session::{SessionStartOptions, start_session, stop_session};
|
||||
use crate::daemon::ipc::STOP_IN_PROGRESS_REASON;
|
||||
|
||||
/// Max wait for the user to click 结束 in the browser (24 hours).
|
||||
const RECORD_AWAIT_TIMEOUT_MS: u32 = 86_400_000;
|
||||
|
||||
/// Tear down the recording session, tolerating the teardown the other half of
|
||||
/// the pair already started. `record start` blocks in `record_await` and stops
|
||||
/// the session once the trace arrives; the documented `record stop` fallback
|
||||
/// stops it too. Whichever loses that race must not report a failure — the
|
||||
/// trace is already written by then.
|
||||
fn stop_recording_session(sock: PathBuf, session_id: &str) -> Result<(), CliError> {
|
||||
match stop_session(sock, session_id) {
|
||||
Ok(_) => Ok(()),
|
||||
Err(err) if session_teardown_raced(&err) => Ok(()),
|
||||
Err(err) => Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
fn session_teardown_raced(err: &CliError) -> bool {
|
||||
if err.code() == Some(ErrorCode::NotFound) {
|
||||
return true;
|
||||
}
|
||||
err.data()
|
||||
.and_then(|data| data.get("reason"))
|
||||
.and_then(serde_json::Value::as_str)
|
||||
== Some(STOP_IN_PROGRESS_REASON)
|
||||
}
|
||||
|
||||
/// `bsk record …` subcommand tree.
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct RecordCmd {
|
||||
@@ -58,15 +84,25 @@ pub struct RecordStartArgs {
|
||||
#[arg(long)]
|
||||
pub purpose: Option<String>,
|
||||
|
||||
/// Output path for the trace JSON (default `./trace.json`).
|
||||
#[arg(long, default_value = "trace.json")]
|
||||
/// Max VOM tokens per page observation file (default 3000).
|
||||
#[arg(long = "max-page-tokens")]
|
||||
pub max_page_tokens: Option<u32>,
|
||||
|
||||
/// Redact all form values in page observations (`[filled]` only).
|
||||
#[arg(long = "redact-values")]
|
||||
pub redact_values: bool,
|
||||
|
||||
/// Output directory for the trace bundle (default `./trace`).
|
||||
/// Writes `<dir>/trace.json` and `<dir>/pages/*.vom.txt`.
|
||||
#[arg(long, default_value = "trace")]
|
||||
pub output: PathBuf,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Args)]
|
||||
pub struct RecordStopArgs {
|
||||
/// Output path for the trace JSON (default `./trace.json`).
|
||||
#[arg(long, default_value = "trace.json")]
|
||||
/// Output directory for the trace bundle (default `./trace`).
|
||||
/// Writes `<dir>/trace.json` and `<dir>/pages/*.vom.txt`.
|
||||
#[arg(long, default_value = "trace")]
|
||||
pub output: PathBuf,
|
||||
}
|
||||
|
||||
@@ -98,6 +134,8 @@ fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError>
|
||||
tab_id: args.tab_id,
|
||||
url: args.url,
|
||||
purpose: args.purpose.clone(),
|
||||
max_page_tokens: args.max_page_tokens,
|
||||
redact_values: Some(args.redact_values),
|
||||
};
|
||||
let start_result = business_rpc::call::<RecordStartParams, RecordStartResult>(
|
||||
info.sock_path.clone(),
|
||||
@@ -142,13 +180,13 @@ fn dispatch_start(args: RecordStartArgs, format: Format) -> Result<(), CliError>
|
||||
// Keep write/render inside a Result so `?` cannot skip session teardown.
|
||||
let run_result: Result<(), CliError> = match await_result {
|
||||
Ok(result) => (|| {
|
||||
write_trace_file(&args.output, &result.trace)?;
|
||||
render_finish(&result.trace, &args.output, format)
|
||||
let pages_dir = write_trace_bundle(&args.output, &result.trace)?;
|
||||
render_finish(&result.trace, &args.output, &pages_dir, format)
|
||||
})(),
|
||||
Err(err) => Err(err),
|
||||
};
|
||||
|
||||
let session_stop_result = stop_session(info.sock_path, &session.session_id);
|
||||
let session_stop_result = stop_recording_session(info.sock_path, &session.session_id);
|
||||
record_state::clear();
|
||||
|
||||
run_result?;
|
||||
@@ -161,22 +199,23 @@ fn dispatch_stop(args: RecordStopArgs, format: Format) -> Result<(), CliError> {
|
||||
let info = ensure_daemon().context("ensure daemon is running")?;
|
||||
let session_id = state.session_id.clone();
|
||||
|
||||
let params = RecordStopParams {
|
||||
session_id: session_id.clone(),
|
||||
};
|
||||
let result = business_rpc::call::<RecordStopParams, RecordStopResult>(
|
||||
info.sock_path.clone(),
|
||||
"record-stop",
|
||||
Method::ToolRecordStop,
|
||||
Some(params),
|
||||
TOOL_IPC_TIMEOUT,
|
||||
)?;
|
||||
|
||||
let run_result: Result<(), CliError> = (|| {
|
||||
let params = RecordStopParams {
|
||||
session_id: session_id.clone(),
|
||||
};
|
||||
let result = business_rpc::call::<RecordStopParams, RecordStopResult>(
|
||||
info.sock_path.clone(),
|
||||
"record-stop",
|
||||
Method::ToolRecordStop,
|
||||
Some(params),
|
||||
TOOL_IPC_TIMEOUT,
|
||||
)?;
|
||||
write_trace_file(&args.output, &result.trace)?;
|
||||
render_stop(&result, &args.output, format)
|
||||
let pages_dir = write_trace_bundle(&args.output, &result.trace)?;
|
||||
render_stop(&result, &args.output, &pages_dir, format)
|
||||
})();
|
||||
|
||||
let session_stop_result = stop_session(info.sock_path, &session_id);
|
||||
let session_stop_result = stop_recording_session(info.sock_path, &session_id);
|
||||
record_state::clear();
|
||||
|
||||
run_result?;
|
||||
@@ -190,30 +229,313 @@ fn record_await_ipc_timeout(timeout_ms: u32) -> Duration {
|
||||
.unwrap_or(Duration::from_secs(u64::from(timeout_ms / 1_000) + 15))
|
||||
}
|
||||
|
||||
fn write_trace_file(output: &PathBuf, trace: &Trace) -> Result<(), CliError> {
|
||||
let json = serde_json::to_string_pretty(trace)
|
||||
.context("serialize trace JSON")
|
||||
.map_err(CliError::Local)?;
|
||||
if let Some(parent) = output.parent() {
|
||||
if !parent.as_os_str().is_empty() {
|
||||
fs::create_dir_all(parent)
|
||||
.with_context(|| format!("create output directory {}", parent.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
fn pages_dir_for_output(output_dir: &Path) -> PathBuf {
|
||||
output_dir.join("pages")
|
||||
}
|
||||
|
||||
fn trace_json_path(output_dir: &Path) -> PathBuf {
|
||||
output_dir.join("trace.json")
|
||||
}
|
||||
|
||||
fn is_canonical_state_id(id: &str) -> bool {
|
||||
let Some(number) = id.strip_prefix('s') else {
|
||||
return false;
|
||||
};
|
||||
let mut digits = number.bytes();
|
||||
matches!(digits.next(), Some(b'1'..=b'9')) && digits.all(|byte| byte.is_ascii_digit())
|
||||
}
|
||||
|
||||
fn validate_bundle_directory(path: &Path, description: &str) -> Result<(), CliError> {
|
||||
match fs::symlink_metadata(path) {
|
||||
Ok(metadata) if metadata.file_type().is_symlink() => Err(CliError::Local(anyhow::anyhow!(
|
||||
"{description} {} must not be a symlink",
|
||||
path.display()
|
||||
))),
|
||||
Ok(metadata) if metadata.is_dir() => Ok(()),
|
||||
Ok(_) => Err(CliError::Local(anyhow::anyhow!(
|
||||
"{description} {} is not a directory",
|
||||
path.display()
|
||||
))),
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
|
||||
Err(err) => Err(CliError::Local(
|
||||
anyhow::Error::new(err).context(format!("inspect {description} {}", path.display())),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_replaceable_file(path: &Path, description: &str) -> Result<(), CliError> {
|
||||
match fs::symlink_metadata(path) {
|
||||
Ok(metadata) if metadata.file_type().is_file() => Ok(()),
|
||||
Ok(_) => Err(CliError::Local(anyhow::anyhow!(
|
||||
"{description} {} is not a regular file",
|
||||
path.display()
|
||||
))),
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => Ok(()),
|
||||
Err(err) => Err(CliError::Local(
|
||||
anyhow::Error::new(err).context(format!("inspect {description} {}", path.display())),
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
fn commit_staged_file(staged: &Path, target: &Path, backup: &Path) -> io::Result<Option<PathBuf>> {
|
||||
let old_file = match fs::symlink_metadata(target) {
|
||||
Ok(_) => {
|
||||
fs::rename(target, backup)?;
|
||||
Some(backup.to_path_buf())
|
||||
}
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => None,
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
|
||||
if let Err(err) = fs::rename(staged, target) {
|
||||
if let Some(old_file) = old_file.as_deref() {
|
||||
if let Err(restore_err) = fs::rename(old_file, target) {
|
||||
return Err(io::Error::new(
|
||||
restore_err.kind(),
|
||||
format!(
|
||||
"{err}; additionally failed to restore {}: {restore_err}",
|
||||
target.display()
|
||||
),
|
||||
));
|
||||
}
|
||||
}
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
Ok(old_file)
|
||||
}
|
||||
|
||||
fn rollback_installed_files(installed: &[(PathBuf, Option<PathBuf>)]) -> io::Result<()> {
|
||||
let mut rollback_error = None;
|
||||
for (target, backup) in installed.iter().rev() {
|
||||
match fs::remove_file(target) {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(err) => {
|
||||
rollback_error.get_or_insert(err);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if let Some(backup) = backup {
|
||||
if let Err(err) = fs::rename(backup, target) {
|
||||
rollback_error.get_or_insert(err);
|
||||
}
|
||||
}
|
||||
}
|
||||
match rollback_error {
|
||||
Some(err) => Err(err),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Drop page files left by an earlier, longer recording without deleting the
|
||||
/// directory itself: `record start` and the `record stop` fallback export the
|
||||
/// same recording concurrently, and a `remove_dir_all` here would fail the
|
||||
/// writer whose files the other one just erased.
|
||||
fn prune_stale_pages(pages_dir: &Path, keep: &HashSet<String>) -> Result<(), CliError> {
|
||||
let entries = match fs::read_dir(pages_dir) {
|
||||
Ok(entries) => entries,
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => return Ok(()),
|
||||
Err(err) => {
|
||||
return Err(CliError::Local(
|
||||
anyhow::Error::new(err).context(format!("read pages dir {}", pages_dir.display())),
|
||||
));
|
||||
}
|
||||
};
|
||||
for entry in entries.flatten() {
|
||||
let name = entry.file_name();
|
||||
let Some(name) = name.to_str() else { continue };
|
||||
if !name.ends_with(".vom.txt") || keep.contains(name) {
|
||||
continue;
|
||||
}
|
||||
match fs::remove_file(entry.path()) {
|
||||
Ok(()) => {}
|
||||
Err(err) if err.kind() == io::ErrorKind::NotFound => {}
|
||||
Err(err) => {
|
||||
return Err(CliError::Local(
|
||||
anyhow::Error::new(err)
|
||||
.context(format!("remove stale page {}", entry.path().display())),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
fs::write(output, format!("{json}\n"))
|
||||
.with_context(|| format!("write trace to {}", output.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn render_finish(trace: &Trace, output: &PathBuf, format: Format) -> Result<(), CliError> {
|
||||
fn write_trace_bundle(output_dir: &Path, trace: &Trace) -> Result<PathBuf, CliError> {
|
||||
let pages_dir = pages_dir_for_output(output_dir);
|
||||
let trace_path = trace_json_path(output_dir);
|
||||
|
||||
if trace.version != TRACE_VERSION {
|
||||
return Err(CliError::Local(anyhow::anyhow!(
|
||||
"trace version {} is not supported (expected {})",
|
||||
trace.version,
|
||||
TRACE_VERSION
|
||||
)));
|
||||
}
|
||||
|
||||
let mut state_ids = HashSet::with_capacity(trace.states.len());
|
||||
for state in &trace.states {
|
||||
if !is_canonical_state_id(&state.id) {
|
||||
return Err(CliError::Local(anyhow::anyhow!(
|
||||
"state id {:?} is not canonical (expected s1, s2, ...)",
|
||||
state.id
|
||||
)));
|
||||
}
|
||||
if !state_ids.insert(state.id.as_str()) {
|
||||
return Err(CliError::Local(anyhow::anyhow!(
|
||||
"duplicate state id {:?}",
|
||||
state.id
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
let mut disk_states: Vec<TraceState> = Vec::with_capacity(trace.states.len());
|
||||
let mut page_writes = Vec::new();
|
||||
let mut keep = HashSet::new();
|
||||
for state in &trace.states {
|
||||
let mut disk = state.clone();
|
||||
if let Some(body) = state.body.as_deref() {
|
||||
let filename = format!("{}.vom.txt", state.id);
|
||||
let page_path = pages_dir.join(&filename);
|
||||
if page_path.parent() != Some(pages_dir.as_path()) {
|
||||
return Err(CliError::Local(anyhow::anyhow!(
|
||||
"page path {} escapes pages directory {}",
|
||||
page_path.display(),
|
||||
pages_dir.display()
|
||||
)));
|
||||
}
|
||||
keep.insert(filename.clone());
|
||||
page_writes.push((filename.clone(), body));
|
||||
disk.body = None;
|
||||
disk.page = Some(filename);
|
||||
}
|
||||
disk_states.push(disk);
|
||||
}
|
||||
|
||||
let mut disk_trace = trace.clone();
|
||||
disk_trace.states = disk_states;
|
||||
let json = serde_json::to_string_pretty(&disk_trace)
|
||||
.context("serialize trace JSON")
|
||||
.map_err(CliError::Local)?;
|
||||
|
||||
validate_bundle_directory(output_dir, "output bundle directory")?;
|
||||
fs::create_dir_all(output_dir)
|
||||
.with_context(|| format!("create output dir {}", output_dir.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
|
||||
validate_bundle_directory(&pages_dir, "pages directory")?;
|
||||
|
||||
// `record start` and `record stop` can export the same recording at once.
|
||||
// Serialize writers without replacing the shared directory underneath one
|
||||
// another; the lock is released when this file handle is dropped.
|
||||
let lock_path = output_dir.join(".bsk-record-export.lock");
|
||||
let lock_file = OpenOptions::new()
|
||||
.read(true)
|
||||
.write(true)
|
||||
.create(true)
|
||||
.truncate(false)
|
||||
.open(&lock_path)
|
||||
.with_context(|| format!("open export lock {}", lock_path.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
fs2::FileExt::lock_exclusive(&lock_file)
|
||||
.with_context(|| format!("lock export bundle {}", output_dir.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
|
||||
fs::create_dir_all(&pages_dir)
|
||||
.with_context(|| format!("create pages dir {}", pages_dir.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
|
||||
for (filename, _) in &page_writes {
|
||||
validate_replaceable_file(&pages_dir.join(filename), "page observation")?;
|
||||
}
|
||||
validate_replaceable_file(&trace_path, "trace JSON")?;
|
||||
|
||||
let transaction_id = uuid::Uuid::new_v4();
|
||||
let staging_dir = output_dir.join(format!(".bsk-record-stage-{transaction_id}"));
|
||||
let staging_pages = staging_dir.join("pages");
|
||||
fs::create_dir_all(&staging_pages)
|
||||
.with_context(|| format!("create staging dir {}", staging_pages.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
|
||||
let stage_result: Result<(), CliError> = (|| {
|
||||
for (filename, body) in &page_writes {
|
||||
let staged_path = staging_pages.join(filename);
|
||||
fs::write(&staged_path, body)
|
||||
.with_context(|| format!("write staged page observation {}", staged_path.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
}
|
||||
let staged_trace = staging_dir.join("trace.json");
|
||||
fs::write(&staged_trace, format!("{json}\n"))
|
||||
.with_context(|| format!("write staged trace to {}", staged_trace.display()))
|
||||
.map_err(CliError::Local)?;
|
||||
Ok(())
|
||||
})();
|
||||
if let Err(err) = stage_result {
|
||||
let _ = fs::remove_dir_all(&staging_dir);
|
||||
return Err(err);
|
||||
}
|
||||
|
||||
let mut replacements: Vec<(PathBuf, PathBuf, PathBuf)> =
|
||||
Vec::with_capacity(page_writes.len() + 1);
|
||||
for (filename, _) in &page_writes {
|
||||
replacements.push((
|
||||
staging_pages.join(filename),
|
||||
pages_dir.join(filename),
|
||||
pages_dir.join(format!(".{filename}.{transaction_id}.bak")),
|
||||
));
|
||||
}
|
||||
replacements.push((
|
||||
staging_dir.join("trace.json"),
|
||||
trace_path.clone(),
|
||||
output_dir.join(format!(".trace.json.{transaction_id}.bak")),
|
||||
));
|
||||
|
||||
let mut installed = Vec::with_capacity(replacements.len());
|
||||
for (staged, target, backup) in replacements {
|
||||
match commit_staged_file(&staged, &target, &backup) {
|
||||
Ok(old_file) => installed.push((target, old_file)),
|
||||
Err(err) => {
|
||||
let rollback_result = rollback_installed_files(&installed);
|
||||
let _ = fs::remove_dir_all(&staging_dir);
|
||||
let message = match rollback_result {
|
||||
Ok(()) => format!("commit replacement {}: {err}", target.display()),
|
||||
Err(rollback_err) => format!(
|
||||
"commit replacement {}: {err}; rollback also failed: {rollback_err}",
|
||||
target.display()
|
||||
),
|
||||
};
|
||||
return Err(CliError::Local(anyhow::anyhow!(message)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (_, backup) in &installed {
|
||||
if let Some(backup) = backup {
|
||||
let _ = fs::remove_file(backup);
|
||||
}
|
||||
}
|
||||
let _ = fs::remove_dir_all(&staging_dir);
|
||||
prune_stale_pages(&pages_dir, &keep)?;
|
||||
|
||||
Ok(pages_dir)
|
||||
}
|
||||
|
||||
fn render_finish(
|
||||
trace: &Trace,
|
||||
output_dir: &PathBuf,
|
||||
pages_dir: &PathBuf,
|
||||
format: Format,
|
||||
) -> Result<(), CliError> {
|
||||
let trace_path = trace_json_path(output_dir);
|
||||
match format {
|
||||
Format::Json => {
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string(&serde_json::json!({
|
||||
"output": output,
|
||||
"output": output_dir,
|
||||
"trace_json": trace_path,
|
||||
"pages_dir": pages_dir,
|
||||
"trace": trace,
|
||||
"window_closed": true,
|
||||
}))
|
||||
@@ -221,7 +543,20 @@ fn render_finish(trace: &Trace, output: &PathBuf, format: Format) -> Result<(),
|
||||
);
|
||||
}
|
||||
Format::Human => {
|
||||
println!("saved {} steps to {}", trace.steps.len(), output.display());
|
||||
println!(
|
||||
"saved {} steps and {} states to {} (+ {})",
|
||||
trace.steps.len(),
|
||||
trace.states.len(),
|
||||
trace_path.display(),
|
||||
pages_dir.display()
|
||||
);
|
||||
if let Some(frame_capture) = &trace.frame_capture {
|
||||
if matches!(frame_capture.status, FrameCaptureStatus::Partial) {
|
||||
eprintln!(
|
||||
"warning: trace is partial — some embedded frames were not recorded (see frame_capture.failures in trace.json)"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
@@ -230,33 +565,252 @@ fn render_finish(trace: &Trace, output: &PathBuf, format: Format) -> Result<(),
|
||||
fn render_stop(
|
||||
result: &RecordStopResult,
|
||||
output: &PathBuf,
|
||||
pages_dir: &PathBuf,
|
||||
format: Format,
|
||||
) -> Result<(), CliError> {
|
||||
render_finish(&result.trace, output, format)
|
||||
render_finish(&result.trace, output, pages_dir, format)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use bsk_protocol::tools::{
|
||||
NavigationCause, RecorderInfo, Step, StepCommon, StepResult, StopReason, TraceEntry,
|
||||
VOM_FORMAT_VERSION,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn default_output_is_trace_json() {
|
||||
let args = RecordStopArgs {
|
||||
output: PathBuf::from("trace.json"),
|
||||
};
|
||||
assert_eq!(args.output, PathBuf::from("trace.json"));
|
||||
fn sample_trace(state_id: &str, body: &str) -> Trace {
|
||||
Trace {
|
||||
version: TRACE_VERSION,
|
||||
recorded_at: "2026-08-10T02:12:55.360Z".into(),
|
||||
stopped_by: StopReason::UserFinish,
|
||||
entry: TraceEntry {
|
||||
start_url: "https://example.com/".into(),
|
||||
},
|
||||
recorder: RecorderInfo {
|
||||
bsk: "0.1.10".into(),
|
||||
vom: VOM_FORMAT_VERSION,
|
||||
},
|
||||
frame_capture: None,
|
||||
states: vec![TraceState {
|
||||
id: state_id.into(),
|
||||
url: "https://example.com/".into(),
|
||||
title: Some("Example".into()),
|
||||
body: Some(body.into()),
|
||||
page: None,
|
||||
truncated: false,
|
||||
}],
|
||||
steps: vec![Step::Navigate {
|
||||
common: StepCommon {
|
||||
id: 1,
|
||||
state: state_id.into(),
|
||||
result: StepResult {
|
||||
state: state_id.into(),
|
||||
},
|
||||
},
|
||||
to: "https://example.com/".into(),
|
||||
cause: NavigationCause::UserTyped,
|
||||
}],
|
||||
purpose: None,
|
||||
started_at: None,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn start_args_default_output_is_trace_json() {
|
||||
fn concurrent_teardown_is_not_a_record_failure() {
|
||||
let stopping = CliError::from_rpc(bsk_protocol::RpcError {
|
||||
code: ErrorCode::Timeout,
|
||||
message: "session stop is already in progress".into(),
|
||||
data: Some(serde_json::json!({ "reason": STOP_IN_PROGRESS_REASON })),
|
||||
});
|
||||
assert!(session_teardown_raced(&stopping));
|
||||
|
||||
let gone = CliError::from_rpc(bsk_protocol::RpcError {
|
||||
code: ErrorCode::NotFound,
|
||||
message: "session is not registered".into(),
|
||||
data: None,
|
||||
});
|
||||
assert!(session_teardown_raced(&gone));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unrelated_stop_timeout_still_fails() {
|
||||
let stalled = CliError::from_rpc(bsk_protocol::RpcError {
|
||||
code: ErrorCode::Timeout,
|
||||
message: "tool.session_stop timed out".into(),
|
||||
data: None,
|
||||
});
|
||||
assert!(!session_teardown_raced(&stalled));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn default_output_is_trace_dir() {
|
||||
let args = RecordStopArgs {
|
||||
output: PathBuf::from("trace"),
|
||||
};
|
||||
assert_eq!(args.output, PathBuf::from("trace"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn start_args_default_output_is_trace_dir() {
|
||||
let args = RecordStartArgs {
|
||||
browser: None,
|
||||
tab_id: None,
|
||||
url: None,
|
||||
purpose: None,
|
||||
output: PathBuf::from("trace.json"),
|
||||
max_page_tokens: None,
|
||||
redact_values: false,
|
||||
output: PathBuf::from("trace"),
|
||||
};
|
||||
assert_eq!(args.output, PathBuf::from("trace.json"));
|
||||
assert_eq!(args.output, PathBuf::from("trace"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_trace_bundle_writes_dir_layout() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let output = dir.path().join("flow");
|
||||
let trace = sample_trace("s1", "@vom 1\nL1 page");
|
||||
let pages_dir = write_trace_bundle(&output, &trace).unwrap();
|
||||
let trace_path = output.join("trace.json");
|
||||
assert!(trace_path.exists());
|
||||
assert_eq!(pages_dir, output.join("pages"));
|
||||
assert!(pages_dir.join("s1.vom.txt").exists());
|
||||
let disk: serde_json::Value =
|
||||
serde_json::from_str(&fs::read_to_string(&trace_path).unwrap()).unwrap();
|
||||
assert_eq!(disk["states"][0]["page"], "s1.vom.txt");
|
||||
assert!(disk["states"][0].get("body").is_none());
|
||||
assert_eq!(
|
||||
fs::read_to_string(pages_dir.join("s1.vom.txt")).unwrap(),
|
||||
"@vom 1\nL1 page"
|
||||
);
|
||||
|
||||
// Re-exporting drops pages from the previous run but keeps the
|
||||
// directory in place for the concurrent writer.
|
||||
fs::write(pages_dir.join("s9.vom.txt"), "stale").unwrap();
|
||||
fs::write(pages_dir.join("notes.md"), "keep me").unwrap();
|
||||
write_trace_bundle(&output, &trace).unwrap();
|
||||
assert!(!pages_dir.join("s9.vom.txt").exists());
|
||||
assert!(pages_dir.join("s1.vom.txt").exists());
|
||||
assert!(pages_dir.join("notes.md").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_trace_bundle_rejects_traversal_state_id() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let output = dir.path().join("flow");
|
||||
let trace = sample_trace("../escaped", "@vom 1\nL1 page");
|
||||
|
||||
let err = write_trace_bundle(&output, &trace).unwrap_err();
|
||||
|
||||
assert!(err.to_string().contains("state id"));
|
||||
assert!(!output.join("escaped.vom.txt").exists());
|
||||
assert!(!output.join("trace.json").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn write_trace_bundle_rejects_duplicate_state_id() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let output = dir.path().join("flow");
|
||||
let mut trace = sample_trace("s1", "@vom 1\nL1 first");
|
||||
trace.states.push(TraceState {
|
||||
body: Some("@vom 1\nL1 duplicate".into()),
|
||||
..trace.states[0].clone()
|
||||
});
|
||||
|
||||
let err = write_trace_bundle(&output, &trace).unwrap_err();
|
||||
|
||||
assert!(err.to_string().contains("duplicate state id"));
|
||||
assert!(!output.exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsupported_version_preserves_existing_bundle() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let output = dir.path().join("flow");
|
||||
write_trace_bundle(&output, &sample_trace("s1", "original")).unwrap();
|
||||
let original_json = fs::read(output.join("trace.json")).unwrap();
|
||||
|
||||
let mut replacement = sample_trace("s2", "replacement");
|
||||
replacement.version += 1;
|
||||
let err = write_trace_bundle(&output, &replacement).unwrap_err();
|
||||
|
||||
assert!(err.to_string().contains("not supported"));
|
||||
assert_eq!(fs::read(output.join("trace.json")).unwrap(), original_json);
|
||||
assert_eq!(
|
||||
fs::read_to_string(output.join("pages/s1.vom.txt")).unwrap(),
|
||||
"original"
|
||||
);
|
||||
assert!(!output.join("pages/s2.vom.txt").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn page_replacement_failure_preserves_existing_bundle() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let output = dir.path().join("flow");
|
||||
write_trace_bundle(&output, &sample_trace("s1", "original")).unwrap();
|
||||
let original_json = fs::read(output.join("trace.json")).unwrap();
|
||||
fs::write(output.join("pages/notes.md"), "keep me").unwrap();
|
||||
fs::create_dir(output.join("pages/s2.vom.txt")).unwrap();
|
||||
|
||||
let err = write_trace_bundle(&output, &sample_trace("s2", "replacement")).unwrap_err();
|
||||
|
||||
assert!(err.to_string().contains("page"));
|
||||
assert_eq!(fs::read(output.join("trace.json")).unwrap(), original_json);
|
||||
assert_eq!(
|
||||
fs::read_to_string(output.join("pages/s1.vom.txt")).unwrap(),
|
||||
"original"
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read_to_string(output.join("pages/notes.md")).unwrap(),
|
||||
"keep me"
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn write_trace_bundle_rejects_symlinked_pages_directory() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let output = dir.path().join("flow");
|
||||
let external_pages = dir.path().join("external-pages");
|
||||
fs::create_dir_all(&output).unwrap();
|
||||
fs::create_dir_all(&external_pages).unwrap();
|
||||
fs::write(external_pages.join("s9.vom.txt"), "external").unwrap();
|
||||
symlink(&external_pages, output.join("pages")).unwrap();
|
||||
|
||||
let err = write_trace_bundle(&output, &sample_trace("s1", "replacement")).unwrap_err();
|
||||
|
||||
assert!(err.to_string().contains("symlink"));
|
||||
assert_eq!(
|
||||
fs::read_to_string(external_pages.join("s9.vom.txt")).unwrap(),
|
||||
"external"
|
||||
);
|
||||
assert!(!external_pages.join("s1.vom.txt").exists());
|
||||
assert!(!output.join("trace.json").exists());
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn write_trace_bundle_rejects_symlinked_output_directory() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
let external_output = dir.path().join("external-output");
|
||||
let output = dir.path().join("flow");
|
||||
fs::create_dir_all(&external_output).unwrap();
|
||||
fs::write(external_output.join("notes.md"), "external").unwrap();
|
||||
symlink(&external_output, &output).unwrap();
|
||||
|
||||
let err = write_trace_bundle(&output, &sample_trace("s1", "replacement")).unwrap_err();
|
||||
|
||||
assert!(err.to_string().contains("symlink"));
|
||||
assert_eq!(
|
||||
fs::read_to_string(external_output.join("notes.md")).unwrap(),
|
||||
"external"
|
||||
);
|
||||
assert!(!external_output.join("trace.json").exists());
|
||||
assert!(!external_output.join("pages").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -141,9 +141,9 @@ pub fn info_for(code: ErrorCode) -> RenderInfo {
|
||||
exit_code: 2,
|
||||
},
|
||||
ErrorCode::VersionTooOld => RenderInfo {
|
||||
summary: "peer version is too old to communicate with this build",
|
||||
summary: "the installed bsk CLI and BrowserSkill extension are incompatible",
|
||||
hint: Some(
|
||||
"upgrade both bsk CLI and the browser-skill extension so both sides satisfy `min_compatible_protocol`",
|
||||
"update bsk and the BrowserSkill extension, reload the extension, then run `bsk daemon restart`",
|
||||
),
|
||||
exit_code: 5,
|
||||
},
|
||||
@@ -157,7 +157,7 @@ pub fn info_for(code: ErrorCode) -> RenderInfo {
|
||||
ErrorCode::NoBrowserConnected => RenderInfo {
|
||||
summary: "no browser is connected to the daemon",
|
||||
hint: Some(
|
||||
"open the browser-skill extension in your browser and wait for the popup to show \"connected\"",
|
||||
"open the BrowserSkill extension and wait for it to show \"connected\"; if the extension is already open, update and reload it, then run `bsk daemon restart`",
|
||||
),
|
||||
exit_code: 1,
|
||||
},
|
||||
@@ -310,6 +310,30 @@ mod tests {
|
||||
assert_eq!(exit_code_for(ErrorCode::VersionTooOld), 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_error_names_the_components_and_gives_update_steps() {
|
||||
let info = info_for(ErrorCode::VersionTooOld);
|
||||
assert!(info.summary.contains("bsk CLI"));
|
||||
assert!(info.summary.contains("BrowserSkill extension"));
|
||||
let hint = info
|
||||
.hint
|
||||
.expect("version mismatch should have a repair hint");
|
||||
assert!(hint.contains("update bsk"));
|
||||
assert!(hint.contains("reload the extension"));
|
||||
assert!(hint.contains("bsk daemon restart"));
|
||||
assert!(!hint.contains("min_compatible_protocol"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_browser_hint_explains_version_mismatch_recovery() {
|
||||
let hint = info_for(ErrorCode::NoBrowserConnected)
|
||||
.hint
|
||||
.expect("no browser should have a repair hint");
|
||||
assert!(hint.contains("extension is already open"));
|
||||
assert!(hint.contains("update"));
|
||||
assert!(hint.contains("bsk daemon restart"));
|
||||
}
|
||||
|
||||
/// Every code must carry a non-empty summary; an empty `summary`
|
||||
/// would render `error: ` with nothing after it.
|
||||
#[test]
|
||||
|
||||
@@ -206,15 +206,15 @@ pub fn full_handler(status: DaemonStatus, state: Arc<DaemonState>) -> RpcHandler
|
||||
Ok(v) => ResponseBody::Ok(v),
|
||||
Err(e) => ResponseBody::Err(e),
|
||||
},
|
||||
Method::SessionStart => match handle_session_start(&state, rpc_id, params).await {
|
||||
Method::SessionStart => match handle_session_start(&state, params).await {
|
||||
Ok(v) => ResponseBody::Ok(v),
|
||||
Err(e) => ResponseBody::Err(e),
|
||||
},
|
||||
Method::SessionStop => match handle_session_stop(&state, rpc_id, params).await {
|
||||
Method::SessionStop => match handle_session_stop(&state, params).await {
|
||||
Ok(v) => ResponseBody::Ok(v),
|
||||
Err(e) => ResponseBody::Err(e),
|
||||
},
|
||||
Method::SessionStopAll => match handle_session_stop_all_rpc(&state, rpc_id).await {
|
||||
Method::SessionStopAll => match handle_session_stop_all(&state).await {
|
||||
Ok(v) => ResponseBody::Ok(v),
|
||||
Err(e) => ResponseBody::Err(e),
|
||||
},
|
||||
@@ -434,9 +434,8 @@ async fn handle_wait_ms(
|
||||
/// cancellation surfaces (M10.2 + review C2):
|
||||
///
|
||||
/// 1. [`AbortRegistry`] — answers daemon-local cancellable runners
|
||||
/// (`tool.wait_ms` and `session.*` lifecycle calls). If a token is
|
||||
/// registered we trip it and stop. Lifecycle handlers forward their
|
||||
/// own WS cancel after preserving request-before-cancel ordering.
|
||||
/// (currently `tool.wait_ms`). If a token is registered we trip it
|
||||
/// and stop.
|
||||
/// 2. [`super::inflight::ToolInflightRegistry`] — every IPC-tracked
|
||||
/// `tool.*` RPC, registered the moment the IPC handler accepts the
|
||||
/// request. Trips the entry's cancel token regardless of whether
|
||||
@@ -444,10 +443,10 @@ async fn handle_wait_ms(
|
||||
/// * **Queued** — the worker's pre-flight observes the cancelled
|
||||
/// token and short-circuits with `cancelled` before any WS
|
||||
/// frame leaves the daemon (review C2 fix).
|
||||
/// * **Forwarded** — we push a WS-side `cancel { rpc_id: ws_rpc_id }`
|
||||
/// frame so the extension's dispatcher can trip its
|
||||
/// `AbortController`; the worker keeps the session busy until the
|
||||
/// extension returns its final result or the cleanup timeout expires.
|
||||
/// * **Forwarded** — the worker's `tokio::select!` returns
|
||||
/// `cancelled` immediately, AND we additionally push a WS-side
|
||||
/// `cancel { rpc_id: ws_rpc_id }` frame so the extension's
|
||||
/// dispatcher can trip its `AbortController`.
|
||||
///
|
||||
/// Returns `{ cancelled }` reflecting whether either surface
|
||||
/// matched. The RPC itself never errors — a cancelled tool surfaces
|
||||
@@ -635,20 +634,7 @@ fn clamp_browser_wait(wait_ms: Option<u64>) -> Option<Duration> {
|
||||
))
|
||||
}
|
||||
|
||||
async fn handle_session_start(
|
||||
state: &Arc<DaemonState>,
|
||||
rpc_id: RpcId,
|
||||
params: Value,
|
||||
) -> Result<Value, RpcError> {
|
||||
let abort_guard = state
|
||||
.abort_registry
|
||||
.register(rpc_id)
|
||||
.map_err(|err| RpcError {
|
||||
code: ErrorCode::ProtocolError,
|
||||
message: format!("session.start cancellation registration failed: {err:?}"),
|
||||
data: None,
|
||||
})?;
|
||||
let cancel = abort_guard.token().clone();
|
||||
async fn handle_session_start(state: &Arc<DaemonState>, params: Value) -> Result<Value, RpcError> {
|
||||
let params: CliSessionStartParams = if params.is_null() {
|
||||
CliSessionStartParams {
|
||||
browser_instance_id: None,
|
||||
@@ -687,7 +673,6 @@ async fn handle_session_start(
|
||||
},
|
||||
state.config.extension_connect_wait,
|
||||
DEFAULT_RPC_TIMEOUT,
|
||||
Some(cancel),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -711,8 +696,6 @@ fn map_start_error(err: StartSessionError) -> RpcError {
|
||||
StartSessionError::AmbiguousBrowserLabel { .. } => ErrorCode::InvalidParams,
|
||||
StartSessionError::IdExhausted => ErrorCode::ProtocolError,
|
||||
StartSessionError::Timeout => ErrorCode::Timeout,
|
||||
StartSessionError::Cancelled => ErrorCode::Cancelled,
|
||||
StartSessionError::CleanupFailed { .. } => ErrorCode::ProtocolError,
|
||||
StartSessionError::TransportClosed => ErrorCode::ProtocolError,
|
||||
StartSessionError::ExtensionError(inner) => inner.code,
|
||||
};
|
||||
@@ -728,16 +711,6 @@ fn map_start_error(err: StartSessionError) -> RpcError {
|
||||
"label": label,
|
||||
"instance_ids": instance_ids,
|
||||
})),
|
||||
StartSessionError::CleanupFailed {
|
||||
session_id,
|
||||
agent_window_id,
|
||||
..
|
||||
} => Some(serde_json::json!({
|
||||
"reason": "cleanup_failed",
|
||||
"session_id": session_id.0,
|
||||
"agent_window_id": agent_window_id,
|
||||
})),
|
||||
StartSessionError::ExtensionError(inner) => inner.data.clone(),
|
||||
_ => None,
|
||||
};
|
||||
RpcError {
|
||||
@@ -747,20 +720,7 @@ fn map_start_error(err: StartSessionError) -> RpcError {
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_session_stop(
|
||||
state: &Arc<DaemonState>,
|
||||
rpc_id: RpcId,
|
||||
params: Value,
|
||||
) -> Result<Value, RpcError> {
|
||||
let abort_guard = state
|
||||
.abort_registry
|
||||
.register(rpc_id)
|
||||
.map_err(|err| RpcError {
|
||||
code: ErrorCode::ProtocolError,
|
||||
message: format!("session.stop cancellation registration failed: {err:?}"),
|
||||
data: None,
|
||||
})?;
|
||||
let cancel = abort_guard.token().clone();
|
||||
async fn handle_session_stop(state: &Arc<DaemonState>, params: Value) -> Result<Value, RpcError> {
|
||||
let params: CliSessionStopParams = if params.is_null() {
|
||||
CliSessionStopParams {
|
||||
session_id: None,
|
||||
@@ -774,7 +734,7 @@ async fn handle_session_stop(
|
||||
})?
|
||||
};
|
||||
if params.all {
|
||||
return handle_session_stop_all(state, Some(cancel)).await;
|
||||
return handle_session_stop_all(state).await;
|
||||
}
|
||||
let session_id = match params.session_id {
|
||||
Some(s) => SessionId(s),
|
||||
@@ -793,7 +753,6 @@ async fn handle_session_stop(
|
||||
&state.session_interrupts,
|
||||
&session_id,
|
||||
DEFAULT_SESSION_STOP_TIMEOUT,
|
||||
Some(cancel),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -827,7 +786,19 @@ async fn handle_session_stop(
|
||||
}
|
||||
}
|
||||
|
||||
/// Marks a `session.stop` rejected because another teardown already owns the
|
||||
/// session, so callers that race on purpose (`bsk record`) can tell it apart
|
||||
/// from a browser that stopped responding.
|
||||
pub const STOP_IN_PROGRESS_REASON: &str = "session_stop_in_progress";
|
||||
|
||||
fn map_stop_error(err: StopSessionError) -> RpcError {
|
||||
if matches!(err, StopSessionError::Stopping) {
|
||||
return RpcError {
|
||||
code: ErrorCode::Timeout,
|
||||
message: err.to_string(),
|
||||
data: Some(serde_json::json!({ "reason": STOP_IN_PROGRESS_REASON })),
|
||||
};
|
||||
}
|
||||
let code = match &err {
|
||||
StopSessionError::NotFound => ErrorCode::NotFound,
|
||||
StopSessionError::Stopping => ErrorCode::Timeout,
|
||||
@@ -845,25 +816,7 @@ fn map_stop_error(err: StopSessionError) -> RpcError {
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_session_stop_all_rpc(
|
||||
state: &Arc<DaemonState>,
|
||||
rpc_id: RpcId,
|
||||
) -> Result<Value, RpcError> {
|
||||
let abort_guard = state
|
||||
.abort_registry
|
||||
.register(rpc_id)
|
||||
.map_err(|err| RpcError {
|
||||
code: ErrorCode::ProtocolError,
|
||||
message: format!("session.stop_all cancellation registration failed: {err:?}"),
|
||||
data: None,
|
||||
})?;
|
||||
handle_session_stop_all(state, Some(abort_guard.token().clone())).await
|
||||
}
|
||||
|
||||
async fn handle_session_stop_all(
|
||||
state: &Arc<DaemonState>,
|
||||
cancel: Option<super::abort::AbortToken>,
|
||||
) -> Result<Value, RpcError> {
|
||||
async fn handle_session_stop_all(state: &Arc<DaemonState>) -> Result<Value, RpcError> {
|
||||
let ids: Vec<SessionId> = state
|
||||
.sessions
|
||||
.snapshot()
|
||||
@@ -875,16 +828,6 @@ async fn handle_session_stop_all(
|
||||
let mut returned_tab_ids = Vec::new();
|
||||
let mut return_failures = Vec::new();
|
||||
for id in ids {
|
||||
if cancel
|
||||
.as_ref()
|
||||
.is_some_and(super::abort::AbortToken::is_cancelled)
|
||||
{
|
||||
return Err(RpcError {
|
||||
code: ErrorCode::Cancelled,
|
||||
message: "session.stop_all was cancelled".into(),
|
||||
data: None,
|
||||
});
|
||||
}
|
||||
match stop_session(
|
||||
&state.browsers,
|
||||
&state.sessions,
|
||||
@@ -892,7 +835,6 @@ async fn handle_session_stop_all(
|
||||
&state.session_interrupts,
|
||||
&id,
|
||||
DEFAULT_SESSION_STOP_TIMEOUT,
|
||||
cancel.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -918,15 +860,6 @@ async fn handle_session_stop_all(
|
||||
returned_tab_ids.extend(stop.returned_tab_ids);
|
||||
return_failures.extend(stop.return_failures);
|
||||
}
|
||||
Err(err)
|
||||
if matches!(
|
||||
&err,
|
||||
StopSessionError::ExtensionError(inner)
|
||||
if inner.code == ErrorCode::Cancelled
|
||||
) =>
|
||||
{
|
||||
return Err(map_stop_error(err));
|
||||
}
|
||||
Err(err) => {
|
||||
debug!(session = %id, ?err, "session.stop_all: failure (continuing)");
|
||||
let rpc = map_stop_error(err);
|
||||
|
||||
@@ -43,7 +43,6 @@ use serde_json::Value;
|
||||
use tokio::sync::{mpsc, oneshot};
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use super::abort::AbortToken;
|
||||
use super::browsers::BrowserRegistry;
|
||||
use super::inflight::{PromoteOutcome, ToolInflightEntry};
|
||||
use super::sessions::{SessionId, SessionRegistry};
|
||||
@@ -59,11 +58,6 @@ pub const QUEUE_CAPACITY: usize = 64;
|
||||
/// so tests can default it.
|
||||
pub const DEFAULT_TOOL_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
/// Once a forwarded request is cancelled, keep the session queue busy while
|
||||
/// the extension finishes handler-side compensation. This is deliberately
|
||||
/// bounded so a wedged extension cannot pin the session forever.
|
||||
pub const CANCEL_CLEANUP_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
|
||||
/// Job submitted into a session queue. Carries everything the worker
|
||||
/// needs to forward one RPC and one oneshot to deliver the answer.
|
||||
pub struct ToolJob {
|
||||
@@ -76,22 +70,14 @@ pub struct ToolJob {
|
||||
/// the queued/forwarded state machine; the worker checks it on
|
||||
/// pre-flight (so a cancel arriving while the job was still
|
||||
/// queued short-circuits without ever touching the extension)
|
||||
/// and observes `cancel.cancelled()` while awaiting the WS response.
|
||||
/// A forwarded cancel keeps the worker busy until the extension's
|
||||
/// final response arrives or the bounded cleanup timeout expires.
|
||||
/// and selects on `cancel.cancelled()` while awaiting the WS
|
||||
/// response (so a cancel arriving mid-flight unblocks the worker
|
||||
/// the same way the WS-side cancel frame already did).
|
||||
///
|
||||
/// `None` for daemon-internal callers that do not flow through an
|
||||
/// IPC request id (e.g. `session.stop`'s queued teardown call —
|
||||
/// those already carry their own bespoke retry / abort path).
|
||||
pub inflight: Option<Arc<ToolInflightEntry>>,
|
||||
/// Cancellation token for session-lifecycle work registered in the
|
||||
/// daemon-local abort registry. Unlike `inflight`, lifecycle callers
|
||||
/// forward their own WS cancel after the request has been queued.
|
||||
pub lifecycle_cancel: Option<AbortToken>,
|
||||
/// How long cancellation keeps the worker busy while the extension
|
||||
/// finishes compensation. Session teardown uses its full RPC budget
|
||||
/// so daemon state is reconciled even after the CLI exits.
|
||||
pub cancel_cleanup_timeout: Duration,
|
||||
/// Owning session — used by `ToolInflightRegistry::cancel_session`
|
||||
/// to drain queued jobs that have not yet been promoted to
|
||||
/// "forwarded". Worker callers also have access to it via the
|
||||
@@ -331,8 +317,6 @@ impl ToolQueueRegistry {
|
||||
timeout,
|
||||
false,
|
||||
inflight,
|
||||
None,
|
||||
CANCEL_CLEANUP_TIMEOUT,
|
||||
)
|
||||
.await;
|
||||
// A long-running request may outlive the idle threshold. Touching
|
||||
@@ -371,8 +355,6 @@ impl ToolQueueRegistry {
|
||||
timeout,
|
||||
respond: respond_tx,
|
||||
inflight,
|
||||
lifecycle_cancel: None,
|
||||
cancel_cleanup_timeout: CANCEL_CLEANUP_TIMEOUT,
|
||||
session_id: sid.clone(),
|
||||
};
|
||||
match forward_one(sid, &self.browsers, &self.sessions, &job).await {
|
||||
@@ -391,7 +373,6 @@ impl ToolQueueRegistry {
|
||||
method: Method,
|
||||
params: Value,
|
||||
timeout: Duration,
|
||||
cancel: Option<AbortToken>,
|
||||
) -> Result<Value, DispatchError> {
|
||||
let (sender, state) = {
|
||||
let mut guard = self.queues.lock().expect("tool queue registry poisoned");
|
||||
@@ -418,8 +399,6 @@ impl ToolQueueRegistry {
|
||||
timeout,
|
||||
true,
|
||||
None,
|
||||
cancel,
|
||||
timeout,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -435,8 +414,6 @@ async fn dispatch_with_sender(
|
||||
timeout: Duration,
|
||||
wait_for_capacity: bool,
|
||||
inflight: Option<Arc<ToolInflightEntry>>,
|
||||
lifecycle_cancel: Option<AbortToken>,
|
||||
cancel_cleanup_timeout: Duration,
|
||||
) -> Result<Value, DispatchError> {
|
||||
let (respond_tx, respond_rx) = oneshot::channel();
|
||||
let job = ToolJob {
|
||||
@@ -445,11 +422,8 @@ async fn dispatch_with_sender(
|
||||
timeout,
|
||||
respond: respond_tx,
|
||||
inflight,
|
||||
lifecycle_cancel,
|
||||
cancel_cleanup_timeout,
|
||||
session_id,
|
||||
};
|
||||
let lifecycle_cancellable = job.lifecycle_cancel.is_some();
|
||||
if wait_for_capacity {
|
||||
let waited = tokio::time::timeout(
|
||||
timeout.saturating_add(Duration::from_secs(1)),
|
||||
@@ -474,15 +448,8 @@ async fn dispatch_with_sender(
|
||||
mpsc::error::TrySendError::Closed(_) => DispatchError::QueueClosed,
|
||||
});
|
||||
}
|
||||
let mut response_timeout = timeout.saturating_add(Duration::from_secs(1));
|
||||
if lifecycle_cancellable {
|
||||
// A lifecycle cancel can arrive at the original RPC deadline and
|
||||
// intentionally keeps the worker alive while extension cleanup
|
||||
// settles. Do not let this outer waiter abandon that reconciliation
|
||||
// early, which could reopen a daemon session whose window is gone.
|
||||
response_timeout = response_timeout.saturating_add(cancel_cleanup_timeout);
|
||||
}
|
||||
let waited = tokio::time::timeout(response_timeout, respond_rx).await;
|
||||
let waited =
|
||||
tokio::time::timeout(timeout.saturating_add(Duration::from_secs(1)), respond_rx).await;
|
||||
match waited {
|
||||
Ok(Ok(Ok(v))) => Ok(v),
|
||||
Ok(Ok(Err(rpc))) => Err(DispatchError::Rpc(rpc)),
|
||||
@@ -520,17 +487,11 @@ async fn forward_one(
|
||||
// / browser resolution work, and before any WS frame leaves the
|
||||
// daemon (review C2). We keep the `tokio::select!` later so
|
||||
// cancels arriving mid-WS-hop also unblock the worker.
|
||||
if job
|
||||
.inflight
|
||||
.as_ref()
|
||||
.is_some_and(|entry| entry.is_cancelled())
|
||||
|| job
|
||||
.lifecycle_cancel
|
||||
.as_ref()
|
||||
.is_some_and(AbortToken::is_cancelled)
|
||||
if let Some(entry) = job.inflight.as_ref()
|
||||
&& entry.is_cancelled()
|
||||
{
|
||||
return Err(cancelled_error(
|
||||
job.inflight.as_deref(),
|
||||
Some(entry),
|
||||
"tool dispatch cancelled before forwarding",
|
||||
));
|
||||
}
|
||||
@@ -594,20 +555,11 @@ async fn forward_one(
|
||||
}
|
||||
}
|
||||
None => {
|
||||
// Daemon-internal callers do not carry a tool-inflight entry.
|
||||
// Session lifecycle jobs instead use the abort-registry token
|
||||
// below; other internal callers keep the plain send path.
|
||||
if job
|
||||
.lifecycle_cancel
|
||||
.as_ref()
|
||||
.is_some_and(AbortToken::is_cancelled)
|
||||
{
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
return Err(cancelled_error(
|
||||
None,
|
||||
"session lifecycle cancelled before forwarding",
|
||||
));
|
||||
}
|
||||
// Daemon-internal callers (e.g. queued session.stop drain)
|
||||
// don't carry an inflight entry; they cannot be cancelled
|
||||
// through the IPC `cancel` surface, so a plain sink send
|
||||
// is sufficient and keeps the existing transport-error
|
||||
// behaviour.
|
||||
if client.sink.send(request).is_err() {
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
return Err(RpcError {
|
||||
@@ -616,90 +568,23 @@ async fn forward_one(
|
||||
data: None,
|
||||
});
|
||||
}
|
||||
job.lifecycle_cancel.clone()
|
||||
None
|
||||
}
|
||||
};
|
||||
let forward_lifecycle_cancel = || {
|
||||
if job.lifecycle_cancel.is_none() {
|
||||
return;
|
||||
}
|
||||
let cancel = Frame::Request(RequestFrame {
|
||||
id: format!("cancel-{rpc_id}"),
|
||||
method: Method::Cancel,
|
||||
params: Some(serde_json::json!({ "rpc_id": rpc_id })),
|
||||
});
|
||||
if let Err(err) = client.sink.send(cancel) {
|
||||
warn!(%rpc_id, ?err, "failed to forward session lifecycle cancel");
|
||||
}
|
||||
};
|
||||
let on_abort: Option<&(dyn Fn() + Sync)> = job
|
||||
.lifecycle_cancel
|
||||
.as_ref()
|
||||
.map(|_| &forward_lifecycle_cancel as &(dyn Fn() + Sync));
|
||||
let waited = await_with_optional_cancel(
|
||||
job.timeout,
|
||||
job.cancel_cleanup_timeout,
|
||||
waiter,
|
||||
cancel_token.as_ref(),
|
||||
on_abort,
|
||||
)
|
||||
.await;
|
||||
let waited = await_with_optional_cancel(job.timeout, waiter, cancel_token.as_ref()).await;
|
||||
let response = match waited {
|
||||
WaitOutcome::Response(resp) => resp,
|
||||
WaitOutcome::CancelledAfterResponse(resp) => {
|
||||
if job.method == Method::ToolSessionStop
|
||||
&& let ResponseBody::Ok(value) = resp.body
|
||||
{
|
||||
// Teardown crossed its final irreversible boundary before
|
||||
// the cancel landed. Commit the real extension result so the
|
||||
// daemon does not retain a session whose window is gone.
|
||||
return Ok(value);
|
||||
}
|
||||
// Cancellation wins the external verdict, but only after the
|
||||
// extension's original RPC has settled. Preserve any non-cancel
|
||||
// error so compensation failures remain explicit instead of
|
||||
// being hidden behind a generic cancelled result.
|
||||
if let ResponseBody::Err(err) = resp.body
|
||||
&& !matches!(err.code, ErrorCode::Cancelled | ErrorCode::UserAborted)
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
WaitOutcome::Cancelled => {
|
||||
// Drop the WS waiter so a late extension reply is dropped
|
||||
// cleanly (otherwise pending.resolve would log a stale
|
||||
// entry). The CLI caller will get a synthesised
|
||||
// `cancelled` here even if the extension never answers.
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
return Err(cancelled_error(
|
||||
job.inflight.as_deref(),
|
||||
"tool dispatch cancelled after extension cleanup",
|
||||
"tool dispatch cancelled mid-flight",
|
||||
));
|
||||
}
|
||||
WaitOutcome::TimedOutAfterResponse(resp) => match resp.body {
|
||||
ResponseBody::Ok(value) if job.method == Method::ToolSessionStop => {
|
||||
// The close crossed its irreversible boundary during the
|
||||
// timeout cleanup grace. Reconcile daemon state instead of
|
||||
// reopening a session whose window is already gone.
|
||||
return Ok(value);
|
||||
}
|
||||
ResponseBody::Err(err)
|
||||
if !matches!(err.code, ErrorCode::Cancelled | ErrorCode::UserAborted) =>
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
_ => {
|
||||
return Err(RpcError {
|
||||
code: ErrorCode::Timeout,
|
||||
message: format!("tool RPC timed out after {:?}", job.timeout),
|
||||
data: None,
|
||||
});
|
||||
}
|
||||
},
|
||||
WaitOutcome::CleanupTimeout => {
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
return Err(RpcError {
|
||||
code: ErrorCode::Timeout,
|
||||
message: format!(
|
||||
"cancelled tool did not finish cleanup within {:?}",
|
||||
job.cancel_cleanup_timeout
|
||||
),
|
||||
data: Some(serde_json::json!({ "reason": "cancel_cleanup_timeout" })),
|
||||
});
|
||||
}
|
||||
WaitOutcome::WaiterClosed => {
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
return Err(RpcError {
|
||||
@@ -726,57 +611,47 @@ async fn forward_one(
|
||||
#[derive(Debug)]
|
||||
enum WaitOutcome {
|
||||
Response(bsk_protocol::ResponseFrame),
|
||||
CancelledAfterResponse(bsk_protocol::ResponseFrame),
|
||||
TimedOutAfterResponse(bsk_protocol::ResponseFrame),
|
||||
CleanupTimeout,
|
||||
Cancelled,
|
||||
WaiterClosed,
|
||||
Timeout,
|
||||
}
|
||||
|
||||
async fn await_with_optional_cancel(
|
||||
timeout: Duration,
|
||||
cleanup_timeout: Duration,
|
||||
mut waiter: oneshot::Receiver<bsk_protocol::ResponseFrame>,
|
||||
waiter: oneshot::Receiver<bsk_protocol::ResponseFrame>,
|
||||
cancel: Option<&super::abort::AbortToken>,
|
||||
on_abort: Option<&(dyn Fn() + Sync)>,
|
||||
) -> WaitOutcome {
|
||||
match cancel {
|
||||
Some(token) => {
|
||||
let deadline = tokio::time::sleep(timeout);
|
||||
tokio::pin!(deadline);
|
||||
tokio::select! {
|
||||
// Cancel keeps same-tick priority, but it no longer drops the
|
||||
// waiter. The worker remains busy until the extension replies
|
||||
// after compensation or the cleanup deadline expires.
|
||||
biased;
|
||||
_ = token.cancelled() => {
|
||||
if let Some(on_abort) = on_abort {
|
||||
on_abort();
|
||||
}
|
||||
match tokio::time::timeout(cleanup_timeout, &mut waiter).await {
|
||||
Ok(Ok(resp)) => WaitOutcome::CancelledAfterResponse(resp),
|
||||
Ok(Err(_)) => WaitOutcome::WaiterClosed,
|
||||
Err(_) => WaitOutcome::CleanupTimeout,
|
||||
}
|
||||
},
|
||||
outcome = &mut waiter => match outcome {
|
||||
Ok(resp) => WaitOutcome::Response(resp),
|
||||
Err(_) => WaitOutcome::WaiterClosed,
|
||||
},
|
||||
_ = &mut deadline => {
|
||||
let Some(on_abort) = on_abort else {
|
||||
return WaitOutcome::Timeout;
|
||||
};
|
||||
on_abort();
|
||||
match tokio::time::timeout(cleanup_timeout, &mut waiter).await {
|
||||
Ok(Ok(resp)) => WaitOutcome::TimedOutAfterResponse(resp),
|
||||
Ok(Err(_)) => WaitOutcome::WaiterClosed,
|
||||
Err(_) => WaitOutcome::CleanupTimeout,
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
None => match tokio::time::timeout(timeout, &mut waiter).await {
|
||||
// Cancel wins if both ready: when a cancel notification and
|
||||
// the extension's response are both observable on the same
|
||||
// tokio tick, `biased;` polls `token.cancelled()` first and
|
||||
// resolves the whole `select!` to `WaitOutcome::Cancelled`.
|
||||
// The already-arrived tool result is intentionally dropped.
|
||||
//
|
||||
// This extends design §4.6 — which only specifies "CLI sends
|
||||
// `cancel` on SIGINT and waits for the extension's
|
||||
// `cancelled` reply (≤ 2s before forced exit)" — by pinning
|
||||
// the same-tick race to a single, observable verdict.
|
||||
// Without `biased;`, an extension racing a fast successful
|
||||
// reply against a cancel notification could occasionally
|
||||
// resolve as `ok` even though the caller had already moved on
|
||||
// to compensation logic, leaving the agent trusting state the
|
||||
// daemon had just been asked to roll back. Picking
|
||||
// cancel-wins keeps the external observation rule simple
|
||||
// ("once a cancel is in flight, the in-flight RPC's verdict
|
||||
// is `cancelled`") and matches what the CLI / agent already
|
||||
// assumes after firing SIGINT. Round 3 M1 / round 4 M2
|
||||
// nail-down.
|
||||
Some(token) => tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => WaitOutcome::Cancelled,
|
||||
outcome = tokio::time::timeout(timeout, waiter) => match outcome {
|
||||
Ok(Ok(resp)) => WaitOutcome::Response(resp),
|
||||
Ok(Err(_)) => WaitOutcome::WaiterClosed,
|
||||
Err(_) => WaitOutcome::Timeout,
|
||||
},
|
||||
},
|
||||
None => match tokio::time::timeout(timeout, waiter).await {
|
||||
Ok(Ok(resp)) => WaitOutcome::Response(resp),
|
||||
Ok(Err(_)) => WaitOutcome::WaiterClosed,
|
||||
Err(_) => WaitOutcome::Timeout,
|
||||
@@ -858,17 +733,10 @@ mod await_with_optional_cancel_tests {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
tx.send(dummy_response()).unwrap();
|
||||
|
||||
let outcome = await_with_optional_cancel(
|
||||
Duration::from_secs(10),
|
||||
Duration::from_secs(1),
|
||||
rx,
|
||||
Some(&token),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
let outcome = await_with_optional_cancel(Duration::from_secs(10), rx, Some(&token)).await;
|
||||
assert!(
|
||||
matches!(outcome, WaitOutcome::CancelledAfterResponse(_)),
|
||||
"expected cancellation to win while retaining the ready response"
|
||||
matches!(outcome, WaitOutcome::Cancelled),
|
||||
"expected Cancelled when both arms are ready under biased; (got something else)"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -881,14 +749,7 @@ mod await_with_optional_cancel_tests {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
tx.send(dummy_response()).unwrap();
|
||||
|
||||
let outcome = await_with_optional_cancel(
|
||||
Duration::from_secs(10),
|
||||
Duration::from_secs(1),
|
||||
rx,
|
||||
Some(&token),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
let outcome = await_with_optional_cancel(Duration::from_secs(10), rx, Some(&token)).await;
|
||||
match outcome {
|
||||
WaitOutcome::Response(frame) => {
|
||||
assert!(matches!(frame.body, ResponseBody::Ok(_)));
|
||||
@@ -901,83 +762,9 @@ mod await_with_optional_cancel_tests {
|
||||
async fn no_cancel_token_still_returns_response() {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
tx.send(dummy_response()).unwrap();
|
||||
let outcome = await_with_optional_cancel(
|
||||
Duration::from_secs(10),
|
||||
Duration::from_secs(1),
|
||||
rx,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
let outcome = await_with_optional_cancel(Duration::from_secs(10), rx, None).await;
|
||||
assert!(matches!(outcome, WaitOutcome::Response(_)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_waiter_stays_pending_until_cleanup_response_arrives() {
|
||||
let token = AbortToken::new();
|
||||
let (tx, rx) = oneshot::channel();
|
||||
token.cancel();
|
||||
let pending = tokio::spawn(async move {
|
||||
await_with_optional_cancel(
|
||||
Duration::from_secs(10),
|
||||
Duration::from_secs(1),
|
||||
rx,
|
||||
Some(&token),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
});
|
||||
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!pending.is_finished());
|
||||
tx.send(dummy_response()).unwrap();
|
||||
assert!(matches!(
|
||||
pending.await.unwrap(),
|
||||
WaitOutcome::CancelledAfterResponse(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancelled_waiter_releases_after_cleanup_timeout() {
|
||||
let token = AbortToken::new();
|
||||
let (_tx, rx) = oneshot::channel();
|
||||
token.cancel();
|
||||
let outcome = await_with_optional_cancel(
|
||||
Duration::from_secs(10),
|
||||
Duration::from_millis(10),
|
||||
rx,
|
||||
Some(&token),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(outcome, WaitOutcome::CleanupTimeout));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn lifecycle_timeout_forwards_abort_and_waits_for_cleanup_response() {
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
|
||||
let token = AbortToken::new();
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let abort_forwarded = AtomicBool::new(false);
|
||||
let forward_abort = || abort_forwarded.store(true, Ordering::SeqCst);
|
||||
tokio::spawn(async move {
|
||||
tokio::time::sleep(Duration::from_millis(20)).await;
|
||||
tx.send(dummy_response()).unwrap();
|
||||
});
|
||||
|
||||
let outcome = await_with_optional_cancel(
|
||||
Duration::from_millis(5),
|
||||
Duration::from_millis(100),
|
||||
rx,
|
||||
Some(&token),
|
||||
Some(&forward_abort),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(abort_forwarded.load(Ordering::SeqCst));
|
||||
assert!(matches!(outcome, WaitOutcome::TimedOutAfterResponse(_)));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -1040,8 +827,6 @@ mod tool_job_session_id_tests {
|
||||
timeout: Duration::from_secs(1),
|
||||
respond: tx,
|
||||
inflight: None,
|
||||
lifecycle_cancel: None,
|
||||
cancel_cleanup_timeout: CANCEL_CLEANUP_TIMEOUT,
|
||||
session_id: SessionId("sess-A".into()),
|
||||
};
|
||||
assert_eq!(job.session_id.0, "sess-A");
|
||||
|
||||
@@ -12,14 +12,13 @@ use bsk_protocol::system::{BrowserStatusEntry, SessionStatusEntry};
|
||||
use bsk_protocol::tools::{
|
||||
SessionStartParams, SessionStartResult, SessionStopParams, SessionStopResult,
|
||||
};
|
||||
use bsk_protocol::{ErrorCode, Frame, Method, RequestFrame, ResponseBody, RpcError, RpcId};
|
||||
use bsk_protocol::{Frame, RequestFrame, ResponseBody, RpcError, RpcId};
|
||||
use rand::Rng;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::time::{Duration, timeout};
|
||||
|
||||
use super::abort::AbortToken;
|
||||
use super::browsers::{BrowserClient, BrowserId, BrowserRegistry, SelectError};
|
||||
use super::queue::{CANCEL_CLEANUP_TIMEOUT, DispatchError, ToolQueueRegistry};
|
||||
use super::queue::{DispatchError, ToolQueueRegistry};
|
||||
use super::session_interrupt::SessionInterruptRegistry;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
@@ -340,14 +339,6 @@ pub enum StartSessionError {
|
||||
IdExhausted,
|
||||
#[error("session creation timed out waiting for extension")]
|
||||
Timeout,
|
||||
#[error("session creation was cancelled")]
|
||||
Cancelled,
|
||||
#[error("session creation cancellation cleanup failed for {session_id}: {message}")]
|
||||
CleanupFailed {
|
||||
session_id: SessionId,
|
||||
agent_window_id: Option<i64>,
|
||||
message: String,
|
||||
},
|
||||
#[error("extension rejected tool.session_start: {0:?}")]
|
||||
ExtensionError(RpcError),
|
||||
#[error("transport closed while waiting for extension response")]
|
||||
@@ -363,8 +354,6 @@ impl StartSessionError {
|
||||
StartSessionError::AmbiguousBrowserLabel { .. } => "invalid_params",
|
||||
StartSessionError::IdExhausted => "protocol_error",
|
||||
StartSessionError::Timeout => "timeout",
|
||||
StartSessionError::Cancelled => "cancelled",
|
||||
StartSessionError::CleanupFailed { .. } => "protocol_error",
|
||||
StartSessionError::TransportClosed => "protocol_error",
|
||||
StartSessionError::ExtensionError(err) => match err.code {
|
||||
bsk_protocol::ErrorCode::Timeout => "timeout",
|
||||
@@ -418,7 +407,6 @@ pub struct AgentWindowOptions {
|
||||
/// On success the matching per-session dispatch queue (M6.5,
|
||||
/// design §5) is also spawned so that subsequent `tool.*` RPCs serialise
|
||||
/// against this session's Agent Window / ref-store.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn start_session(
|
||||
registry: &Arc<BrowserRegistry>,
|
||||
sessions: &Arc<SessionRegistry>,
|
||||
@@ -427,32 +415,24 @@ pub async fn start_session(
|
||||
window: AgentWindowOptions,
|
||||
connect_wait: Duration,
|
||||
timeout_dur: Duration,
|
||||
cancel: Option<AbortToken>,
|
||||
) -> Result<Session, StartSessionError> {
|
||||
let selection = registry.select_with_connect_wait(requested, connect_wait);
|
||||
tokio::pin!(selection);
|
||||
let selected = match cancel.as_ref() {
|
||||
Some(token) => tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => return Err(StartSessionError::Cancelled),
|
||||
selected = &mut selection => selected,
|
||||
},
|
||||
None => selection.await,
|
||||
};
|
||||
let client: Arc<BrowserClient> = selected.map_err(|e| match e {
|
||||
SelectError::NoBrowserConnected => StartSessionError::NoBrowserConnected,
|
||||
SelectError::MultipleBrowsersOnline => StartSessionError::MultipleBrowsersOnline {
|
||||
browsers: snapshot_status_entries(registry, sessions),
|
||||
},
|
||||
SelectError::NotFound => StartSessionError::BrowserNotFound,
|
||||
SelectError::AmbiguousLabel {
|
||||
label,
|
||||
instance_ids,
|
||||
} => StartSessionError::AmbiguousBrowserLabel {
|
||||
label,
|
||||
instance_ids,
|
||||
},
|
||||
})?;
|
||||
let client: Arc<BrowserClient> = registry
|
||||
.select_with_connect_wait(requested, connect_wait)
|
||||
.await
|
||||
.map_err(|e| match e {
|
||||
SelectError::NoBrowserConnected => StartSessionError::NoBrowserConnected,
|
||||
SelectError::MultipleBrowsersOnline => StartSessionError::MultipleBrowsersOnline {
|
||||
browsers: snapshot_status_entries(registry, sessions),
|
||||
},
|
||||
SelectError::NotFound => StartSessionError::BrowserNotFound,
|
||||
SelectError::AmbiguousLabel {
|
||||
label,
|
||||
instance_ids,
|
||||
} => StartSessionError::AmbiguousBrowserLabel {
|
||||
label,
|
||||
instance_ids,
|
||||
},
|
||||
})?;
|
||||
let session_id = sessions
|
||||
.reserve_id(client.id.clone(), SESSION_ID_MAX_RESERVE_ATTEMPTS, now_ms)
|
||||
.ok_or(StartSessionError::IdExhausted)?;
|
||||
@@ -473,71 +453,22 @@ pub async fn start_session(
|
||||
let mut pending = client.pending.lock().unwrap();
|
||||
pending.register(rpc_id.clone())
|
||||
};
|
||||
if cancel.as_ref().is_some_and(AbortToken::is_cancelled) {
|
||||
sessions.cancel_reservation(&session_id);
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
return Err(StartSessionError::Cancelled);
|
||||
}
|
||||
if client.sink.send(Frame::Request(request)).is_err() {
|
||||
sessions.cancel_reservation(&session_id);
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
return Err(StartSessionError::TransportClosed);
|
||||
}
|
||||
let mut waiter = waiter;
|
||||
enum StartWaitOutcome {
|
||||
Response(bsk_protocol::ResponseFrame),
|
||||
WaiterClosed,
|
||||
Cancelled,
|
||||
Timeout,
|
||||
}
|
||||
let wait_outcome = match cancel.as_ref() {
|
||||
Some(token) => {
|
||||
let deadline = tokio::time::sleep(timeout_dur);
|
||||
tokio::pin!(deadline);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = token.cancelled() => StartWaitOutcome::Cancelled,
|
||||
response = &mut waiter => match response {
|
||||
Ok(response) => StartWaitOutcome::Response(response),
|
||||
Err(_) => StartWaitOutcome::WaiterClosed,
|
||||
},
|
||||
_ = &mut deadline => StartWaitOutcome::Timeout,
|
||||
}
|
||||
}
|
||||
None => match timeout(timeout_dur, &mut waiter).await {
|
||||
Ok(Ok(response)) => StartWaitOutcome::Response(response),
|
||||
Ok(Err(_)) => StartWaitOutcome::WaiterClosed,
|
||||
Err(_) => StartWaitOutcome::Timeout,
|
||||
},
|
||||
};
|
||||
let response = match wait_outcome {
|
||||
StartWaitOutcome::Response(response) => response,
|
||||
StartWaitOutcome::WaiterClosed => {
|
||||
let response = match timeout(timeout_dur, waiter).await {
|
||||
Ok(Ok(resp)) => resp,
|
||||
Ok(Err(_)) => {
|
||||
sessions.cancel_reservation(&session_id);
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
return Err(StartSessionError::TransportClosed);
|
||||
}
|
||||
StartWaitOutcome::Cancelled => {
|
||||
return Err(finish_aborted_start(
|
||||
&client,
|
||||
sessions,
|
||||
&session_id,
|
||||
&rpc_id,
|
||||
waiter,
|
||||
StartAbortReason::Cancelled,
|
||||
)
|
||||
.await);
|
||||
}
|
||||
StartWaitOutcome::Timeout => {
|
||||
return Err(finish_aborted_start(
|
||||
&client,
|
||||
sessions,
|
||||
&session_id,
|
||||
&rpc_id,
|
||||
waiter,
|
||||
StartAbortReason::Timeout,
|
||||
)
|
||||
.await);
|
||||
Err(_) => {
|
||||
sessions.cancel_reservation(&session_id);
|
||||
client.pending.lock().unwrap().cancel(&rpc_id);
|
||||
return Err(StartSessionError::Timeout);
|
||||
}
|
||||
};
|
||||
let agent_window_id = match response.body {
|
||||
@@ -573,124 +504,6 @@ pub async fn start_session(
|
||||
Ok(session)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum StartAbortReason {
|
||||
Cancelled,
|
||||
Timeout,
|
||||
}
|
||||
|
||||
impl StartAbortReason {
|
||||
fn error(self) -> StartSessionError {
|
||||
match self {
|
||||
Self::Cancelled => StartSessionError::Cancelled,
|
||||
Self::Timeout => StartSessionError::Timeout,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn finish_aborted_start(
|
||||
client: &Arc<BrowserClient>,
|
||||
sessions: &Arc<SessionRegistry>,
|
||||
session_id: &SessionId,
|
||||
rpc_id: &RpcId,
|
||||
mut waiter: tokio::sync::oneshot::Receiver<bsk_protocol::ResponseFrame>,
|
||||
reason: StartAbortReason,
|
||||
) -> StartSessionError {
|
||||
sessions.cancel_reservation(session_id);
|
||||
let cancel = Frame::Request(RequestFrame {
|
||||
id: format!("cancel-{rpc_id}"),
|
||||
method: Method::Cancel,
|
||||
params: Some(serde_json::json!({ "rpc_id": rpc_id })),
|
||||
});
|
||||
if client.sink.send(cancel).is_err() {
|
||||
client.pending.lock().unwrap().cancel(rpc_id);
|
||||
return StartSessionError::TransportClosed;
|
||||
}
|
||||
|
||||
match timeout(CANCEL_CLEANUP_TIMEOUT, &mut waiter).await {
|
||||
Ok(Ok(response)) => match response.body {
|
||||
ResponseBody::Err(err)
|
||||
if matches!(err.code, ErrorCode::Cancelled | ErrorCode::UserAborted) =>
|
||||
{
|
||||
reason.error()
|
||||
}
|
||||
ResponseBody::Err(err) => StartSessionError::ExtensionError(err),
|
||||
ResponseBody::Ok(value) => {
|
||||
let agent_window_id = serde_json::from_value::<SessionStartResult>(value)
|
||||
.ok()
|
||||
.and_then(|result| result.agent_window_id);
|
||||
match rollback_extension_session(client, session_id).await {
|
||||
Ok(()) => reason.error(),
|
||||
Err(message) => StartSessionError::CleanupFailed {
|
||||
session_id: session_id.clone(),
|
||||
agent_window_id,
|
||||
message,
|
||||
},
|
||||
}
|
||||
}
|
||||
},
|
||||
Ok(Err(_)) => {
|
||||
client.pending.lock().unwrap().cancel(rpc_id);
|
||||
StartSessionError::TransportClosed
|
||||
}
|
||||
Err(_) => {
|
||||
// The cancel frame is already queued behind session_start. Even
|
||||
// if chrome.windows.create is still blocked, the extension will
|
||||
// observe the aborted signal and compensate before replying.
|
||||
client.pending.lock().unwrap().cancel(rpc_id);
|
||||
reason.error()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn rollback_extension_session(
|
||||
client: &Arc<BrowserClient>,
|
||||
session_id: &SessionId,
|
||||
) -> Result<(), String> {
|
||||
let rollback_id = next_rpc_id("sess-start-rollback");
|
||||
let waiter = {
|
||||
let mut pending = client.pending.lock().unwrap();
|
||||
pending.register(rollback_id.clone())
|
||||
};
|
||||
let request = RequestFrame {
|
||||
id: rollback_id.clone(),
|
||||
method: Method::ToolSessionStop,
|
||||
params: Some(
|
||||
serde_json::to_value(SessionStopParams {
|
||||
session_id: session_id.0.clone(),
|
||||
})
|
||||
.expect("serialize session_start rollback params"),
|
||||
),
|
||||
};
|
||||
if client.sink.send(Frame::Request(request)).is_err() {
|
||||
client.pending.lock().unwrap().cancel(&rollback_id);
|
||||
return Err("browser disconnected before rollback was queued".into());
|
||||
}
|
||||
let response = match timeout(CANCEL_CLEANUP_TIMEOUT, waiter).await {
|
||||
Ok(Ok(response)) => response,
|
||||
Ok(Err(_)) => return Err("browser disconnected during rollback".into()),
|
||||
Err(_) => {
|
||||
client.pending.lock().unwrap().cancel(&rollback_id);
|
||||
return Err(format!(
|
||||
"extension did not finish rollback within {CANCEL_CLEANUP_TIMEOUT:?}"
|
||||
));
|
||||
}
|
||||
};
|
||||
match response.body {
|
||||
ResponseBody::Ok(value) => {
|
||||
let result: SessionStopResult = serde_json::from_value(value)
|
||||
.map_err(|err| format!("invalid rollback response: {err}"))?;
|
||||
if result.return_failures.is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err("rollback reported borrowed-tab return failures".into())
|
||||
}
|
||||
}
|
||||
ResponseBody::Err(err) if err.code == ErrorCode::NotFound => Ok(()),
|
||||
ResponseBody::Err(err) => Err(format!("rollback failed: {} ({:?})", err.message, err.code)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Tear down a single session id (round-trips `tool.session_stop` to the
|
||||
/// owning extension and unregisters on success). Also tears down the
|
||||
/// per-session dispatch queue so the worker task exits.
|
||||
@@ -701,7 +514,6 @@ pub async fn stop_session(
|
||||
interrupts: &Arc<SessionInterruptRegistry>,
|
||||
session_id: &SessionId,
|
||||
timeout_dur: Duration,
|
||||
cancel: Option<AbortToken>,
|
||||
) -> Result<SessionStopResult, StopSessionError> {
|
||||
let session = sessions.get(session_id).ok_or(StopSessionError::NotFound)?;
|
||||
if registry.get(&session.browser_id).is_none() {
|
||||
@@ -712,17 +524,12 @@ pub async fn stop_session(
|
||||
session_id: session_id.0.clone(),
|
||||
})
|
||||
.unwrap();
|
||||
// Internal callers (idle reaping and shutdown) do not have a CLI-owned
|
||||
// token, but still need timeout-driven WS cancellation so a late window
|
||||
// close cannot leave daemon state pointing at a vanished session.
|
||||
let lifecycle_cancel = Some(cancel.unwrap_or_default());
|
||||
match queues
|
||||
.dispatch_after_closing(
|
||||
session_id,
|
||||
bsk_protocol::Method::ToolSessionStop,
|
||||
params,
|
||||
timeout_dur,
|
||||
lifecycle_cancel,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -760,10 +567,6 @@ pub async fn stop_session(
|
||||
drop_session_local(queues, interrupts, session_id);
|
||||
return Ok(SessionStopResult::default());
|
||||
}
|
||||
if err.code == bsk_protocol::ErrorCode::Timeout {
|
||||
queues.reopen(session_id);
|
||||
return Err(StopSessionError::Timeout);
|
||||
}
|
||||
queues.reopen(session_id);
|
||||
Err(StopSessionError::ExtensionError(err))
|
||||
}
|
||||
|
||||
@@ -27,7 +27,7 @@ use crate::daemon::{
|
||||
browsers::{BROWSER_LIVENESS_TICK, BROWSER_LIVENESS_TIMEOUT, EXTENSION_CONNECT_WAIT},
|
||||
info as daemon_info, ipc, lockfile, paths,
|
||||
sessions::{StopSessionError, forget_session, stop_session},
|
||||
state::DaemonState,
|
||||
state::{DaemonState, PROTOCOL_VERSION},
|
||||
ws,
|
||||
};
|
||||
|
||||
@@ -299,7 +299,7 @@ pub fn run_foreground(cfg: DaemonConfig) -> Result<()> {
|
||||
ws_port,
|
||||
sock_path: sock_path.clone(),
|
||||
daemon_version: env!("CARGO_PKG_VERSION"),
|
||||
protocol_version: "1.0",
|
||||
protocol_version: PROTOCOL_VERSION,
|
||||
};
|
||||
let handler = ipc::full_handler(status, Arc::clone(&state));
|
||||
|
||||
@@ -491,7 +491,6 @@ pub(crate) fn spawn_session_idle_reaper(state: Arc<DaemonState>) -> tokio::task:
|
||||
&state.session_interrupts,
|
||||
&session_id,
|
||||
Duration::from_secs(10),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
|
||||
@@ -16,9 +16,9 @@ use super::start::DaemonConfig;
|
||||
use super::ws::WsHandle;
|
||||
|
||||
pub const DAEMON_VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
pub const PROTOCOL_VERSION: &str = "1.0";
|
||||
/// Lowest **protocol** version peers must speak (e.g. `"1.0"`).
|
||||
pub const MIN_COMPATIBLE_PROTOCOL: &str = "1.0";
|
||||
pub const PROTOCOL_VERSION: &str = "1.1";
|
||||
/// Lowest **protocol** version peers must speak (e.g. `"1.1"`).
|
||||
pub const MIN_COMPATIBLE_PROTOCOL: &str = "1.1";
|
||||
/// Legacy app-semver floor used only when `HandshakeResult.min_compatible_peer`
|
||||
/// is emitted for old extensions. New code ignores this on read.
|
||||
pub const LEGACY_MIN_COMPATIBLE_PEER: &str = "0.0.0";
|
||||
@@ -34,8 +34,8 @@ pub struct DaemonState {
|
||||
/// `stop_session` / browser disconnect.
|
||||
pub tool_queues: Arc<ToolQueueRegistry>,
|
||||
/// Per-rpc-id cancellation tokens for daemon-side long-runners
|
||||
/// (`tool.wait_ms` plus `session.*` lifecycle calls). The CLI's
|
||||
/// `cancel { rpc_id }` consults this registry first.
|
||||
/// (M9.3 — currently only `tool.wait_ms`). The CLI's `cancel
|
||||
/// { rpc_id }` consults this registry first.
|
||||
pub abort_registry: Arc<AbortRegistry>,
|
||||
/// Tracks `tool.*` RPCs that have been forwarded to an extension
|
||||
/// over WS but have not yet received a response. Indexed by the
|
||||
|
||||
@@ -112,14 +112,14 @@ async fn handshake_as_ext(
|
||||
let params = HandshakeParams {
|
||||
client: "browser-skill-extension".into(),
|
||||
version: "0.1.0-dev.0".parse().unwrap(),
|
||||
protocol_version: "1.0".into(),
|
||||
protocol_version: "1.1".into(),
|
||||
instance_id: TEST_EXT_ID.into(),
|
||||
browser: BrowserPeerInfo {
|
||||
name: "chrome".into(),
|
||||
version: "131.0".into(),
|
||||
},
|
||||
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
||||
min_compatible_protocol: Some("1.0".into()),
|
||||
min_compatible_protocol: Some("1.1".into()),
|
||||
label: "Test".into(),
|
||||
};
|
||||
let req = RequestFrame {
|
||||
|
||||
@@ -75,7 +75,7 @@ async fn handshake_as_ext(
|
||||
let params = HandshakeParams {
|
||||
client: "browser-skill-extension".into(),
|
||||
version: "0.1.0-dev.0".parse().unwrap(),
|
||||
protocol_version: "1.0".into(),
|
||||
protocol_version: "1.1".into(),
|
||||
instance_id: TEST_EXT_ID.into(),
|
||||
browser: BrowserPeerInfo {
|
||||
name: "chrome".into(),
|
||||
@@ -83,7 +83,7 @@ async fn handshake_as_ext(
|
||||
},
|
||||
label: "Test".into(),
|
||||
min_compatible_peer: Some("0.1.0-dev.0".parse().unwrap()),
|
||||
min_compatible_protocol: Some("1.0".into()),
|
||||
min_compatible_protocol: Some("1.1".into()),
|
||||
};
|
||||
let req = RequestFrame {
|
||||
id: "hs".into(),
|
||||
@@ -583,11 +583,11 @@ async fn cancel_for_unknown_rpc_id_returns_false() {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cancel_keeps_session_busy_until_delayed_extension_cleanup_finishes() {
|
||||
// A forwarded cancel is acknowledged immediately, but the extension
|
||||
// deliberately delays the original RPC's final response to model
|
||||
// handler-side compensation. The daemon must keep the session busy
|
||||
// throughout that delay and release it only after cleanup settles.
|
||||
async fn concurrent_tool_call_returns_session_busy_while_slow_rpc_inflight() {
|
||||
// With session busy fast-fail, a second tool RPC submitted while
|
||||
// the worker is occupied returns `session_busy` immediately
|
||||
// instead of queuing behind the slow call. Cancel still unblocks
|
||||
// the in-flight RPC through the existing forwarded-cancel path.
|
||||
use std::sync::atomic::{AtomicUsize, Ordering as AOrdering};
|
||||
let (handle, sock) = spawn_daemon().await;
|
||||
let mut ws = connect_ext(handle.ws_addr()).await;
|
||||
@@ -596,10 +596,6 @@ async fn cancel_keeps_session_busy_until_delayed_extension_cleanup_finishes() {
|
||||
let ws = Arc::new(tokio::sync::Mutex::new(ws));
|
||||
let snapshots_seen = Arc::new(AtomicUsize::new(0));
|
||||
let snapshots_seen_clone = Arc::clone(&snapshots_seen);
|
||||
let cleanup_started = Arc::new(tokio::sync::Notify::new());
|
||||
let cleanup_started_clone = Arc::clone(&cleanup_started);
|
||||
let cleanup_release = Arc::new(tokio::sync::Notify::new());
|
||||
let cleanup_release_clone = Arc::clone(&cleanup_release);
|
||||
let ws_clone = Arc::clone(&ws);
|
||||
let responder = tokio::spawn(async move {
|
||||
let mut pending_snapshot: Option<String> = None;
|
||||
@@ -640,16 +636,6 @@ async fn cancel_keeps_session_busy_until_delayed_extension_cleanup_finishes() {
|
||||
snapshots_seen_clone.fetch_add(1, AOrdering::SeqCst);
|
||||
pending_snapshot = Some(req.id);
|
||||
}
|
||||
Method::ToolTabList => {
|
||||
let reply = ResponseFrame {
|
||||
id: req.id,
|
||||
body: ResponseBody::Ok(json!({ "tabs": [] })),
|
||||
};
|
||||
let mut g = ws_clone.lock().await;
|
||||
g.send(Message::Text(serde_json::to_string(&reply).unwrap()))
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
Method::Cancel => {
|
||||
let target = req
|
||||
.params
|
||||
@@ -661,8 +647,6 @@ async fn cancel_keeps_session_busy_until_delayed_extension_cleanup_finishes() {
|
||||
if let (Some(target), Some(snap)) = (target, snapshot_id)
|
||||
&& target == snap
|
||||
{
|
||||
cleanup_started_clone.notify_one();
|
||||
cleanup_release_clone.notified().await;
|
||||
let reply = ResponseFrame {
|
||||
id: snap,
|
||||
body: ResponseBody::Err(RpcError {
|
||||
@@ -769,33 +753,6 @@ async fn cancel_keeps_session_busy_until_delayed_extension_cleanup_finishes() {
|
||||
.expect("cancel slow rpc succeeds");
|
||||
assert!(cancel_slow.cancelled);
|
||||
|
||||
cleanup_started.notified().await;
|
||||
let busy_after_cancel = tokio::time::timeout(
|
||||
Duration::from_millis(200),
|
||||
busy_ipc.call_with_id::<_, serde_json::Value>(
|
||||
"snap-busy-after-cancel".into(),
|
||||
Method::ToolSnapshot,
|
||||
Some(json!({"session_id": session_id.clone()})),
|
||||
Duration::from_secs(10),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("post-cancel RPC should fast-fail while cleanup is pending")
|
||||
.unwrap();
|
||||
let busy_after_cancel_err =
|
||||
busy_after_cancel.expect_err("session must remain busy during delayed compensation");
|
||||
assert_eq!(busy_after_cancel_err.code, ErrorCode::Timeout);
|
||||
assert_eq!(
|
||||
busy_after_cancel_err
|
||||
.data
|
||||
.as_ref()
|
||||
.and_then(|d| d.get("reason"))
|
||||
.and_then(|v| v.as_str()),
|
||||
Some(bsk::rpc_reason::SESSION_BUSY)
|
||||
);
|
||||
|
||||
cleanup_release.notify_one();
|
||||
|
||||
let slow_outcome = tokio::time::timeout(Duration::from_secs(5), slow_handle)
|
||||
.await
|
||||
.expect("slow snapshot did not resolve")
|
||||
@@ -804,18 +761,6 @@ async fn cancel_keeps_session_busy_until_delayed_extension_cleanup_finishes() {
|
||||
let slow_err = slow_outcome.expect_err("slow snapshot must surface cancelled");
|
||||
assert_eq!(slow_err.code, ErrorCode::Cancelled);
|
||||
|
||||
let after_cleanup: serde_json::Value = busy_ipc
|
||||
.call(
|
||||
"tabs-after-cleanup",
|
||||
Method::ToolTabList,
|
||||
Some(json!({"session_id": session_id})),
|
||||
Duration::from_secs(3),
|
||||
)
|
||||
.await
|
||||
.unwrap()
|
||||
.expect("session queue should reopen after extension cleanup");
|
||||
assert_eq!(after_cleanup, json!({ "tabs": [] }));
|
||||
|
||||
assert_eq!(
|
||||
snapshots_seen.load(AOrdering::SeqCst),
|
||||
1,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user