mirror of
https://github.com/Tencent/BrowserSkill.git
synced 2026-09-10 13:01:23 +08:00
Compare commits
86
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
005a6fd534 | ||
|
|
20554508db | ||
|
|
6aa931b60a | ||
|
|
3df2738280 | ||
|
|
f86e506d47 | ||
|
|
45a246c41b | ||
|
|
6b78211e79 | ||
|
|
eebf334b40 | ||
|
|
a284ffc5ae | ||
|
|
e720f1ffe5 | ||
|
|
2b7e10ca2b | ||
|
|
1ef93463dd | ||
|
|
c7252529f4 | ||
|
|
0b65b4b295 | ||
|
|
94ec7e73fc | ||
|
|
42403e55f2 | ||
|
|
22019cdf07 | ||
|
|
8d7c0aa66c | ||
|
|
a9489282fa | ||
|
|
d4cddd6224 | ||
|
|
8e4de6cf4b | ||
|
|
fea66ce404 | ||
|
|
8bae85127a | ||
|
|
edb149611b | ||
|
|
14c5884cbc | ||
|
|
e7aad9bc16 | ||
|
|
c8c5c4a7c4 | ||
|
|
3fb9e9f581 | ||
|
|
9854ee2f45 | ||
|
|
51085b960b | ||
|
|
51df868008 | ||
|
|
43ebfab7ba | ||
|
|
ce11325de2 | ||
|
|
4d1dcd122f | ||
|
|
a4e668542d | ||
|
|
52aa356153 | ||
|
|
d1001e14b2 | ||
|
|
eb71ddc1da | ||
|
|
b07b8fec65 | ||
|
|
47070fdc14 | ||
|
|
916229dc62 | ||
|
|
dde707b558 | ||
|
|
e2d07574d7 | ||
|
|
461d204ca3 | ||
|
|
bc5d9610a3 | ||
|
|
2955260d11 | ||
|
|
ce89e95211 | ||
|
|
24faa10659 | ||
|
|
a7bc8bacf6 | ||
|
|
3d050e0f26 | ||
|
|
afdc398730 | ||
|
|
6b96fcca32 | ||
|
|
0771292336 | ||
|
|
54869cabd0 | ||
|
|
2de7c5c6ff | ||
|
|
703fc64399 | ||
|
|
4549ed0892 | ||
|
|
37fbaacc94 | ||
|
|
f81a1e7796 | ||
|
|
78c2719718 | ||
|
|
f443cfae74 | ||
|
|
0ce497f017 | ||
|
|
08212497f1 | ||
|
|
f98c0606a1 | ||
|
|
1371c92721 | ||
|
|
c75f97119d | ||
|
|
ac806ae8f4 | ||
|
|
89200975f3 | ||
|
|
238a98476e | ||
|
|
1da2d54ddc | ||
|
|
6e35e62199 | ||
|
|
610782698b | ||
|
|
9504072914 | ||
|
|
b4450fbb8c | ||
|
|
1268761611 | ||
|
|
37d3e2b22f | ||
|
|
ce8bc6c870 | ||
|
|
93df62a356 | ||
|
|
20002e1ff2 | ||
|
|
674c53339f | ||
|
|
d0bee1a09b | ||
|
|
66684c4ce3 | ||
|
|
aab3b7a8e5 | ||
|
|
2dad162a87 | ||
|
|
063b0bbc91 | ||
|
|
0de67c537a |
@@ -0,0 +1,105 @@
|
||||
name: Release dsh plugin
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- dsh-plugin-v*
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: release-dsh-plugin-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
PACKAGE_NAME: "@wxg-prc-cpg/browser-skill-dsh-plugin"
|
||||
PACKAGE_DIR: packages/dsh-plugin-browserskill
|
||||
|
||||
jobs:
|
||||
resolve:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
version: ${{ steps.version.outputs.version }}
|
||||
tag: ${{ steps.version.outputs.tag }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Resolve plugin version
|
||||
id: version
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
VERSION="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').version")"
|
||||
else
|
||||
VERSION="${GITHUB_REF_NAME#dsh-plugin-v}"
|
||||
fi
|
||||
VERSION="${VERSION#v}"
|
||||
echo "version=${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
echo "tag=dsh-plugin-v${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
|
||||
guard:
|
||||
needs: resolve
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- name: Verify tag matches package.json name and version
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
PKG_NAME="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').name")"
|
||||
PKG_VERSION="$(node -p "require('./${{ env.PACKAGE_DIR }}/package.json').version")"
|
||||
RELEASE_VERSION="${{ needs.resolve.outputs.version }}"
|
||||
if [ "$PKG_NAME" != "$PACKAGE_NAME" ]; then
|
||||
echo "package.json name (${PKG_NAME}) does not match expected ${PACKAGE_NAME}"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$PKG_VERSION" != "$RELEASE_VERSION" ]; then
|
||||
echo "${PACKAGE_DIR}/package.json version (${PKG_VERSION}) does not match release version (${RELEASE_VERSION})"
|
||||
exit 1
|
||||
fi
|
||||
echo "Version guard passed: ${PKG_NAME}@${RELEASE_VERSION}"
|
||||
|
||||
publish:
|
||||
needs: [resolve, guard]
|
||||
runs-on: ubuntu-latest
|
||||
# Secret NPM_TOKEN is stored on the "npm-publish" GitHub Environment.
|
||||
environment: npm-publish
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: pnpm/action-setup@v6
|
||||
|
||||
- uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 22
|
||||
cache: pnpm
|
||||
registry-url: https://registry.npmjs.org
|
||||
scope: "@wxg-prc-cpg"
|
||||
|
||||
- name: Install dependencies
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Typecheck plugin
|
||||
run: pnpm --filter "${{ env.PACKAGE_NAME }}" typecheck
|
||||
|
||||
- name: Test plugin
|
||||
run: pnpm --filter "${{ env.PACKAGE_NAME }}" test
|
||||
|
||||
- name: Publish to npm
|
||||
working-directory: ${{ env.PACKAGE_DIR }}
|
||||
env:
|
||||
NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${NODE_AUTH_TOKEN:-}" ]; then
|
||||
echo "::error::Environment secret NPM_TOKEN is not configured"
|
||||
exit 1
|
||||
fi
|
||||
pnpm publish --access public --no-git-checks
|
||||
echo "Published ${PACKAGE_NAME}@${{ needs.resolve.outputs.version }}"
|
||||
@@ -126,3 +126,202 @@ jobs:
|
||||
make_latest: false
|
||||
generate_release_notes: true
|
||||
files: dist/*.zip
|
||||
|
||||
publish-cws:
|
||||
needs: [resolve, build-extension]
|
||||
runs-on: ubuntu-latest
|
||||
environment: chrome-web-store
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Download build artifact
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: extension-zip
|
||||
path: dist
|
||||
|
||||
- name: Validate Chrome Web Store configuration
|
||||
shell: bash
|
||||
env:
|
||||
PUBLISHER_ID: ${{ vars.CHROME_WEBSTORE_PUBLISHER_ID }}
|
||||
EXTENSION_ID: ${{ vars.CHROME_WEBSTORE_EXTENSION_ID }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "$PUBLISHER_ID" ]; then
|
||||
echo "::error::Repository variable CHROME_WEBSTORE_PUBLISHER_ID is not configured"
|
||||
exit 1
|
||||
fi
|
||||
if [ -z "$EXTENSION_ID" ]; then
|
||||
echo "::error::Repository variable CHROME_WEBSTORE_EXTENSION_ID is not configured"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
ARCHIVE="$(find dist -maxdepth 1 -name '*.zip' -type f -print -quit)"
|
||||
if [ -z "$ARCHIVE" ]; then
|
||||
echo "::error::No extension zip found in the downloaded artifact"
|
||||
exit 1
|
||||
fi
|
||||
echo "ARCHIVE=$ARCHIVE" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Authenticate with service account
|
||||
id: cws-auth
|
||||
env:
|
||||
SERVICE_ACCOUNT_JSON: ${{ secrets.CHROME_WEBSTORE_SERVICE_ACCOUNT_JSON }}
|
||||
shell: bash
|
||||
run: |
|
||||
set -euo pipefail
|
||||
node --input-type=module <<'NODE'
|
||||
import { sign } from "node:crypto";
|
||||
import { appendFileSync } from "node:fs";
|
||||
|
||||
const rawCredentials = process.env.SERVICE_ACCOUNT_JSON;
|
||||
if (!rawCredentials) {
|
||||
throw new Error(
|
||||
"Repository secret CHROME_WEBSTORE_SERVICE_ACCOUNT_JSON is not configured",
|
||||
);
|
||||
}
|
||||
|
||||
let credentials;
|
||||
try {
|
||||
credentials = JSON.parse(rawCredentials);
|
||||
} catch {
|
||||
throw new Error(
|
||||
"Repository secret CHROME_WEBSTORE_SERVICE_ACCOUNT_JSON is not valid JSON",
|
||||
);
|
||||
}
|
||||
if (!credentials.client_email || !credentials.private_key) {
|
||||
throw new Error(
|
||||
"Service account JSON must contain client_email and private_key",
|
||||
);
|
||||
}
|
||||
|
||||
const encode = (value) =>
|
||||
Buffer.from(JSON.stringify(value)).toString("base64url");
|
||||
const issuedAt = Math.floor(Date.now() / 1000);
|
||||
const header = {
|
||||
alg: "RS256",
|
||||
typ: "JWT",
|
||||
kid: credentials.private_key_id,
|
||||
};
|
||||
const claim = {
|
||||
iss: credentials.client_email,
|
||||
scope: "https://www.googleapis.com/auth/chromewebstore",
|
||||
aud: "https://oauth2.googleapis.com/token",
|
||||
iat: issuedAt,
|
||||
exp: issuedAt + 3600,
|
||||
};
|
||||
const unsignedJwt = `${encode(header)}.${encode(claim)}`;
|
||||
const signature = sign(
|
||||
"RSA-SHA256",
|
||||
Buffer.from(unsignedJwt),
|
||||
credentials.private_key,
|
||||
).toString("base64url");
|
||||
const assertion = `${unsignedJwt}.${signature}`;
|
||||
|
||||
const response = await fetch("https://oauth2.googleapis.com/token", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/x-www-form-urlencoded" },
|
||||
body: new URLSearchParams({
|
||||
grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
|
||||
assertion,
|
||||
}),
|
||||
});
|
||||
const responseText = await response.text();
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`Service account token exchange failed (${response.status}): ${responseText}`,
|
||||
);
|
||||
}
|
||||
|
||||
const tokenResponse = JSON.parse(responseText);
|
||||
if (!tokenResponse.access_token) {
|
||||
throw new Error("Service account token response has no access_token");
|
||||
}
|
||||
|
||||
console.log(`::add-mask::${tokenResponse.access_token}`);
|
||||
appendFileSync(
|
||||
process.env.GITHUB_OUTPUT,
|
||||
`access_token=${tokenResponse.access_token}\n`,
|
||||
);
|
||||
NODE
|
||||
|
||||
- name: Upload and publish Chrome Web Store extension
|
||||
shell: bash
|
||||
env:
|
||||
ACCESS_TOKEN: ${{ steps.cws-auth.outputs.access_token }}
|
||||
PUBLISHER_ID: ${{ vars.CHROME_WEBSTORE_PUBLISHER_ID }}
|
||||
EXTENSION_ID: ${{ vars.CHROME_WEBSTORE_EXTENSION_ID }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
ITEM_NAME="publishers/${PUBLISHER_ID}/items/${EXTENSION_ID}"
|
||||
API_ROOT="https://chromewebstore.googleapis.com"
|
||||
UPLOAD_RESPONSE="$RUNNER_TEMP/cws-upload-response.json"
|
||||
STATUS_RESPONSE="$RUNNER_TEMP/cws-status-response.json"
|
||||
PUBLISH_RESPONSE="$RUNNER_TEMP/cws-publish-response.json"
|
||||
|
||||
HTTP_STATUS="$(curl --silent --show-error \
|
||||
--output "$UPLOAD_RESPONSE" \
|
||||
--write-out '%{http_code}' \
|
||||
--request POST \
|
||||
--header "Authorization: Bearer $ACCESS_TOKEN" \
|
||||
--header "Content-Type: application/zip" \
|
||||
--data-binary "@$ARCHIVE" \
|
||||
"$API_ROOT/upload/v2/$ITEM_NAME:upload")"
|
||||
if [ "$HTTP_STATUS" -lt 200 ] || [ "$HTTP_STATUS" -ge 300 ]; then
|
||||
echo "::error::Chrome Web Store upload failed with HTTP $HTTP_STATUS"
|
||||
jq . "$UPLOAD_RESPONSE" || cat "$UPLOAD_RESPONSE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
UPLOAD_STATE="$(jq -r '.uploadState // empty' "$UPLOAD_RESPONSE")"
|
||||
echo "Chrome Web Store upload state: $UPLOAD_STATE"
|
||||
|
||||
if [ "$UPLOAD_STATE" = "IN_PROGRESS" ] || [ "$UPLOAD_STATE" = "UPLOAD_IN_PROGRESS" ]; then
|
||||
for attempt in $(seq 1 30); do
|
||||
sleep 5
|
||||
HTTP_STATUS="$(curl --silent --show-error \
|
||||
--output "$STATUS_RESPONSE" \
|
||||
--write-out '%{http_code}' \
|
||||
--header "Authorization: Bearer $ACCESS_TOKEN" \
|
||||
"$API_ROOT/v2/$ITEM_NAME:fetchStatus")"
|
||||
if [ "$HTTP_STATUS" -lt 200 ] || [ "$HTTP_STATUS" -ge 300 ]; then
|
||||
echo "::error::Chrome Web Store status check failed with HTTP $HTTP_STATUS"
|
||||
jq . "$STATUS_RESPONSE" || cat "$STATUS_RESPONSE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
UPLOAD_STATE="$(jq -r '.lastAsyncUploadState // empty' "$STATUS_RESPONSE")"
|
||||
echo "Upload status check $attempt/30: $UPLOAD_STATE"
|
||||
if [ "$UPLOAD_STATE" = "SUCCEEDED" ]; then
|
||||
break
|
||||
fi
|
||||
if [ "$UPLOAD_STATE" != "IN_PROGRESS" ] && [ "$UPLOAD_STATE" != "UPLOAD_IN_PROGRESS" ]; then
|
||||
echo "::error::Chrome Web Store upload did not succeed: $UPLOAD_STATE"
|
||||
jq . "$STATUS_RESPONSE"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [ "$UPLOAD_STATE" != "SUCCEEDED" ]; then
|
||||
echo "::error::Chrome Web Store upload did not complete: $UPLOAD_STATE"
|
||||
jq . "$UPLOAD_RESPONSE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
HTTP_STATUS="$(curl --silent --show-error \
|
||||
--output "$PUBLISH_RESPONSE" \
|
||||
--write-out '%{http_code}' \
|
||||
--request POST \
|
||||
--header "Authorization: Bearer $ACCESS_TOKEN" \
|
||||
--header "Content-Type: application/json" \
|
||||
--data '{"publishType":"DEFAULT_PUBLISH"}' \
|
||||
"$API_ROOT/v2/$ITEM_NAME:publish")"
|
||||
if [ "$HTTP_STATUS" -lt 200 ] || [ "$HTTP_STATUS" -ge 300 ]; then
|
||||
echo "::error::Chrome Web Store publish failed with HTTP $HTTP_STATUS"
|
||||
jq . "$PUBLISH_RESPONSE" || cat "$PUBLISH_RESPONSE"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Chrome Web Store submission created successfully:"
|
||||
jq '{itemId, state, warningInfo}' "$PUBLISH_RESPONSE"
|
||||
|
||||
@@ -3,3 +3,6 @@
|
||||
**/.output/**
|
||||
**/target/**
|
||||
**/node_modules/**
|
||||
# Generated client CSS (tailwind compile output); built by
|
||||
# packages/dsh-plugin-browserskill/scripts/build-client-css.mjs.
|
||||
**/dsh-plugin-browserskill/src/client/bsk-ui.nomodule.css
|
||||
|
||||
+9
-5
@@ -34,13 +34,17 @@ where only `extension connected` fails is expected; go to Step 3.
|
||||
## 3. Open the extension install page
|
||||
|
||||
If `extension connected` is `FAIL` (`0 browsers connected`), the user likely
|
||||
has not installed or enabled the browser extension yet. Open the
|
||||
[Chrome Web Store install page](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||
in the user's browser, then tell the user:
|
||||
has not installed or enabled the browser extension yet. Open the install page
|
||||
matching the user's browser —
|
||||
[Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||
for Chrome and other Chromium browsers,
|
||||
[Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg)
|
||||
for Microsoft Edge — then tell the user:
|
||||
|
||||
> Install **BrowserSkill** from the
|
||||
> [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi),
|
||||
> open the popup, and wait until it turns green. Reply when done.
|
||||
> [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||
> (or [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg)
|
||||
> on Edge), open the popup, and wait until it turns green. Reply when done.
|
||||
|
||||
If opening the page fails, give the user the same link. Then run `bsk doctor`
|
||||
once more. All `ok`/`na` → tell the user it's ready.
|
||||
|
||||
Generated
+2
-2
@@ -134,7 +134,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bsk"
|
||||
version = "0.1.9"
|
||||
version = "0.1.10"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bsk-protocol",
|
||||
@@ -168,7 +168,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "bsk-protocol"
|
||||
version = "0.1.9"
|
||||
version = "0.1.10"
|
||||
dependencies = [
|
||||
"schemars",
|
||||
"semver",
|
||||
|
||||
+1
-1
@@ -7,7 +7,7 @@ members = [
|
||||
default-members = ["crates/bsk-cli"]
|
||||
|
||||
[workspace.package]
|
||||
version = "0.1.9"
|
||||
version = "0.1.10"
|
||||
edition = "2024"
|
||||
rust-version = "1.85"
|
||||
license = "MIT"
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
</p>
|
||||
|
||||
**BrowserSkill** connects Cursor, Claude Code, Codex, OpenClaw, CodeBuddy,
|
||||
WorkBuddy, Pi, Hermes Agent, and other shell-capable AI agents to your already logged-in
|
||||
WorkBuddy, Pi, Hermes Agent, DeepSeek Harness, and other AI agents to your already logged-in
|
||||
browser.
|
||||
|
||||
Need the agent to touch a tab you already have open? It must borrow that tab
|
||||
@@ -67,7 +67,8 @@ Set up browser-skill on this machine by following https://raw.githubusercontent.
|
||||
|
||||
<br>
|
||||
|
||||
Install the CLI, then install the extension from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi).
|
||||
Install the CLI, then install the extension from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||
or [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg).
|
||||
|
||||
#### 1. Install the `bsk` CLI
|
||||
|
||||
@@ -91,7 +92,14 @@ bsk --version
|
||||
|
||||
#### 2. Install the browser extension
|
||||
|
||||
Install BrowserSkill from the [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi).
|
||||
Install BrowserSkill from your browser's store:
|
||||
|
||||
| Browser | Store listing |
|
||||
| --- | --- |
|
||||
| Chrome | [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) |
|
||||
| Microsoft Edge | [Edge Add-ons](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) |
|
||||
|
||||
On other Chromium-based browsers, install the Chrome Web Store build.
|
||||
|
||||
#### 3. Install the skill
|
||||
|
||||
@@ -123,7 +131,8 @@ internal variants and install paths.
|
||||
|
||||
Other shell-capable agent harnesses are supported too. Copy
|
||||
[`skill/SKILL.md`](skill/SKILL.md) into your harness's skills directory as
|
||||
`browser-skill/SKILL.md` to install the skill manually.
|
||||
`browser-skill/SKILL.md` to install the skill manually. DeepSeek Harness uses a
|
||||
dedicated plugin instead — see [DeepSeek Harness plugin](#deepseek-harness-plugin).
|
||||
|
||||
</details>
|
||||
|
||||
@@ -133,6 +142,26 @@ Start a new Agent session and write a prompt that needs the browser, for example
|
||||
/browser-skill open example.com and summarize what is on the page.
|
||||
```
|
||||
|
||||
## DeepSeek Harness plugin
|
||||
|
||||
Using [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness) (`dsh`)?
|
||||
BrowserSkill ships a first-class dsh plugin on npm as
|
||||
[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin).
|
||||
It injects native `browser_*` tools (no shelling out to `bsk`) and a live Web UI
|
||||
overlay of each Agent Window.
|
||||
|
||||
Add it to a dsh profile, then start that profile:
|
||||
|
||||
```sh
|
||||
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
|
||||
dsh --profile web
|
||||
```
|
||||
|
||||
The plugin carries its own copy of the skill, so `bsk install-skill` is not needed
|
||||
for dsh — but the `bsk` CLI and the browser extension are still prerequisites. See
|
||||
the [plugin README](packages/dsh-plugin-browserskill/README.md) for the tool list,
|
||||
configuration, and the observation overlay.
|
||||
|
||||
## How It Works
|
||||
|
||||
BrowserSkill is a local bridge between your agent harness and your browser.
|
||||
@@ -166,7 +195,9 @@ flowchart TB
|
||||
|
||||
The agent never talks to the browser directly. It asks the `bsk` CLI to perform a
|
||||
browser task; the local daemon routes that request to the extension; the
|
||||
extension runs it in an Agent Window.
|
||||
extension runs it in an Agent Window. DeepSeek Harness takes the same path
|
||||
through the [plugin](#deepseek-harness-plugin): the agent calls injected
|
||||
`browser_*` tools, and the plugin invokes `bsk` on its behalf.
|
||||
|
||||
## For Developers
|
||||
|
||||
@@ -176,6 +207,7 @@ The repository is a Cargo + pnpm workspace:
|
||||
- `crates/bsk-protocol` — shared wire types and JSON schemas
|
||||
- `apps/extension` — browser extension
|
||||
- `packages/ui` and `packages/i18n` — shared extension UI support
|
||||
- `packages/dsh-plugin-browserskill` — DeepSeek Harness plugin (`@wxg-prc-cpg/browser-skill-dsh-plugin`)
|
||||
|
||||
## License
|
||||
|
||||
|
||||
+32
-7
@@ -12,7 +12,7 @@
|
||||
<a href="README.md">English</a> · 中文
|
||||
</p>
|
||||
|
||||
**BrowserSkill** 把 Cursor、Claude Code、Codex、OpenClaw、CodeBuddy、WorkBuddy、Pi、Hermes Agent 等支持 Shell 的 AI Agent 连接到你已登录的浏览器。
|
||||
**BrowserSkill** 把 Cursor、Claude Code、Codex、OpenClaw、CodeBuddy、WorkBuddy、Pi、Hermes Agent、DeepSeek Harness 等 AI Agent 连接到你已登录的浏览器。
|
||||
|
||||
需要 Agent 操作你已打开的标签页?必须显式借用该标签,任务结束后归还,其余浏览器窗口不受影响。
|
||||
|
||||
@@ -54,7 +54,8 @@ BrowserSkill 由两个本地运行组件组成:`bsk` CLI/daemon 和浏览器
|
||||
|
||||
<br>
|
||||
|
||||
先安装 CLI,再从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) 安装浏览器扩展。
|
||||
先安装 CLI,再从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi)
|
||||
或 [Edge 加载项商店](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) 安装浏览器扩展。
|
||||
|
||||
#### 1. 安装 `bsk` CLI
|
||||
|
||||
@@ -64,8 +65,11 @@ BrowserSkill 由两个本地运行组件组成:`bsk` CLI/daemon 和浏览器
|
||||
curl -fsSL https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.sh | sh
|
||||
```
|
||||
|
||||
**Windows**:从 [最新 CLI release](https://github.com/Tencent/BrowserSkill/releases/latest)
|
||||
下载 `bsk-v<version>-x86_64-pc-windows-msvc.zip`,解压后将 `bsk.exe` 加入 `PATH`。
|
||||
**Windows**(PowerShell,安装到 `~/.local/bin`):
|
||||
|
||||
```powershell
|
||||
irm https://raw.githubusercontent.com/Tencent/BrowserSkill/main/install.ps1 | iex
|
||||
```
|
||||
|
||||
验证二进制:
|
||||
|
||||
@@ -75,7 +79,14 @@ bsk --version
|
||||
|
||||
#### 2. 安装浏览器扩展
|
||||
|
||||
从 [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) 安装 BrowserSkill。
|
||||
在对应浏览器的商店安装 BrowserSkill:
|
||||
|
||||
| 浏览器 | 商店页面 |
|
||||
| --- | --- |
|
||||
| Chrome | [Chrome Web Store](https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi) |
|
||||
| Microsoft Edge | [Edge 加载项商店](https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg) |
|
||||
|
||||
其他基于 Chromium 的浏览器,安装 Chrome Web Store 版本即可。
|
||||
|
||||
#### 3. 安装 skill
|
||||
|
||||
@@ -102,7 +113,7 @@ bsk install-skill
|
||||
|
||||
用 <kbd>Space</kbd> 选择需要安装的 Agent harness,然后按 <kbd>Enter</kbd> 安装 skill。运行 `bsk install-skill --list` 可查看 internal 变体及安装路径。
|
||||
|
||||
其他支持 Shell 的 Agent harness 也可使用 BrowserSkill,但需手动将 [`skill/SKILL.md`](skill/SKILL.md) 复制到对应 skills 目录下的 `browser-skill/SKILL.md`。
|
||||
其他支持 Shell 的 Agent harness 也可使用 BrowserSkill,但需手动将 [`skill/SKILL.md`](skill/SKILL.md) 复制到对应 skills 目录下的 `browser-skill/SKILL.md`。DeepSeek Harness 走独立插件,见 [DeepSeek Harness 插件](#deepseek-harness-插件)。
|
||||
|
||||
</details>
|
||||
|
||||
@@ -112,6 +123,19 @@ bsk install-skill
|
||||
/browser-skill open example.com and summarize what is on the page.
|
||||
```
|
||||
|
||||
## DeepSeek Harness 插件
|
||||
|
||||
在用 [DeepSeek Harness](https://github.com/deepseek-ai/deepseek-harness)(`dsh`)?BrowserSkill 提供了官方 dsh 插件,已发布到 npm:[`@wxg-prc-cpg/browser-skill-dsh-plugin`](https://www.npmjs.com/package/@wxg-prc-cpg/browser-skill-dsh-plugin)。它会注入原生 `browser_*` 工具(无需再通过 Shell 调用 `bsk`),并在 Web UI 中实时观察每个 Agent Window。
|
||||
|
||||
把它装进某个 dsh profile,然后启动该 profile:
|
||||
|
||||
```sh
|
||||
dsh plugin --profile web add @wxg-prc-cpg/browser-skill-dsh-plugin
|
||||
dsh --profile web
|
||||
```
|
||||
|
||||
插件自带 skill,所以在 dsh 下无需执行 `bsk install-skill`;但 `bsk` CLI 和浏览器扩展仍是前置条件。工具清单、配置项与观察浮层见[插件 README](packages/dsh-plugin-browserskill/README.md)。
|
||||
|
||||
## 工作原理
|
||||
|
||||
BrowserSkill 是 Agent 运行时与浏览器之间的本地桥接层。
|
||||
@@ -143,7 +167,7 @@ flowchart TB
|
||||
style UserWindows fill:#f8fafc,stroke:#cbd5e1,color:#334155
|
||||
```
|
||||
|
||||
Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;本地 daemon 把请求路由到扩展;扩展在 Agent Window 中执行。
|
||||
Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;本地 daemon 把请求路由到扩展;扩展在 Agent Window 中执行。DeepSeek Harness 走同一条链路,只是经由 [插件](#deepseek-harness-插件):Agent 调用注入的 `browser_*` 工具,由插件代为执行 `bsk`。
|
||||
|
||||
## 面向开发者
|
||||
|
||||
@@ -153,6 +177,7 @@ Agent 不直接与浏览器通信。它通过 `bsk` CLI 下发浏览器任务;
|
||||
- `crates/bsk-protocol` — 共享协议类型与 JSON Schema
|
||||
- `apps/extension` — 浏览器扩展
|
||||
- `packages/ui` 和 `packages/i18n` — 扩展 UI 共享支持
|
||||
- `packages/dsh-plugin-browserskill` — DeepSeek Harness 插件(`@wxg-prc-cpg/browser-skill-dsh-plugin`)
|
||||
|
||||
## 许可证
|
||||
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
<svg width="24" height="24" viewBox="0 0 24 24" fill="none" xmlns="http://www.w3.org/2000/svg">
|
||||
<path d="M6.30589 17.9712C6.21652 18.4603 6.63298 18.8722 7.12749 18.7838C7.88158 18.6491 8.86635 18.4075 9.25454 18.0236C9.49544 17.7853 9.61164 17.4432 9.57757 17.0726C9.5435 16.702 9.36195 16.3331 9.07287 16.0472C8.78378 15.7613 8.41084 15.5818 8.03608 15.5481C7.66133 15.5144 7.31545 15.6293 7.07455 15.8676C6.6865 16.2513 6.4422 17.2253 6.30589 17.9712Z" fill="#E94500"/>
|
||||
<path d="M7.77425 8.64281C10.7374 4.12636 15.4876 3.7066 17.9364 4.13499C18.3335 4.20461 18.6474 4.48261 18.7821 4.84105C20.6715 9.87191 18.4167 13.7056 16.5008 15.5334C16.1262 15.8909 15.9395 16.3745 16.0282 16.8215L16.12 17.2834C16.24 17.887 15.871 18.4819 15.2723 18.6496L12.7811 19.3469C12.1057 19.536 11.4124 19.1112 11.285 18.4289L11.036 17.0959C10.9844 16.82 10.8353 16.5712 10.6151 16.3948L8.12777 14.4035C7.90665 14.2266 7.62818 14.1345 7.34359 14.1428L5.72835 14.1897C4.9877 14.2112 4.40832 13.5657 4.51644 12.8401L4.89632 10.2912C4.98194 9.71734 5.47627 9.2895 6.06234 9.28148L6.69124 9.27171C7.11716 9.26594 7.52243 9.02649 7.77425 8.64281ZM14.4959 7.91429C13.8284 7.91436 13.2869 8.45568 13.2869 9.12327C13.2871 9.79078 13.8284 10.3322 14.4959 10.3323C15.1635 10.3323 15.7048 9.79082 15.7049 9.12327C15.7049 8.45564 15.1636 7.91429 14.4959 7.91429Z" fill="#E94500"/>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 1.3 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.9 KiB |
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@browser-skill/extension",
|
||||
"version": "0.1.5",
|
||||
"version": "0.1.6",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"scripts": {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { type CdpDebuggerApi, ChromiumCdp } from "../chromium-cdp";
|
||||
import { type CdpDebuggee, type CdpDebuggerApi, ChromiumCdp } from "../chromium-cdp";
|
||||
|
||||
function fakeChromeEvent<TArgs extends unknown[]>() {
|
||||
const listeners = new Set<(...args: TArgs) => void>();
|
||||
@@ -14,7 +14,7 @@ function fakeChromeEvent<TArgs extends unknown[]>() {
|
||||
}
|
||||
|
||||
function fakeApi() {
|
||||
const onEvent = fakeChromeEvent<[chrome.debugger.Debuggee, string, unknown]>();
|
||||
const onEvent = fakeChromeEvent<[CdpDebuggee, string, unknown]>();
|
||||
const onDetach = fakeChromeEvent<[chrome.debugger.Debuggee, string]>();
|
||||
const api: CdpDebuggerApi = {
|
||||
attach: vi.fn(async () => {}),
|
||||
@@ -29,6 +29,135 @@ function fakeApi() {
|
||||
}
|
||||
|
||||
describe("ChromiumCdp", () => {
|
||||
it("discovers multiple iframe targets and recursively routes nested OOPIF commands", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
async (
|
||||
target: chrome.debugger.Debuggee & { sessionId?: string },
|
||||
method: string,
|
||||
params?: { frameId?: string },
|
||||
) => {
|
||||
if (method === "DOM.getFrameOwner") {
|
||||
return { backendNodeId: params?.frameId === "nested" ? 300 : 200 };
|
||||
}
|
||||
if (method !== "Page.getFrameTree") return {};
|
||||
if (target.sessionId === "right-session") {
|
||||
return {
|
||||
frameTree: {
|
||||
frame: { id: "right", url: "https://right.test" },
|
||||
childFrames: [
|
||||
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
||||
],
|
||||
},
|
||||
};
|
||||
}
|
||||
if (target.sessionId === "nested-session") {
|
||||
return { frameTree: { frame: { id: "nested", url: "https://nested.test" } } };
|
||||
}
|
||||
return {
|
||||
frameTree: {
|
||||
frame: { id: "main", url: "https://app.test" },
|
||||
childFrames: [
|
||||
{ frame: { id: "left", parentId: "main", url: "https://left.test" } },
|
||||
{
|
||||
frame: { id: "right", parentId: "main", url: "https://right.test" },
|
||||
childFrames: [
|
||||
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
},
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
await cdp.ensureAttached(4);
|
||||
onEvent.fire({ tabId: 4 }, "Target.attachedToTarget", { sessionId: "right-session" });
|
||||
onEvent.fire({ tabId: 4 } as chrome.debugger.Debuggee, "Target.attachedToTarget", {
|
||||
sessionId: "nested-session",
|
||||
});
|
||||
|
||||
const graph = await cdp.getFrameGraph(4);
|
||||
|
||||
expect(graph.frames).toHaveLength(4);
|
||||
expect(graph.frames.find((frame) => frame.frameId === "left")?.target).toEqual({ tabId: 4 });
|
||||
expect(graph.frames.find((frame) => frame.frameId === "right")?.target.sessionId).toBe(
|
||||
"right-session",
|
||||
);
|
||||
expect(graph.frames.find((frame) => frame.frameId === "nested")?.target.sessionId).toBe(
|
||||
"nested-session",
|
||||
);
|
||||
expect(graph.frames.find((frame) => frame.frameId === "right")?.ownerBackendNodeId).toBe(200);
|
||||
expect(graph.frames.find((frame) => frame.frameId === "nested")?.ownerBackendNodeId).toBe(300);
|
||||
expect(api.sendCommand).toHaveBeenCalledWith(
|
||||
{ tabId: 4, sessionId: "right-session" },
|
||||
"DOM.getFrameOwner",
|
||||
{ frameId: "nested" },
|
||||
);
|
||||
expect(api.sendCommand).toHaveBeenCalledWith(
|
||||
{ tabId: 4, sessionId: "nested-session" },
|
||||
"Target.setAutoAttach",
|
||||
expect.objectContaining({ flatten: true }),
|
||||
);
|
||||
});
|
||||
|
||||
it("waits for recursively attached iframe targets without a depth limit", async () => {
|
||||
const { api, onEvent } = fakeApi();
|
||||
const depth = 6;
|
||||
const scheduled = new Set<string>();
|
||||
const nestedTree = (index: number): Record<string, unknown> => ({
|
||||
frame: {
|
||||
id: index === 0 ? "main" : `frame-${index}`,
|
||||
...(index > 0 ? { parentId: index === 1 ? "main" : `frame-${index - 1}` } : {}),
|
||||
},
|
||||
...(index < depth ? { childFrames: [nestedTree(index + 1)] } : {}),
|
||||
});
|
||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
async (target: CdpDebuggee, method: string, params?: { frameId?: string }) => {
|
||||
if (method === "Target.setAutoAttach") {
|
||||
const parentIndex = target.sessionId
|
||||
? Number(target.sessionId.replace("session-", ""))
|
||||
: 0;
|
||||
const nextIndex = parentIndex + 1;
|
||||
const key = `${target.sessionId ?? "root"}:${nextIndex}`;
|
||||
if (nextIndex <= depth && !scheduled.has(key)) {
|
||||
scheduled.add(key);
|
||||
setTimeout(() => {
|
||||
onEvent.fire(
|
||||
{ tabId: 4, ...(target.sessionId ? { sessionId: target.sessionId } : {}) },
|
||||
"Target.attachedToTarget",
|
||||
{
|
||||
sessionId: `session-${nextIndex}`,
|
||||
targetInfo: { type: "iframe" },
|
||||
},
|
||||
);
|
||||
}, 5);
|
||||
}
|
||||
return {};
|
||||
}
|
||||
if (method === "Page.getFrameTree") {
|
||||
if (!target.sessionId) return { frameTree: nestedTree(0) };
|
||||
const index = Number(target.sessionId.replace("session-", ""));
|
||||
return { frameTree: nestedTree(index) };
|
||||
}
|
||||
if (method === "DOM.getFrameOwner") {
|
||||
return { backendNodeId: Number(params?.frameId?.replace("frame-", "")) + 100 };
|
||||
}
|
||||
return {};
|
||||
},
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
|
||||
const graph = await cdp.getFrameGraph(4);
|
||||
|
||||
expect(graph.frames).toHaveLength(depth + 1);
|
||||
for (let index = 1; index <= depth; index += 1) {
|
||||
expect(
|
||||
graph.frames.find((frame) => frame.frameId === `frame-${index}`)?.target.sessionId,
|
||||
).toBe(`session-${index}`);
|
||||
}
|
||||
});
|
||||
|
||||
it("coalesces concurrent attach calls for the same tab", async () => {
|
||||
const { api } = fakeApi();
|
||||
let releaseAttach!: () => void;
|
||||
@@ -67,6 +196,30 @@ describe("ChromiumCdp", () => {
|
||||
expect(cdp.isAttached(5)).toBe(false);
|
||||
});
|
||||
|
||||
it("rolls back the raw attach when a domain enable fails so the tab is not left stuck", async () => {
|
||||
const { api } = fakeApi();
|
||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
async (_target, method: string) => {
|
||||
if (method === "Page.enable") throw new Error("Page.enable rejected");
|
||||
return {};
|
||||
},
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
|
||||
// Raw attach succeeds, but Page.enable fails. Without the rollback
|
||||
// the debugger would stay attached while `attachedTabs` omits the id,
|
||||
// leaving the tab stuck on "Another debugger is already attached" for
|
||||
// every later call until the extension is reloaded.
|
||||
await expect(cdp.ensureAttached(42)).rejects.toThrow(/Page\.enable rejected/);
|
||||
expect(cdp.isAttached(42)).toBe(false);
|
||||
expect(api.detach).toHaveBeenCalledWith({ tabId: 42 });
|
||||
|
||||
// The rollback detached, so a fresh attach can succeed afterwards.
|
||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockResolvedValue({});
|
||||
await expect(cdp.ensureAttached(42)).resolves.toBeUndefined();
|
||||
expect(cdp.isAttached(42)).toBe(true);
|
||||
});
|
||||
|
||||
it("send() rejects on chrome.runtime.lastError-style failures", async () => {
|
||||
const { api } = fakeApi();
|
||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
@@ -132,11 +285,33 @@ describe("ChromiumCdp", () => {
|
||||
expect(cdp.isAttached(12)).toBe(true);
|
||||
expect(api.sendCommand).toHaveBeenCalledWith({ tabId: 12 }, "Runtime.enable", {});
|
||||
expect(api.sendCommand).toHaveBeenCalledWith({ tabId: 12 }, "Log.enable", {});
|
||||
await cdp.ensureConsoleCapture(12);
|
||||
const enableCalls = (api.sendCommand as ReturnType<typeof vi.fn>).mock.calls.filter(
|
||||
});
|
||||
|
||||
it("retries console capture after a domain enable fails during attach", async () => {
|
||||
const { api } = fakeApi();
|
||||
(api.sendCommand as ReturnType<typeof vi.fn>).mockImplementation(
|
||||
async (_target, method: string) => {
|
||||
if (method === "Log.enable") throw new Error("restricted");
|
||||
return {};
|
||||
},
|
||||
);
|
||||
const cdp = new ChromiumCdp(api);
|
||||
// Attach is best-effort: Log.enable fails but attach still succeeds,
|
||||
// leaving the tab unmarked so capture can be retried.
|
||||
await cdp.ensureAttached(12);
|
||||
const afterAttach = (api.sendCommand as ReturnType<typeof vi.fn>).mock.calls.filter(
|
||||
([, method]) => method === "Runtime.enable" || method === "Log.enable",
|
||||
);
|
||||
expect(enableCalls).toHaveLength(2);
|
||||
expect(afterAttach).toHaveLength(2);
|
||||
|
||||
// Before the fix the "attempted" flag was set before success, so this
|
||||
// was a no-op and the tab silently returned no console output forever.
|
||||
// Now it retries both domains.
|
||||
await cdp.ensureConsoleCapture(12);
|
||||
const afterRetry = (api.sendCommand as ReturnType<typeof vi.fn>).mock.calls.filter(
|
||||
([, method]) => method === "Runtime.enable" || method === "Log.enable",
|
||||
);
|
||||
expect(afterRetry).toHaveLength(4);
|
||||
});
|
||||
|
||||
it("retries and surfaces Network.enable failures for explicit capture", async () => {
|
||||
|
||||
@@ -0,0 +1,136 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildFrameGraph, type CdpFrameTreeNode, type CdpFrameTreeSource } from "../frame-graph";
|
||||
|
||||
describe("buildFrameGraph", () => {
|
||||
it("keeps sibling frames distinct and routes nested OOPIFs to their child sessions", () => {
|
||||
const graph = buildFrameGraph([
|
||||
{
|
||||
target: { tabId: 4 },
|
||||
tree: {
|
||||
frame: { id: "main", url: "https://app.test" },
|
||||
childFrames: [
|
||||
{ frame: { id: "left", parentId: "main", url: "https://left.test" } },
|
||||
{
|
||||
frame: { id: "right", parentId: "main", url: "https://right.test" },
|
||||
childFrames: [
|
||||
{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } },
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
target: { tabId: 4, sessionId: "right-session" },
|
||||
tree: {
|
||||
frame: { id: "right", url: "https://right.test" },
|
||||
childFrames: [{ frame: { id: "nested", parentId: "right", url: "https://nested.test" } }],
|
||||
},
|
||||
},
|
||||
{
|
||||
target: { tabId: 4, sessionId: "nested-session" },
|
||||
tree: { frame: { id: "nested", url: "https://nested.test" } },
|
||||
},
|
||||
]);
|
||||
|
||||
expect(graph?.frames).toHaveLength(4);
|
||||
expect(graph?.frames.find((frame) => frame.frameId === "left")?.target).toEqual({ tabId: 4 });
|
||||
expect(graph?.frames.find((frame) => frame.frameId === "right")).toMatchObject({
|
||||
parentFrameId: "main",
|
||||
target: { tabId: 4, sessionId: "right-session" },
|
||||
});
|
||||
expect(graph?.frames.find((frame) => frame.frameId === "nested")).toMatchObject({
|
||||
parentFrameId: "right",
|
||||
target: { tabId: 4, sessionId: "nested-session" },
|
||||
});
|
||||
});
|
||||
|
||||
it("propagates each target boundary to its same-process descendants", () => {
|
||||
const sources: CdpFrameTreeSource[] = [
|
||||
{
|
||||
target: { tabId: 4 },
|
||||
tree: {
|
||||
frame: { id: "main" },
|
||||
childFrames: [
|
||||
{
|
||||
frame: { id: "oopif-a", parentId: "main" },
|
||||
childFrames: [
|
||||
{
|
||||
frame: { id: "same-process-b", parentId: "oopif-a" },
|
||||
childFrames: [
|
||||
{
|
||||
frame: { id: "oopif-c", parentId: "same-process-b" },
|
||||
childFrames: [{ frame: { id: "same-process-d", parentId: "oopif-c" } }],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
target: { tabId: 4, sessionId: "session-a" },
|
||||
tree: {
|
||||
frame: { id: "oopif-a" },
|
||||
childFrames: [
|
||||
{
|
||||
frame: { id: "same-process-b", parentId: "oopif-a" },
|
||||
childFrames: [{ frame: { id: "oopif-c", parentId: "same-process-b" } }],
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
{
|
||||
target: { tabId: 4, sessionId: "session-c" },
|
||||
tree: {
|
||||
frame: { id: "oopif-c" },
|
||||
childFrames: [{ frame: { id: "same-process-d", parentId: "oopif-c" } }],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
const graph = buildFrameGraph(sources);
|
||||
|
||||
expect(graph?.frames.map((frame) => [frame.frameId, frame.target.sessionId])).toEqual([
|
||||
["main", undefined],
|
||||
["oopif-a", "session-a"],
|
||||
["same-process-b", "session-a"],
|
||||
["oopif-c", "session-c"],
|
||||
["same-process-d", "session-c"],
|
||||
]);
|
||||
expect(graph?.frames.find((frame) => frame.frameId === "oopif-a")?.parentFrameId).toBe("main");
|
||||
expect(graph?.frames.find((frame) => frame.frameId === "oopif-c")?.parentFrameId).toBe(
|
||||
"same-process-b",
|
||||
);
|
||||
|
||||
const reordered = buildFrameGraph([sources[2], sources[1], sources[0]]);
|
||||
expect(reordered).toEqual(graph);
|
||||
});
|
||||
|
||||
it("walks deeply nested frame trees without consuming the JavaScript call stack", () => {
|
||||
const depth = 5_000;
|
||||
const root: CdpFrameTreeNode = { frame: { id: "frame-0" } };
|
||||
let parent = root;
|
||||
for (let index = 1; index <= depth; index += 1) {
|
||||
const child: CdpFrameTreeNode = {
|
||||
frame: { id: `frame-${index}`, parentId: `frame-${index - 1}` },
|
||||
};
|
||||
parent.childFrames = [child];
|
||||
parent = child;
|
||||
}
|
||||
|
||||
const graph = buildFrameGraph([{ target: { tabId: 4 }, tree: root }]);
|
||||
|
||||
expect(graph?.frames).toHaveLength(depth + 1);
|
||||
expect(graph?.frames.at(-1)?.frameId).toBe(`frame-${depth}`);
|
||||
});
|
||||
|
||||
it("does not loop when an in-memory frame tree contains an object cycle", () => {
|
||||
const root: CdpFrameTreeNode = { frame: { id: "main" } };
|
||||
root.childFrames = [root];
|
||||
|
||||
expect(buildFrameGraph([{ target: { tabId: 4 }, tree: root }])?.frames).toEqual([
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -31,6 +31,15 @@ import type {
|
||||
NetworkEntryKind,
|
||||
NetworkResult,
|
||||
} from "@/transport/types";
|
||||
import {
|
||||
buildFrameGraph,
|
||||
type CdpFrameGraph,
|
||||
type CdpFrameTreeNode,
|
||||
type CdpFrameTreeSource,
|
||||
type CdpTarget,
|
||||
} from "./frame-graph";
|
||||
|
||||
export type CdpDebuggee = chrome.debugger.Debuggee & { sessionId?: string };
|
||||
|
||||
/**
|
||||
* Minimal slice of `chrome.debugger` the rest of the extension
|
||||
@@ -38,21 +47,15 @@ import type {
|
||||
* fake without monkey-patching the real `chrome` global.
|
||||
*/
|
||||
export interface CdpDebuggerApi {
|
||||
attach(target: chrome.debugger.Debuggee, requiredVersion: string): Promise<void>;
|
||||
detach(target: chrome.debugger.Debuggee): Promise<void>;
|
||||
sendCommand(
|
||||
target: chrome.debugger.Debuggee,
|
||||
method: string,
|
||||
commandParams?: object,
|
||||
): Promise<unknown>;
|
||||
attach(target: CdpDebuggee, requiredVersion: string): Promise<void>;
|
||||
detach(target: CdpDebuggee): Promise<void>;
|
||||
sendCommand(target: CdpDebuggee, method: string, commandParams?: object): Promise<unknown>;
|
||||
/**
|
||||
* Fires for every CDP event (`Page.lifecycleEvent`, `DOM.documentUpdated`,
|
||||
* …). The first callback argument is the source debuggee; the second
|
||||
* is the CDP method name; the third is the payload.
|
||||
*/
|
||||
onEvent: chrome.events.Event<
|
||||
(source: chrome.debugger.Debuggee, method: string, params: unknown) => void
|
||||
>;
|
||||
onEvent: chrome.events.Event<(source: CdpDebuggee, method: string, params: unknown) => void>;
|
||||
/**
|
||||
* Fires when Chrome unilaterally detaches us — most commonly because
|
||||
* the tab navigated to a chrome:// URL or the user clicked
|
||||
@@ -110,6 +113,8 @@ const MAX_CONSOLE_STACK_FRAMES = 20;
|
||||
const MAX_NETWORK_BUFFER = 200;
|
||||
const MAX_NETWORK_FIELD_LENGTH = 4096;
|
||||
const MAX_NETWORK_REQUEST_META = 1024;
|
||||
const FRAME_DISCOVERY_TIMEOUT_MS = 1000;
|
||||
const FRAME_DISCOVERY_QUIET_MS = 20;
|
||||
|
||||
interface ParsedDialogOpening {
|
||||
type: JavaScriptDialogType;
|
||||
@@ -131,6 +136,27 @@ interface NetworkRequestMeta {
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
interface FrameDiscoveryState {
|
||||
sessions: Set<string>;
|
||||
pending: Set<Promise<void>>;
|
||||
generation: number;
|
||||
}
|
||||
|
||||
async function settleBeforeDeadline(promises: Promise<void>[], deadline: number): Promise<boolean> {
|
||||
if (promises.length === 0) return true;
|
||||
const remaining = deadline - Date.now();
|
||||
if (remaining <= 0) return false;
|
||||
let timer: ReturnType<typeof setTimeout> | undefined;
|
||||
const timedOut = await Promise.race([
|
||||
Promise.allSettled(promises).then(() => false),
|
||||
new Promise<boolean>((resolve) => {
|
||||
timer = setTimeout(() => resolve(true), remaining);
|
||||
}),
|
||||
]);
|
||||
if (timer) clearTimeout(timer);
|
||||
return !timedOut;
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrapper around `chrome.debugger` that owns the "attach once per
|
||||
* tabId" cache and exposes typed `send<T>()`.
|
||||
@@ -144,15 +170,17 @@ export class ChromiumCdp {
|
||||
private readonly dialogSequences = new Map<number, number>();
|
||||
private readonly consoleBuffers = new Map<number, ConsoleEntry[]>();
|
||||
private readonly consoleSequences = new Map<number, number>();
|
||||
private readonly consoleDomainsAttemptedTabs = new Set<number>();
|
||||
private readonly consoleDomainsEnabledTabs = new Set<number>();
|
||||
private readonly networkBuffers = new Map<number, NetworkEntry[]>();
|
||||
private readonly networkSequences = new Map<number, number>();
|
||||
private readonly networkDomainsEnabledTabs = new Set<number>();
|
||||
private readonly networkRequestMeta = new Map<number, Map<string, NetworkRequestMeta>>();
|
||||
private readonly frameDiscovery = new Map<number, FrameDiscoveryState>();
|
||||
private detachSubscription: { dispose(): void } | null = null;
|
||||
private dialogSubscription: { dispose(): void } | null = null;
|
||||
private consoleSubscription: { dispose(): void } | null = null;
|
||||
private networkSubscription: { dispose(): void } | null = null;
|
||||
private frameTargetSubscription: { dispose(): void } | null = null;
|
||||
|
||||
constructor(api: CdpDebuggerApi = chromeDebuggerApi) {
|
||||
this.api = api;
|
||||
@@ -160,6 +188,7 @@ export class ChromiumCdp {
|
||||
this.bindDialogHandler();
|
||||
this.bindConsoleHandler();
|
||||
this.bindNetworkHandler();
|
||||
this.bindFrameTargetHandler();
|
||||
}
|
||||
|
||||
/** Attach to `tabId` if we haven't already in this driver. */
|
||||
@@ -172,10 +201,29 @@ export class ChromiumCdp {
|
||||
}
|
||||
const attach = (async () => {
|
||||
await this.api.attach({ tabId }, CDP_PROTOCOL_VERSION);
|
||||
await this.enablePageDomain(tabId);
|
||||
await this.enableConsoleDomains(tabId);
|
||||
await this.enableNetworkDomainBestEffort(tabId);
|
||||
this.attachedTabs.add(tabId);
|
||||
try {
|
||||
await this.enablePageDomain(tabId);
|
||||
await this.enableConsoleDomains(tabId);
|
||||
await this.enableNetworkDomainBestEffort(tabId);
|
||||
this.attachedTabs.add(tabId);
|
||||
await this.enableFrameDiscovery({ tabId }).catch((err) => {
|
||||
console.debug("[bsk cdp] frame discovery unavailable", { tabId, err });
|
||||
});
|
||||
} catch (err) {
|
||||
// A CDP domain enable failed after the raw attach succeeded
|
||||
// (e.g. `Page.enable` rejects because the tab just navigated to
|
||||
// a chrome:// URL or the Web Store). `attachedTabs` does not yet
|
||||
// hold `tabId`, so without rolling back the debugger would stay
|
||||
// attached and the next `ensureAttached` would re-attach and hit
|
||||
// "Another debugger is already attached" forever — the tab is
|
||||
// unusable until the extension is reloaded. Detach directly
|
||||
// (`this.detach()` is a no-op here because `attachedTabs` lacks
|
||||
// the id) so the next attempt starts from a clean slate.
|
||||
await this.api.detach({ tabId }).catch((detachErr) => {
|
||||
console.debug("[bsk cdp] rollback detach failed", { tabId, detachErr });
|
||||
});
|
||||
throw err;
|
||||
}
|
||||
})()
|
||||
.catch((err) => {
|
||||
// Chrome surfaces "Another debugger is already attached" when
|
||||
@@ -206,6 +254,73 @@ export class ChromiumCdp {
|
||||
}
|
||||
}
|
||||
|
||||
async sendToTarget<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T> {
|
||||
if (!this.attachedTabs.has(target.tabId)) {
|
||||
await this.ensureAttached(target.tabId);
|
||||
}
|
||||
try {
|
||||
return (await this.api.sendCommand(target, method, params ?? {})) as T;
|
||||
} catch (err) {
|
||||
throw normalizeError(err);
|
||||
}
|
||||
}
|
||||
|
||||
async getFrameGraph(tabId: number): Promise<CdpFrameGraph> {
|
||||
await this.ensureAttached(tabId);
|
||||
await this.enableFrameDiscovery({ tabId }).catch(() => {});
|
||||
await this.drainFrameAttachTasks(tabId);
|
||||
|
||||
const sources: CdpFrameTreeSource[] = [];
|
||||
const root = await this.sendToTarget<{ frameTree?: CdpFrameTreeNode }>(
|
||||
{ tabId },
|
||||
"Page.getFrameTree",
|
||||
{},
|
||||
);
|
||||
if (root.frameTree) sources.push({ target: { tabId }, tree: root.frameTree });
|
||||
|
||||
const sessions = [...(this.frameDiscovery.get(tabId)?.sessions ?? [])];
|
||||
const childTrees = await Promise.all(
|
||||
sessions.map(async (sessionId): Promise<CdpFrameTreeSource | null> => {
|
||||
const target = { tabId, sessionId };
|
||||
try {
|
||||
const reply = await this.sendToTarget<{ frameTree?: CdpFrameTreeNode }>(
|
||||
target,
|
||||
"Page.getFrameTree",
|
||||
{},
|
||||
);
|
||||
return reply.frameTree ? { target, tree: reply.frameTree } : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}),
|
||||
);
|
||||
for (const source of childTrees) {
|
||||
if (source) sources.push(source);
|
||||
}
|
||||
|
||||
const graph = buildFrameGraph(sources);
|
||||
if (!graph) throw new Error("Page.getFrameTree returned no root frame");
|
||||
const frameById = new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
||||
await Promise.all(
|
||||
graph.frames.map(async (frame) => {
|
||||
if (!frame.parentFrameId) return;
|
||||
const parent = frameById.get(frame.parentFrameId);
|
||||
if (!parent) return;
|
||||
try {
|
||||
const owner = await this.sendToTarget<{ backendNodeId?: number }>(
|
||||
parent.target,
|
||||
"DOM.getFrameOwner",
|
||||
{ frameId: frame.frameId },
|
||||
);
|
||||
if (owner.backendNodeId !== undefined) frame.ownerBackendNodeId = owner.backendNodeId;
|
||||
} catch {
|
||||
// The frame may have navigated between tree capture and owner lookup.
|
||||
}
|
||||
}),
|
||||
);
|
||||
return graph;
|
||||
}
|
||||
|
||||
/** Return a cursor marking the current dialog sequence for `tabId`. */
|
||||
dialogCursor(tabId: number): DialogCursor {
|
||||
return this.dialogSequences.get(tabId) ?? 0;
|
||||
@@ -316,6 +431,7 @@ export class ChromiumCdp {
|
||||
this.clearDialogState(tabId);
|
||||
this.clearConsoleState(tabId);
|
||||
this.clearNetworkState(tabId);
|
||||
this.clearFrameState(tabId);
|
||||
try {
|
||||
await this.api.detach({ tabId });
|
||||
} catch (err) {
|
||||
@@ -338,7 +454,7 @@ export class ChromiumCdp {
|
||||
}
|
||||
|
||||
/** Subscribe to all CDP events. Returned disposable removes the listener. */
|
||||
onEvent(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): {
|
||||
onEvent(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
|
||||
dispose(): void;
|
||||
} {
|
||||
this.api.onEvent.addListener(handler);
|
||||
@@ -357,11 +473,12 @@ export class ChromiumCdp {
|
||||
this.dialogSequences.clear();
|
||||
this.consoleBuffers.clear();
|
||||
this.consoleSequences.clear();
|
||||
this.consoleDomainsAttemptedTabs.clear();
|
||||
this.consoleDomainsEnabledTabs.clear();
|
||||
this.networkBuffers.clear();
|
||||
this.networkSequences.clear();
|
||||
this.networkDomainsEnabledTabs.clear();
|
||||
this.networkRequestMeta.clear();
|
||||
this.frameDiscovery.clear();
|
||||
await Promise.all(
|
||||
tabs.map(async (tabId) => {
|
||||
try {
|
||||
@@ -378,15 +495,24 @@ export class ChromiumCdp {
|
||||
}
|
||||
|
||||
private async enableConsoleDomains(tabId: number): Promise<void> {
|
||||
if (this.consoleDomainsAttemptedTabs.has(tabId)) return;
|
||||
this.consoleDomainsAttemptedTabs.add(tabId);
|
||||
if (this.consoleDomainsEnabledTabs.has(tabId)) return;
|
||||
// Mark the tab only after both domains enable successfully — a
|
||||
// transient failure (e.g. restricted page during attach) must leave
|
||||
// the tab unmarked so a later `ensureConsoleCapture` can retry,
|
||||
// instead of silently returning no console output forever. Mirrors
|
||||
// `enableNetworkDomain`, which records the tab after success only.
|
||||
let failed = false;
|
||||
for (const method of ["Runtime.enable", "Log.enable"]) {
|
||||
try {
|
||||
await this.api.sendCommand({ tabId }, method, {});
|
||||
} catch (err) {
|
||||
failed = true;
|
||||
console.debug("[bsk cdp] console domain enable failed", { tabId, method, err });
|
||||
}
|
||||
}
|
||||
if (!failed) {
|
||||
this.consoleDomainsEnabledTabs.add(tabId);
|
||||
}
|
||||
}
|
||||
|
||||
private async enableNetworkDomain(tabId: number): Promise<void> {
|
||||
@@ -403,9 +529,104 @@ export class ChromiumCdp {
|
||||
}
|
||||
}
|
||||
|
||||
private async enableFrameDiscovery(target: CdpTarget): Promise<void> {
|
||||
await this.api.sendCommand(target, "Target.setAutoAttach", {
|
||||
autoAttach: true,
|
||||
waitForDebuggerOnStart: false,
|
||||
flatten: true,
|
||||
filter: [{ type: "iframe", exclude: false }],
|
||||
});
|
||||
}
|
||||
|
||||
private bindFrameTargetHandler(): void {
|
||||
if (this.frameTargetSubscription) return;
|
||||
const listener = (source: CdpDebuggee, method: string, params: unknown) => {
|
||||
const tabId = source.tabId;
|
||||
if (typeof tabId !== "number") return;
|
||||
const raw = (params ?? {}) as Record<string, unknown>;
|
||||
const sessionId = typeof raw.sessionId === "string" ? raw.sessionId : undefined;
|
||||
if (!sessionId) return;
|
||||
|
||||
if (method === "Target.detachedFromTarget") {
|
||||
const state = this.frameDiscovery.get(tabId);
|
||||
if (state?.sessions.delete(sessionId)) state.generation += 1;
|
||||
return;
|
||||
}
|
||||
if (method !== "Target.attachedToTarget") return;
|
||||
const targetInfo = raw.targetInfo as { type?: string } | undefined;
|
||||
if (targetInfo?.type && targetInfo.type !== "iframe") return;
|
||||
|
||||
const state = this.frameDiscoveryState(tabId);
|
||||
if (state.sessions.has(sessionId)) return;
|
||||
state.sessions.add(sessionId);
|
||||
state.generation += 1;
|
||||
|
||||
const task = this.initializeFrameTarget({ tabId, sessionId });
|
||||
state.pending.add(task);
|
||||
void task.finally(() => {
|
||||
state.pending.delete(task);
|
||||
});
|
||||
};
|
||||
this.api.onEvent.addListener(listener);
|
||||
this.frameTargetSubscription = {
|
||||
dispose: () => this.api.onEvent.removeListener(listener),
|
||||
};
|
||||
}
|
||||
|
||||
private async initializeFrameTarget(target: CdpTarget): Promise<void> {
|
||||
try {
|
||||
await this.enableFrameDiscovery(target);
|
||||
} catch (err) {
|
||||
const state = this.frameDiscovery.get(target.tabId);
|
||||
if (state?.sessions.delete(target.sessionId as string)) state.generation += 1;
|
||||
console.debug("[bsk cdp] child frame target initialization failed", { target, err });
|
||||
}
|
||||
}
|
||||
|
||||
private async drainFrameAttachTasks(tabId: number): Promise<void> {
|
||||
const deadline = Date.now() + FRAME_DISCOVERY_TIMEOUT_MS;
|
||||
while (Date.now() < deadline) {
|
||||
const state = this.frameDiscovery.get(tabId);
|
||||
const generation = state?.generation ?? 0;
|
||||
if (state?.pending.size && !(await settleBeforeDeadline([...state.pending], deadline))) break;
|
||||
|
||||
// Target.setAutoAttach may enqueue the next attachedToTarget event after
|
||||
// its command promise settles. Wait for a short quiet window, then finish
|
||||
// only if neither the state object nor its generation changed.
|
||||
const quietTime = Math.min(FRAME_DISCOVERY_QUIET_MS, deadline - Date.now());
|
||||
if (quietTime <= 0) break;
|
||||
await new Promise<void>((resolve) => setTimeout(resolve, quietTime));
|
||||
const current = this.frameDiscovery.get(tabId);
|
||||
if (
|
||||
current === state &&
|
||||
(current?.generation ?? 0) === generation &&
|
||||
!current?.pending.size
|
||||
) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
console.debug("[bsk cdp] frame discovery did not reach quiescence before timeout", { tabId });
|
||||
}
|
||||
|
||||
private frameDiscoveryState(tabId: number): FrameDiscoveryState {
|
||||
const existing = this.frameDiscovery.get(tabId);
|
||||
if (existing) return existing;
|
||||
const created: FrameDiscoveryState = {
|
||||
sessions: new Set(),
|
||||
pending: new Set(),
|
||||
generation: 0,
|
||||
};
|
||||
this.frameDiscovery.set(tabId, created);
|
||||
return created;
|
||||
}
|
||||
|
||||
private clearFrameState(tabId: number): void {
|
||||
this.frameDiscovery.delete(tabId);
|
||||
}
|
||||
|
||||
private bindDialogHandler(): void {
|
||||
if (this.dialogSubscription) return;
|
||||
const listener = (source: chrome.debugger.Debuggee, method: string, params: unknown) => {
|
||||
const listener = (source: CdpDebuggee, method: string, params: unknown) => {
|
||||
if (method !== "Page.javascriptDialogOpening") return;
|
||||
const tabId = source.tabId;
|
||||
if (typeof tabId !== "number") return;
|
||||
@@ -489,7 +710,7 @@ export class ChromiumCdp {
|
||||
private clearConsoleState(tabId: number): void {
|
||||
this.consoleBuffers.delete(tabId);
|
||||
this.consoleSequences.delete(tabId);
|
||||
this.consoleDomainsAttemptedTabs.delete(tabId);
|
||||
this.consoleDomainsEnabledTabs.delete(tabId);
|
||||
}
|
||||
|
||||
private bindNetworkHandler(): void {
|
||||
@@ -589,6 +810,7 @@ export class ChromiumCdp {
|
||||
this.clearDialogState(source.tabId);
|
||||
this.clearConsoleState(source.tabId);
|
||||
this.clearNetworkState(source.tabId);
|
||||
this.clearFrameState(source.tabId);
|
||||
}
|
||||
};
|
||||
this.api.onDetach.addListener(listener);
|
||||
@@ -607,6 +829,8 @@ export class ChromiumCdp {
|
||||
this.consoleSubscription = null;
|
||||
this.networkSubscription?.dispose();
|
||||
this.networkSubscription = null;
|
||||
this.frameTargetSubscription?.dispose();
|
||||
this.frameTargetSubscription = null;
|
||||
}
|
||||
|
||||
/** Detach tabs only when no other live session has claimed them. */
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
export interface CdpTarget {
|
||||
tabId: number;
|
||||
sessionId?: string;
|
||||
}
|
||||
|
||||
export interface CdpFrame {
|
||||
frameId: string;
|
||||
parentFrameId?: string;
|
||||
ownerBackendNodeId?: number;
|
||||
url?: string;
|
||||
target: CdpTarget;
|
||||
}
|
||||
|
||||
export interface CdpFrameGraph {
|
||||
rootFrameId: string;
|
||||
frames: CdpFrame[];
|
||||
}
|
||||
|
||||
export interface CdpFrameTreeNode {
|
||||
frame: {
|
||||
id: string;
|
||||
parentId?: string;
|
||||
url?: string;
|
||||
};
|
||||
childFrames?: CdpFrameTreeNode[];
|
||||
}
|
||||
|
||||
export interface CdpFrameTreeSource {
|
||||
target: CdpTarget;
|
||||
tree: CdpFrameTreeNode;
|
||||
}
|
||||
|
||||
export function cdpTargetKey(target: CdpTarget): string {
|
||||
return `${target.tabId}:${target.sessionId ?? "root"}`;
|
||||
}
|
||||
|
||||
function mergeFrame(
|
||||
frames: Map<string, CdpFrame>,
|
||||
order: string[],
|
||||
node: CdpFrameTreeNode,
|
||||
target: CdpTarget,
|
||||
): void {
|
||||
const existing = frames.get(node.frame.id);
|
||||
if (!existing) order.push(node.frame.id);
|
||||
const parentFrameId = node.frame.parentId || existing?.parentFrameId;
|
||||
const url = node.frame.url || existing?.url;
|
||||
frames.set(node.frame.id, {
|
||||
frameId: node.frame.id,
|
||||
...(parentFrameId ? { parentFrameId } : {}),
|
||||
...(url ? { url } : {}),
|
||||
target,
|
||||
});
|
||||
}
|
||||
|
||||
function walkFrameTree(
|
||||
source: CdpFrameTreeSource,
|
||||
targetByRootFrameId: ReadonlyMap<string, CdpTarget>,
|
||||
frames: Map<string, CdpFrame>,
|
||||
order: string[],
|
||||
): void {
|
||||
const stack: Array<{ node: CdpFrameTreeNode; inheritedTarget: CdpTarget }> = [
|
||||
{ node: source.tree, inheritedTarget: source.target },
|
||||
];
|
||||
const expanded = new WeakSet<CdpFrameTreeNode>();
|
||||
|
||||
while (stack.length > 0) {
|
||||
const current = stack.pop();
|
||||
if (!current || expanded.has(current.node)) continue;
|
||||
expanded.add(current.node);
|
||||
|
||||
const target = targetByRootFrameId.get(current.node.frame.id) ?? current.inheritedTarget;
|
||||
mergeFrame(frames, order, current.node, target);
|
||||
|
||||
const children = current.node.childFrames ?? [];
|
||||
for (let index = children.length - 1; index >= 0; index -= 1) {
|
||||
stack.push({ node: children[index], inheritedTarget: target });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function buildFrameGraph(sources: CdpFrameTreeSource[]): CdpFrameGraph | null {
|
||||
const rootSource = sources.find((source) => source.target.sessionId === undefined) ?? sources[0];
|
||||
if (!rootSource) return null;
|
||||
|
||||
const frames = new Map<string, CdpFrame>();
|
||||
const order: string[] = [];
|
||||
const targetByRootFrameId = new Map(
|
||||
sources.map((source) => [source.tree.frame.id, source.target] as const),
|
||||
);
|
||||
// The top-level source is authoritative if malformed input reports the same
|
||||
// frame as both a root target and a child target.
|
||||
targetByRootFrameId.set(rootSource.tree.frame.id, rootSource.target);
|
||||
|
||||
walkFrameTree(rootSource, targetByRootFrameId, frames, order);
|
||||
for (const source of sources) {
|
||||
if (source === rootSource) continue;
|
||||
walkFrameTree(source, targetByRootFrameId, frames, order);
|
||||
}
|
||||
|
||||
return {
|
||||
rootFrameId: rootSource.tree.frame.id,
|
||||
frames: order.flatMap((frameId) => {
|
||||
const frame = frames.get(frameId);
|
||||
return frame ? [frame] : [];
|
||||
}),
|
||||
};
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
useRef,
|
||||
useState,
|
||||
} from "react";
|
||||
import type { HelpHighlightRect } from "@/lib/help-bridge";
|
||||
import logoUrl from "../../assets/logo.png";
|
||||
|
||||
export interface HelpRequestData {
|
||||
@@ -19,6 +20,10 @@ export interface HelpRequestData {
|
||||
displayMode?: "full" | "compact";
|
||||
/** CSS selectors to scroll to + flash-highlight. */
|
||||
selectors: string[];
|
||||
/** Pre-resolved top-viewport rectangles, used for cross-frame targets. */
|
||||
rects?: HelpHighlightRect[];
|
||||
/** Re-resolve cross-frame rectangles after scroll, resize, or layout changes. */
|
||||
refreshRects?: () => Promise<HelpHighlightRect[] | undefined>;
|
||||
onContinue: (note: string) => void;
|
||||
onCancel: () => void;
|
||||
}
|
||||
@@ -49,6 +54,7 @@ const PANEL_WIDTH = 420;
|
||||
const FALLBACK_PANEL_H = 180;
|
||||
const FALLBACK_PANEL_H_COLLAPSED = 44;
|
||||
const EMPTY_SELECTORS: string[] = [];
|
||||
const EMPTY_RECTS: HelpHighlightRect[] = [];
|
||||
|
||||
function clamp(value: number, min: number, max: number): number {
|
||||
return Math.max(min, Math.min(max, value));
|
||||
@@ -179,6 +185,8 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
const { t } = useTranslation("extension");
|
||||
const isCompact = request?.displayMode === "compact";
|
||||
const effectiveSelectors = isCompact ? EMPTY_SELECTORS : (request?.selectors ?? EMPTY_SELECTORS);
|
||||
const [liveRects, setLiveRects] = useState<HelpHighlightRect[] | null>(null);
|
||||
const effectiveRects = isCompact ? EMPTY_RECTS : (liveRects ?? request?.rects ?? EMPTY_RECTS);
|
||||
const [note, setNote] = useState("");
|
||||
const [collapsed, setCollapsed] = useState(false);
|
||||
const [boxes, setBoxes] = useState<Box[]>([]);
|
||||
@@ -201,6 +209,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
// Reset the note and collapse state whenever a new request appears.
|
||||
useEffect(() => {
|
||||
setNote("");
|
||||
setLiveRects(null);
|
||||
setCollapsed(false);
|
||||
placementRef.current = null;
|
||||
setDragPos(null);
|
||||
@@ -209,6 +218,32 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
dragStartRef.current = null;
|
||||
}, [request?.id]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!request || isCompact || !request.refreshRects) return;
|
||||
let disposed = false;
|
||||
let resolving = false;
|
||||
const refresh = async () => {
|
||||
if (resolving) return;
|
||||
resolving = true;
|
||||
try {
|
||||
const rects = await request.refreshRects?.();
|
||||
if (!disposed && rects) setLiveRects(rects);
|
||||
} finally {
|
||||
resolving = false;
|
||||
}
|
||||
};
|
||||
const onViewportChange = () => void refresh();
|
||||
window.addEventListener("scroll", onViewportChange, true);
|
||||
window.addEventListener("resize", onViewportChange);
|
||||
const interval = window.setInterval(onViewportChange, 500);
|
||||
return () => {
|
||||
disposed = true;
|
||||
window.removeEventListener("scroll", onViewportChange, true);
|
||||
window.removeEventListener("resize", onViewportChange);
|
||||
window.clearInterval(interval);
|
||||
};
|
||||
}, [request, isCompact]);
|
||||
|
||||
// Scroll the first matched target into view once per distinct request id.
|
||||
useLayoutEffect(() => {
|
||||
if (!request) return;
|
||||
@@ -231,7 +266,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
return;
|
||||
}
|
||||
const update = () => {
|
||||
const nextBoxes = measure(effectiveSelectors);
|
||||
const nextBoxes = [...measure(effectiveSelectors), ...effectiveRects];
|
||||
setBoxes((prev) => (boxesEqual(prev, nextBoxes) ? prev : nextBoxes));
|
||||
};
|
||||
update();
|
||||
@@ -243,7 +278,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
window.removeEventListener("resize", update);
|
||||
window.clearInterval(interval);
|
||||
};
|
||||
}, [request, effectiveSelectors]);
|
||||
}, [request, effectiveSelectors, effectiveRects]);
|
||||
|
||||
const onHeaderPointerDown = useCallback(
|
||||
(e: ReactPointerEvent<HTMLDivElement>) => {
|
||||
@@ -305,7 +340,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
return;
|
||||
}
|
||||
if (userMovedRef.current) return;
|
||||
const measured = boxes.length > 0 ? boxes : measure(effectiveSelectors);
|
||||
const measured = boxes.length > 0 ? boxes : [...measure(effectiveSelectors), ...effectiveRects];
|
||||
const anchor = unionBox(measured);
|
||||
const banner = bannerRef.current;
|
||||
if (!anchor || !banner) {
|
||||
@@ -329,7 +364,7 @@ export function HelpRequestOverlay({ request }: Props) {
|
||||
}
|
||||
const nextPos = placePanel(anchor, panelW, panelH, placementRef.current.placement);
|
||||
setPanelPos((prev) => (panelPosEqual(prev, nextPos) ? prev : nextPos));
|
||||
}, [request, boxes, collapsed, isCompact, effectiveSelectors]);
|
||||
}, [request, boxes, collapsed, isCompact, effectiveSelectors, effectiveRects]);
|
||||
|
||||
if (!request) return null;
|
||||
|
||||
|
||||
@@ -1,9 +1,21 @@
|
||||
import { useTranslation } from "@browser-skill/i18n/react";
|
||||
import { RiStopCircleLine } from "@remixicon/react";
|
||||
import { useEffect, useState } from "react";
|
||||
import {
|
||||
type PointerEvent as ReactPointerEvent,
|
||||
useCallback,
|
||||
useEffect,
|
||||
useRef,
|
||||
useState,
|
||||
} from "react";
|
||||
import recordIconUrl from "../../assets/record-icon.png";
|
||||
|
||||
export interface RecordRequestData {
|
||||
id: string;
|
||||
/**
|
||||
* Epoch ms when the recording began. Supplied by the background so the timer
|
||||
* keeps counting the whole session across navigations; falls back to mount
|
||||
* time when the background could not report it.
|
||||
*/
|
||||
startedAtMs?: number;
|
||||
onFinish: () => void;
|
||||
}
|
||||
|
||||
@@ -11,9 +23,51 @@ type Props = {
|
||||
request: RecordRequestData | null;
|
||||
};
|
||||
|
||||
const VIEWPORT_MARGIN = 16;
|
||||
const DRAG_THRESHOLD_PX = 8;
|
||||
const INTRO_DURATION_MS = 10_000;
|
||||
const TIMER_REFRESH_MS = 250;
|
||||
|
||||
function formatElapsed(seconds: number): string {
|
||||
const minutes = Math.floor(seconds / 60);
|
||||
const remainingSeconds = seconds % 60;
|
||||
return `${String(minutes).padStart(2, "0")}:${String(remainingSeconds).padStart(2, "0")}`;
|
||||
}
|
||||
|
||||
function clamp(value: number, min: number, max: number): number {
|
||||
return Math.max(min, Math.min(max, value));
|
||||
}
|
||||
|
||||
function clampDragPos(
|
||||
top: number,
|
||||
left: number,
|
||||
panelW: number,
|
||||
panelH: number,
|
||||
): { top: number; left: number } {
|
||||
const maxLeft = window.innerWidth - panelW - VIEWPORT_MARGIN;
|
||||
const maxTop = window.innerHeight - panelH - VIEWPORT_MARGIN;
|
||||
return {
|
||||
top: clamp(top, VIEWPORT_MARGIN, Math.max(VIEWPORT_MARGIN, maxTop)),
|
||||
left: clamp(left, VIEWPORT_MARGIN, Math.max(VIEWPORT_MARGIN, maxLeft)),
|
||||
};
|
||||
}
|
||||
|
||||
export function RecordOverlay({ request }: Props) {
|
||||
const { t } = useTranslation("extension");
|
||||
const [show, setShow] = useState(false);
|
||||
const [elapsedSeconds, setElapsedSeconds] = useState(0);
|
||||
const [dragPos, setDragPos] = useState<{ top: number; left: number } | null>(null);
|
||||
const [dragging, setDragging] = useState(false);
|
||||
const pillRef = useRef<HTMLDivElement>(null);
|
||||
const dragStartRef = useRef<{
|
||||
pointerX: number;
|
||||
pointerY: number;
|
||||
originLeft: number;
|
||||
originTop: number;
|
||||
panelW: number;
|
||||
panelH: number;
|
||||
armed: boolean;
|
||||
} | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (request) {
|
||||
@@ -23,8 +77,87 @@ export function RecordOverlay({ request }: Props) {
|
||||
setShow(false);
|
||||
}, [request]);
|
||||
|
||||
const requestStartedAtMs = request?.startedAtMs;
|
||||
|
||||
useEffect(() => {
|
||||
setDragPos(null);
|
||||
setDragging(false);
|
||||
dragStartRef.current = null;
|
||||
if (!request) {
|
||||
setElapsedSeconds(0);
|
||||
return;
|
||||
}
|
||||
|
||||
const startedAt = requestStartedAtMs ?? Date.now();
|
||||
const updateElapsed = () => {
|
||||
setElapsedSeconds(Math.max(0, Math.floor((Date.now() - startedAt) / 1_000)));
|
||||
};
|
||||
updateElapsed();
|
||||
const timer = window.setInterval(updateElapsed, TIMER_REFRESH_MS);
|
||||
return () => window.clearInterval(timer);
|
||||
}, [request?.id, requestStartedAtMs]);
|
||||
|
||||
const onPillPointerDown = useCallback((e: ReactPointerEvent<HTMLDivElement>) => {
|
||||
if (e.button !== 0) return;
|
||||
if (e.target instanceof Element && e.target.closest("button, a, input, textarea, select")) {
|
||||
return;
|
||||
}
|
||||
const pill = pillRef.current;
|
||||
if (!pill) return;
|
||||
const rect = pill.getBoundingClientRect();
|
||||
dragStartRef.current = {
|
||||
pointerX: e.clientX,
|
||||
pointerY: e.clientY,
|
||||
originLeft: rect.left,
|
||||
originTop: rect.top,
|
||||
panelW: pill.offsetWidth || rect.width,
|
||||
panelH: pill.offsetHeight || rect.height,
|
||||
armed: false,
|
||||
};
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
const onMove = (e: PointerEvent) => {
|
||||
const start = dragStartRef.current;
|
||||
if (!start) return;
|
||||
const dx = e.clientX - start.pointerX;
|
||||
const dy = e.clientY - start.pointerY;
|
||||
if (!start.armed) {
|
||||
if (Math.hypot(dx, dy) < DRAG_THRESHOLD_PX) return;
|
||||
start.armed = true;
|
||||
setDragging(true);
|
||||
setDragPos({ top: start.originTop, left: start.originLeft });
|
||||
}
|
||||
setDragPos(
|
||||
clampDragPos(start.originTop + dy, start.originLeft + dx, start.panelW, start.panelH),
|
||||
);
|
||||
};
|
||||
const onUp = () => {
|
||||
const start = dragStartRef.current;
|
||||
if (!start) return;
|
||||
if (start.armed) {
|
||||
setDragging(false);
|
||||
}
|
||||
dragStartRef.current = null;
|
||||
};
|
||||
window.addEventListener("pointermove", onMove);
|
||||
window.addEventListener("pointerup", onUp);
|
||||
window.addEventListener("pointercancel", onUp);
|
||||
return () => {
|
||||
window.removeEventListener("pointermove", onMove);
|
||||
window.removeEventListener("pointerup", onUp);
|
||||
window.removeEventListener("pointercancel", onUp);
|
||||
};
|
||||
}, []);
|
||||
|
||||
if (!request) return null;
|
||||
|
||||
const introVisible = elapsedSeconds * 1_000 < INTRO_DURATION_MS;
|
||||
const positioned = dragPos !== null;
|
||||
const defaultTransform = show
|
||||
? "translateX(-50%) translateY(0)"
|
||||
: "translateX(-50%) translateY(8px)";
|
||||
|
||||
return (
|
||||
<>
|
||||
<style>{`
|
||||
@@ -35,78 +168,118 @@ export function RecordOverlay({ request }: Props) {
|
||||
`}</style>
|
||||
|
||||
<div
|
||||
ref={pillRef}
|
||||
data-slot="record-overlay-pill"
|
||||
data-phase={introVisible ? "intro" : "timer"}
|
||||
data-dragging={dragging ? "true" : "false"}
|
||||
onPointerDown={onPillPointerDown}
|
||||
style={{
|
||||
position: "fixed",
|
||||
bottom: 32,
|
||||
left: "50%",
|
||||
...(positioned
|
||||
? { top: dragPos.top, left: dragPos.left, bottom: "auto" }
|
||||
: { bottom: 32, left: "50%" }),
|
||||
zIndex: 2147483647,
|
||||
pointerEvents: "auto",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 10,
|
||||
gap: 5,
|
||||
backgroundColor: "#fff",
|
||||
borderRadius: 9999,
|
||||
padding: "10px 10px 10px 20px",
|
||||
boxShadow: "0 8px 32px rgba(15,23,42,0.16), 0 2px 8px rgba(0,0,0,0.1)",
|
||||
height: 50,
|
||||
boxSizing: "border-box",
|
||||
padding: "6px 8px 6px 16px",
|
||||
boxShadow: dragging
|
||||
? "0 12px 40px rgba(15,23,42,0.22), 0 2px 8px rgba(0,0,0,0.12)"
|
||||
: "0 8px 32px rgba(15,23,42,0.16), 0 2px 8px rgba(0,0,0,0.1)",
|
||||
opacity: show ? 1 : 0,
|
||||
transform: show ? "translateX(-50%) translateY(0)" : "translateX(-50%) translateY(8px)",
|
||||
transition: "opacity 300ms ease-out, transform 300ms ease-out",
|
||||
transform: positioned ? "none" : defaultTransform,
|
||||
transition: dragging
|
||||
? "box-shadow 150ms ease-out"
|
||||
: "opacity 300ms ease-out, transform 300ms ease-out, box-shadow 150ms ease-out",
|
||||
fontFamily:
|
||||
'-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif',
|
||||
maxWidth: "min(420px, calc(100vw - 32px))",
|
||||
cursor: dragging ? "grabbing" : "grab",
|
||||
userSelect: "none",
|
||||
touchAction: "none",
|
||||
}}
|
||||
>
|
||||
<span
|
||||
data-slot="record-overlay-indicator"
|
||||
style={{
|
||||
width: 9,
|
||||
height: 9,
|
||||
borderRadius: "50%",
|
||||
backgroundColor: "#ef4444",
|
||||
flexShrink: 0,
|
||||
animation: "bsk-rec-pulse 1.4s ease-in-out infinite",
|
||||
}}
|
||||
/>
|
||||
<span
|
||||
<div
|
||||
data-slot="record-overlay-status"
|
||||
style={{
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 8,
|
||||
minWidth: 0,
|
||||
flex: 1,
|
||||
fontSize: 16,
|
||||
fontWeight: 500,
|
||||
color: "#333",
|
||||
whiteSpace: "nowrap",
|
||||
userSelect: "none",
|
||||
overflow: "hidden",
|
||||
textOverflow: "ellipsis",
|
||||
borderRadius: 9999,
|
||||
outline: "none",
|
||||
minHeight: 32,
|
||||
paddingRight: 4,
|
||||
}}
|
||||
>
|
||||
{t("recordOverlay.recording")}
|
||||
</span>
|
||||
<span
|
||||
data-slot="record-overlay-indicator"
|
||||
style={{
|
||||
width: 9,
|
||||
height: 9,
|
||||
borderRadius: "50%",
|
||||
backgroundColor: "#ef4444",
|
||||
flexShrink: 0,
|
||||
animation: "bsk-rec-pulse 1.4s ease-in-out infinite",
|
||||
}}
|
||||
/>
|
||||
<span
|
||||
data-slot="record-overlay-label"
|
||||
style={{
|
||||
display: "block",
|
||||
flex: "0 1 auto",
|
||||
fontSize: 16,
|
||||
fontWeight: 500,
|
||||
color: "#333",
|
||||
whiteSpace: "nowrap",
|
||||
userSelect: "none",
|
||||
overflow: "hidden",
|
||||
textOverflow: "ellipsis",
|
||||
width: introVisible ? "auto" : 48,
|
||||
maxWidth: introVisible ? 280 : 48,
|
||||
fontVariantNumeric: introVisible ? undefined : "tabular-nums",
|
||||
textAlign: introVisible ? "left" : "center",
|
||||
margin: 0,
|
||||
padding: 0,
|
||||
minWidth: 0,
|
||||
transition: "max-width 180ms ease-out",
|
||||
}}
|
||||
>
|
||||
{introVisible ? t("recordOverlay.recording") : formatElapsed(elapsedSeconds)}
|
||||
</span>
|
||||
</div>
|
||||
<button
|
||||
type="button"
|
||||
data-slot="record-overlay-finish"
|
||||
aria-label={t("recordOverlay.finish")}
|
||||
onClick={request.onFinish}
|
||||
style={{
|
||||
pointerEvents: "auto",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 6,
|
||||
display: "block",
|
||||
width: 32,
|
||||
height: 32,
|
||||
border: "none",
|
||||
borderRadius: 9999,
|
||||
padding: "8px 20px 8px 16px",
|
||||
fontSize: 15,
|
||||
fontWeight: 600,
|
||||
color: "#fff",
|
||||
backgroundColor: "#f97316",
|
||||
padding: 0,
|
||||
backgroundColor: "transparent",
|
||||
cursor: "pointer",
|
||||
transition: "background-color 150ms ease-out, opacity 150ms ease-out",
|
||||
whiteSpace: "nowrap",
|
||||
lineHeight: 1,
|
||||
flexShrink: 0,
|
||||
}}
|
||||
>
|
||||
<RiStopCircleLine size={18} color="#fff" />
|
||||
{t("recordOverlay.finish")}
|
||||
<img
|
||||
src={recordIconUrl}
|
||||
alt=""
|
||||
width={24}
|
||||
height={24}
|
||||
draggable={false}
|
||||
style={{ display: "block", margin: "auto" }}
|
||||
/>
|
||||
</button>
|
||||
</div>
|
||||
</>
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { i18n } from "@browser-skill/i18n";
|
||||
import { I18nextProvider } from "@browser-skill/i18n/react";
|
||||
import { cleanup, fireEvent, render, screen } from "@testing-library/react";
|
||||
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||
import { createElement } from "react";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { type HelpRequestData, HelpRequestOverlay } from "../HelpRequestOverlay";
|
||||
@@ -38,6 +38,36 @@ describe("HelpRequestOverlay", () => {
|
||||
expect(screen.getByText("Please complete the captcha")).toBeTruthy();
|
||||
});
|
||||
|
||||
it("renders explicit viewport rectangles for cross-frame targets", () => {
|
||||
const { container } = renderOverlay(
|
||||
baseRequest({ rects: [{ top: 30, left: 40, width: 120, height: 50 }] }),
|
||||
);
|
||||
const highlight = container.querySelector<HTMLElement>("[data-slot='help-highlight']");
|
||||
expect(highlight?.style.top).toBe("30px");
|
||||
expect(highlight?.style.left).toBe("40px");
|
||||
expect(highlight?.style.width).toBe("120px");
|
||||
expect(highlight?.style.height).toBe("50px");
|
||||
});
|
||||
|
||||
it("re-resolves cross-frame rectangles after the viewport changes", async () => {
|
||||
const refreshRects = vi.fn(async () => [{ top: 80, left: 90, width: 140, height: 60 }]);
|
||||
const { container } = renderOverlay(
|
||||
baseRequest({
|
||||
rects: [{ top: 30, left: 40, width: 120, height: 50 }],
|
||||
refreshRects,
|
||||
}),
|
||||
);
|
||||
|
||||
window.dispatchEvent(new Event("scroll"));
|
||||
|
||||
await waitFor(() => {
|
||||
const highlight = container.querySelector<HTMLElement>("[data-slot='help-highlight']");
|
||||
expect(highlight?.style.top).toBe("80px");
|
||||
expect(highlight?.style.left).toBe("90px");
|
||||
});
|
||||
expect(refreshRects).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("keeps the inactive render stable", () => {
|
||||
const { container, rerender } = renderOverlay(null);
|
||||
|
||||
|
||||
@@ -1,37 +1,181 @@
|
||||
import { cleanup, fireEvent, render } from "@testing-library/react";
|
||||
import { i18n } from "@browser-skill/i18n";
|
||||
import { I18nextProvider } from "@browser-skill/i18n/react";
|
||||
import { act, cleanup, fireEvent, render, screen } from "@testing-library/react";
|
||||
import { createElement } from "react";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { RecordOverlay } from "../RecordOverlay";
|
||||
|
||||
function renderOverlay(request: { id: string; startedAtMs?: number; onFinish: () => void } | null) {
|
||||
return render(
|
||||
createElement(I18nextProvider, { i18n }, createElement(RecordOverlay, { request })),
|
||||
);
|
||||
}
|
||||
|
||||
function advanceRecording(ms: number) {
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(ms);
|
||||
});
|
||||
}
|
||||
|
||||
describe("RecordOverlay", () => {
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("renders the bottom pill with orange finish CTA and no logo", () => {
|
||||
it("renders the bottom pill with the icon-only finish button", () => {
|
||||
const onFinish = vi.fn();
|
||||
const { container } = render(<RecordOverlay request={{ id: "rec-1", onFinish }} />);
|
||||
const { container } = renderOverlay({ id: "rec-1", onFinish });
|
||||
|
||||
const pill = container.querySelector("[data-slot='record-overlay-pill']");
|
||||
expect(pill).toBeTruthy();
|
||||
expect(pill?.querySelector("img")).toBeNull();
|
||||
expect((pill as HTMLElement).style.borderRadius).toBe("9999px");
|
||||
expect((pill as HTMLElement).style.backgroundColor).toBe("#fff");
|
||||
expect((pill as HTMLElement).style.gap).toBe("5px");
|
||||
expect((pill as HTMLElement).style.padding).toBe("6px 8px 6px 16px");
|
||||
expect((pill as HTMLElement).style.height).toBe("50px");
|
||||
expect((pill as HTMLElement).style.boxSizing).toBe("border-box");
|
||||
expect(container.querySelector("[data-slot='record-overlay-collapse']")).toBeNull();
|
||||
|
||||
const finish = container.querySelector("[data-slot='record-overlay-finish']");
|
||||
const status = container.querySelector("[data-slot='record-overlay-status']") as HTMLElement;
|
||||
expect(status.style.minHeight).toBe("32px");
|
||||
const finish = screen.getByRole("button", {
|
||||
name: i18n.t("recordOverlay.finish", { ns: "extension" }),
|
||||
});
|
||||
expect(finish).toBeTruthy();
|
||||
expect((finish as HTMLElement).style.backgroundColor).toBe("#f97316");
|
||||
expect(finish.textContent).toBe("");
|
||||
const icon = finish.querySelector("img");
|
||||
expect(icon).toBeTruthy();
|
||||
expect(icon?.getAttribute("src")).toContain("record-icon.png");
|
||||
expect((finish as HTMLElement).style.width).toBe("32px");
|
||||
expect((finish as HTMLElement).style.height).toBe("32px");
|
||||
expect(icon?.getAttribute("width")).toBe("24");
|
||||
expect(icon?.getAttribute("height")).toBe("24");
|
||||
expect((finish as HTMLElement).style.borderRadius).toBe("9999px");
|
||||
|
||||
fireEvent.click(finish!);
|
||||
fireEvent.click(finish);
|
||||
expect(onFinish).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("shows a pulsing recording indicator and no full-screen glow layer", () => {
|
||||
const { container } = render(<RecordOverlay request={{ id: "rec-1", onFinish: vi.fn() }} />);
|
||||
const { container } = renderOverlay({ id: "rec-1", onFinish: vi.fn() });
|
||||
expect(container.querySelector("[data-slot='record-overlay']")).toBeNull();
|
||||
|
||||
const indicator = container.querySelector("[data-slot='record-overlay-indicator']");
|
||||
expect(indicator).toBeTruthy();
|
||||
expect((indicator as HTMLElement).style.animation).toContain("bsk-rec-pulse");
|
||||
});
|
||||
|
||||
it("shows the recording message for ten seconds, then displays elapsed time", () => {
|
||||
vi.useFakeTimers();
|
||||
const { container } = renderOverlay({ id: "rec-1", onFinish: vi.fn() });
|
||||
const pill = container.querySelector("[data-slot='record-overlay-pill']") as HTMLElement;
|
||||
const label = container.querySelector("[data-slot='record-overlay-label']") as HTMLElement;
|
||||
|
||||
expect(pill.dataset.phase).toBe("intro");
|
||||
expect(label.textContent).toBe(i18n.t("recordOverlay.recording", { ns: "extension" }));
|
||||
|
||||
advanceRecording(9999);
|
||||
expect(pill.dataset.phase).toBe("intro");
|
||||
|
||||
advanceRecording(1);
|
||||
expect(pill.dataset.phase).toBe("timer");
|
||||
expect(label.textContent).toBe("00:10");
|
||||
});
|
||||
|
||||
it("formats elapsed time as zero-padded minutes and seconds", () => {
|
||||
vi.useFakeTimers();
|
||||
const { container } = renderOverlay({ id: "rec-1", onFinish: vi.fn() });
|
||||
const label = container.querySelector("[data-slot='record-overlay-label']") as HTMLElement;
|
||||
|
||||
advanceRecording(125_000);
|
||||
expect(label.textContent).toBe("02:05");
|
||||
});
|
||||
|
||||
it("keeps a fixed timer label width with tabular numerals", () => {
|
||||
vi.useFakeTimers();
|
||||
const { container } = renderOverlay({ id: "rec-1", onFinish: vi.fn() });
|
||||
const label = container.querySelector("[data-slot='record-overlay-label']") as HTMLElement;
|
||||
|
||||
advanceRecording(10_000);
|
||||
expect(label.style.width).toBe("48px");
|
||||
expect(label.style.maxWidth).toBe("48px");
|
||||
expect(label.style.fontVariantNumeric).toBe("tabular-nums");
|
||||
expect(label.style.textAlign).toBe("center");
|
||||
|
||||
advanceRecording(1_000);
|
||||
expect(label.textContent).toBe("00:11");
|
||||
expect(label.style.width).toBe("48px");
|
||||
});
|
||||
|
||||
it("resets the intro and elapsed time for a new request", () => {
|
||||
vi.useFakeTimers();
|
||||
const first = { id: "rec-1", onFinish: vi.fn() };
|
||||
const second = { id: "rec-2", onFinish: vi.fn() };
|
||||
const { container, rerender } = renderOverlay(first);
|
||||
|
||||
advanceRecording(65_000);
|
||||
rerender(
|
||||
createElement(I18nextProvider, { i18n }, createElement(RecordOverlay, { request: second })),
|
||||
);
|
||||
|
||||
const pill = container.querySelector("[data-slot='record-overlay-pill']") as HTMLElement;
|
||||
const label = container.querySelector("[data-slot='record-overlay-label']") as HTMLElement;
|
||||
expect(pill.dataset.phase).toBe("intro");
|
||||
expect(label.textContent).toBe(i18n.t("recordOverlay.recording", { ns: "extension" }));
|
||||
|
||||
advanceRecording(10_000);
|
||||
expect(pill.dataset.phase).toBe("timer");
|
||||
expect(label.textContent).toBe("00:10");
|
||||
});
|
||||
|
||||
it("continues the session timer when the overlay remounts on a new page", () => {
|
||||
vi.useFakeTimers();
|
||||
const startedAtMs = Date.now() - 65_000;
|
||||
const { container } = renderOverlay({ id: "rec-1", startedAtMs, onFinish: vi.fn() });
|
||||
|
||||
const pill = container.querySelector("[data-slot='record-overlay-pill']") as HTMLElement;
|
||||
const label = container.querySelector("[data-slot='record-overlay-label']") as HTMLElement;
|
||||
expect(pill.dataset.phase).toBe("timer");
|
||||
expect(label.textContent).toBe("01:05");
|
||||
|
||||
advanceRecording(5_000);
|
||||
expect(label.textContent).toBe("01:10");
|
||||
});
|
||||
|
||||
it("does not start drag when pointer down on finish", () => {
|
||||
const { container } = renderOverlay({ id: "rec-1", onFinish: vi.fn() });
|
||||
const pill = container.querySelector("[data-slot='record-overlay-pill']") as HTMLElement;
|
||||
const finish = container.querySelector("[data-slot='record-overlay-finish']") as HTMLElement;
|
||||
|
||||
fireEvent.pointerDown(finish, { button: 0, clientX: 10, clientY: 10, pointerId: 1 });
|
||||
expect(pill.getAttribute("data-dragging")).toBe("false");
|
||||
});
|
||||
|
||||
it("moves the pill when dragging the non-button area without toggling collapse", () => {
|
||||
const { container } = renderOverlay({ id: "rec-1", onFinish: vi.fn() });
|
||||
const pill = container.querySelector("[data-slot='record-overlay-pill']") as HTMLElement;
|
||||
|
||||
pill.getBoundingClientRect = () =>
|
||||
({
|
||||
top: 600,
|
||||
left: 200,
|
||||
width: 320,
|
||||
height: 48,
|
||||
right: 520,
|
||||
bottom: 648,
|
||||
}) as DOMRect;
|
||||
Object.defineProperty(pill, "offsetWidth", { value: 320, configurable: true });
|
||||
Object.defineProperty(pill, "offsetHeight", { value: 48, configurable: true });
|
||||
|
||||
fireEvent.pointerDown(pill, { button: 0, clientX: 300, clientY: 620, pointerId: 1 });
|
||||
fireEvent(window, new PointerEvent("pointermove", { clientX: 360, clientY: 580 }));
|
||||
fireEvent(window, new PointerEvent("pointerup", { pointerId: 1 }));
|
||||
|
||||
expect(pill.getAttribute("data-phase")).toBe("intro");
|
||||
expect(pill.style.left).toBe("260px");
|
||||
expect(pill.style.top).toBe("560px");
|
||||
expect(pill.style.bottom).toBe("auto");
|
||||
expect(pill.style.transform).toBe("none");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { CAPTURE_SUPPRESS, type CaptureSuppressAck } from "@/lib/capture-suppress-bridge";
|
||||
import { CAPTURE_HIDDEN_ATTR, createCaptureSuppressController } from "../capture-suppress";
|
||||
|
||||
/**
|
||||
* rAF is stubbed with a manual queue so tests can drive the compositor
|
||||
* frame-by-frame and assert the ack only lands after the second frame.
|
||||
*/
|
||||
describe("capture-suppress controller", () => {
|
||||
let rafQueue: FrameRequestCallback[];
|
||||
let host: HTMLElement;
|
||||
|
||||
beforeEach(() => {
|
||||
rafQueue = [];
|
||||
vi.stubGlobal("requestAnimationFrame", (cb: FrameRequestCallback) => {
|
||||
rafQueue.push(cb);
|
||||
return rafQueue.length;
|
||||
});
|
||||
vi.stubGlobal("cancelAnimationFrame", vi.fn());
|
||||
document.body.innerHTML = "";
|
||||
host = document.createElement("div");
|
||||
document.body.append(host);
|
||||
});
|
||||
|
||||
/** Run every queued rAF callback, draining microtasks between rounds. */
|
||||
async function flushFrames(rounds = 4): Promise<void> {
|
||||
for (let round = 0; round < rounds && rafQueue.length > 0; round += 1) {
|
||||
const callbacks = rafQueue.splice(0);
|
||||
for (const cb of callbacks) cb(0);
|
||||
await Promise.resolve();
|
||||
await Promise.resolve();
|
||||
}
|
||||
}
|
||||
|
||||
it("begin hides the host immediately but only acks after two frames", async () => {
|
||||
const controller = createCaptureSuppressController(() => host);
|
||||
const sendResponse = vi.fn();
|
||||
|
||||
const needsAsync = controller.handleMessage(
|
||||
{ type: CAPTURE_SUPPRESS, phase: "begin" },
|
||||
sendResponse,
|
||||
);
|
||||
|
||||
expect(needsAsync).toBe(true);
|
||||
expect(controller.pendingCount).toBe(1);
|
||||
expect(host.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(true);
|
||||
expect(sendResponse).not.toHaveBeenCalled();
|
||||
|
||||
// First frame commits the style change; the ack must still be pending.
|
||||
await flushFrames(1);
|
||||
expect(sendResponse).not.toHaveBeenCalled();
|
||||
|
||||
// Second frame: the compositor has produced an overlay-free frame.
|
||||
await flushFrames();
|
||||
const expected: CaptureSuppressAck = { type: CAPTURE_SUPPRESS, ok: true };
|
||||
expect(sendResponse).toHaveBeenCalledWith(expected);
|
||||
});
|
||||
|
||||
it("end removes the attribute once the count returns to zero", async () => {
|
||||
const controller = createCaptureSuppressController(() => host);
|
||||
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "begin" }, vi.fn());
|
||||
await flushFrames();
|
||||
expect(host.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(true);
|
||||
|
||||
const sendResponse = vi.fn();
|
||||
const needsAsync = controller.handleMessage(
|
||||
{ type: CAPTURE_SUPPRESS, phase: "end" },
|
||||
sendResponse,
|
||||
);
|
||||
|
||||
expect(needsAsync).toBe(false);
|
||||
expect(controller.pendingCount).toBe(0);
|
||||
expect(host.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(false);
|
||||
expect(sendResponse).toHaveBeenCalledWith({ type: CAPTURE_SUPPRESS, ok: true });
|
||||
});
|
||||
|
||||
it("reference-counts concurrent begins so one end does not unhide early", async () => {
|
||||
const controller = createCaptureSuppressController(() => host);
|
||||
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "begin" }, vi.fn());
|
||||
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "begin" }, vi.fn());
|
||||
await flushFrames();
|
||||
expect(controller.pendingCount).toBe(2);
|
||||
|
||||
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "end" }, vi.fn());
|
||||
expect(controller.pendingCount).toBe(1);
|
||||
expect(host.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(true);
|
||||
|
||||
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "end" }, vi.fn());
|
||||
expect(controller.pendingCount).toBe(0);
|
||||
expect(host.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(false);
|
||||
});
|
||||
|
||||
it("treats a stray end as a no-op", () => {
|
||||
const controller = createCaptureSuppressController(() => host);
|
||||
const sendResponse = vi.fn();
|
||||
|
||||
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "end" }, sendResponse);
|
||||
|
||||
expect(controller.pendingCount).toBe(0);
|
||||
expect(host.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(false);
|
||||
expect(sendResponse).toHaveBeenCalledWith({ type: CAPTURE_SUPPRESS, ok: true });
|
||||
});
|
||||
|
||||
it("re-applies the attribute when the host is remounted mid-capture", () => {
|
||||
let currentHost: HTMLElement | null = host;
|
||||
const controller = createCaptureSuppressController(() => currentHost);
|
||||
controller.handleMessage({ type: CAPTURE_SUPPRESS, phase: "begin" }, vi.fn());
|
||||
|
||||
// Host lost and rebuilt before `end` arrives.
|
||||
currentHost = null;
|
||||
const rebuilt = document.createElement("div");
|
||||
controller.onHostMounted(rebuilt);
|
||||
|
||||
expect(rebuilt.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(true);
|
||||
});
|
||||
|
||||
it("leaves a remounted host visible once suppression has ended", () => {
|
||||
const controller = createCaptureSuppressController(() => host);
|
||||
const rebuilt = document.createElement("div");
|
||||
|
||||
controller.onHostMounted(rebuilt);
|
||||
|
||||
expect(rebuilt.hasAttribute(CAPTURE_HIDDEN_ATTR)).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,64 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createHelpRequestData } from "../help-request";
|
||||
|
||||
describe("createHelpRequestData", () => {
|
||||
it("gives live delivery and recovery the same rectangle refresh mechanism", async () => {
|
||||
const finish = vi.fn();
|
||||
const query = vi.fn(async () => ({
|
||||
active: true,
|
||||
request: {
|
||||
requestId: "help-1",
|
||||
prompt: "complete the challenge",
|
||||
selectors: [],
|
||||
rects: [{ top: 80, left: 90, width: 140, height: 60 }],
|
||||
timeoutMs: 1_000,
|
||||
},
|
||||
}));
|
||||
const request = createHelpRequestData(
|
||||
{
|
||||
requestId: "help-1",
|
||||
prompt: "complete the challenge",
|
||||
selectors: [],
|
||||
rects: [{ top: 10, left: 20, width: 30, height: 40 }],
|
||||
timeoutMs: 1_000,
|
||||
},
|
||||
{ finish, query },
|
||||
);
|
||||
|
||||
await expect(request.refreshRects?.()).resolves.toEqual([
|
||||
{ top: 80, left: 90, width: 140, height: 60 },
|
||||
]);
|
||||
request.onContinue(" done ");
|
||||
request.onCancel();
|
||||
|
||||
expect(query).toHaveBeenCalledOnce();
|
||||
expect(finish).toHaveBeenNthCalledWith(1, "help-1", "continued", " done ");
|
||||
expect(finish).toHaveBeenNthCalledWith(2, "help-1", "cancelled");
|
||||
});
|
||||
|
||||
it("ignores rectangle replies for a replaced request", async () => {
|
||||
const request = createHelpRequestData(
|
||||
{
|
||||
requestId: "help-1",
|
||||
prompt: "complete the challenge",
|
||||
selectors: [],
|
||||
timeoutMs: 1_000,
|
||||
},
|
||||
{
|
||||
finish: vi.fn(),
|
||||
query: vi.fn(async () => ({
|
||||
active: true,
|
||||
request: {
|
||||
requestId: "help-2",
|
||||
prompt: "new challenge",
|
||||
selectors: [],
|
||||
rects: [{ top: 1, left: 2, width: 3, height: 4 }],
|
||||
timeoutMs: 1_000,
|
||||
},
|
||||
})),
|
||||
},
|
||||
);
|
||||
|
||||
await expect(request.refreshRects?.()).resolves.toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -130,4 +130,31 @@ describe("OverlayController", () => {
|
||||
expect(controller.snapshot().controlVisible).toBe(true);
|
||||
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(true);
|
||||
});
|
||||
|
||||
it("hides the control overlay when the user hides control hints", () => {
|
||||
const controller = new OverlayController();
|
||||
controller.activateAgentSession("sess-1");
|
||||
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(true);
|
||||
|
||||
controller.setControlHintsHidden(true);
|
||||
expect(controller.snapshot().controlHintsHidden).toBe(true);
|
||||
// The session still owns the tab — only the chrome is hidden.
|
||||
expect(controller.snapshot().controlVisible).toBe(true);
|
||||
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(false);
|
||||
|
||||
controller.setControlHintsHidden(false);
|
||||
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(true);
|
||||
});
|
||||
|
||||
it("keeps the control-hints preference across session overlay resets", () => {
|
||||
const controller = new OverlayController();
|
||||
controller.activateAgentSession("sess-1");
|
||||
controller.setControlHintsHidden(true);
|
||||
|
||||
controller.resetAgentOverlays("sess-1");
|
||||
expect(controller.snapshot().controlHintsHidden).toBe(true);
|
||||
|
||||
controller.activateAgentSession("sess-2");
|
||||
expect(shouldShowAgentControlOverlay(controller.snapshot())).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
/**
|
||||
* Content-script side of the capture-suppress bridge: hides the overlay
|
||||
* shadow host while the background captures a screenshot.
|
||||
*
|
||||
* `begin` sets `data-bsk-capture-hidden` on the host (overlay.css maps it
|
||||
* to `display: none !important`, which removes the whole shadow subtree
|
||||
* from compositing) and only acks after two animation frames — the first
|
||||
* frame commits the style change, the second guarantees the compositor
|
||||
* has produced a frame without the overlay, so the capture API cannot
|
||||
* pick up a stale frame that still contains it. `end` is
|
||||
* reference-counted so overlapping captures do not unhide the host early.
|
||||
*/
|
||||
|
||||
import {
|
||||
CAPTURE_SUPPRESS,
|
||||
type CaptureSuppressAck,
|
||||
type CaptureSuppressMessage,
|
||||
} from "@/lib/capture-suppress-bridge";
|
||||
|
||||
export const CAPTURE_HIDDEN_ATTR = "data-bsk-capture-hidden";
|
||||
|
||||
/**
|
||||
* Safety net for hidden/background tabs where rAF callbacks never fire:
|
||||
* the attribute is already set synchronously, so any frame the compositor
|
||||
* produces from now on excludes the overlay. Waiting longer would just
|
||||
* deadlock the capture.
|
||||
*/
|
||||
const FRAME_FALLBACK_MS = 500;
|
||||
|
||||
function nextFrame(): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
const raf = requestAnimationFrame(() => {
|
||||
clearTimeout(timer);
|
||||
resolve();
|
||||
});
|
||||
const timer = setTimeout(() => {
|
||||
cancelAnimationFrame(raf);
|
||||
resolve();
|
||||
}, FRAME_FALLBACK_MS);
|
||||
});
|
||||
}
|
||||
|
||||
export interface CaptureSuppressController {
|
||||
/**
|
||||
* Handle one bridge message. Returns true when the ack is sent
|
||||
* asynchronously and the caller must keep the message channel open.
|
||||
*/
|
||||
handleMessage(
|
||||
message: CaptureSuppressMessage,
|
||||
sendResponse: (ack: CaptureSuppressAck) => void,
|
||||
): boolean;
|
||||
/** Re-apply the hidden attribute when the overlay host is (re)mounted. */
|
||||
onHostMounted(host: HTMLElement): void;
|
||||
/** Number of in-flight `begin` phases (exposed for tests). */
|
||||
readonly pendingCount: number;
|
||||
}
|
||||
|
||||
export function createCaptureSuppressController(
|
||||
getHost: () => HTMLElement | null,
|
||||
): CaptureSuppressController {
|
||||
let pending = 0;
|
||||
|
||||
const ack: CaptureSuppressAck = { type: CAPTURE_SUPPRESS, ok: true };
|
||||
|
||||
function setHidden(host: HTMLElement | null, hidden: boolean): void {
|
||||
if (!host) return;
|
||||
if (hidden) host.setAttribute(CAPTURE_HIDDEN_ATTR, "");
|
||||
else host.removeAttribute(CAPTURE_HIDDEN_ATTR);
|
||||
}
|
||||
|
||||
return {
|
||||
get pendingCount() {
|
||||
return pending;
|
||||
},
|
||||
handleMessage(message, sendResponse) {
|
||||
if (message.phase === "begin") {
|
||||
pending += 1;
|
||||
setHidden(getHost(), true);
|
||||
void (async () => {
|
||||
await nextFrame();
|
||||
await nextFrame();
|
||||
sendResponse(ack);
|
||||
})();
|
||||
return true;
|
||||
}
|
||||
// `end` without a matching `begin` (e.g. a retried message) must not
|
||||
// drive the counter negative or clear a concurrent capture's flag.
|
||||
pending = Math.max(0, pending - 1);
|
||||
if (pending === 0) setHidden(getHost(), false);
|
||||
sendResponse(ack);
|
||||
return false;
|
||||
},
|
||||
onHostMounted(host) {
|
||||
// Host lost + rebuilt mid-capture: the fresh host must stay hidden
|
||||
// until the last `end` arrives.
|
||||
setHidden(host, pending > 0);
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import type { HelpQueryResponse, HelpRequestMessage } from "@/lib/help-bridge";
|
||||
import type { HelpRequestData } from "./HelpRequestOverlay";
|
||||
|
||||
export interface HelpRequestDataDeps {
|
||||
finish(requestId: string, outcome: "continued" | "cancelled", note?: string): void;
|
||||
query(): Promise<HelpQueryResponse | undefined>;
|
||||
}
|
||||
|
||||
/** Build the content-side representation shared by live delivery and recovery. */
|
||||
export function createHelpRequestData(
|
||||
message: Omit<HelpRequestMessage, "type">,
|
||||
deps: HelpRequestDataDeps,
|
||||
): HelpRequestData {
|
||||
return {
|
||||
id: message.requestId,
|
||||
prompt: message.prompt,
|
||||
...(message.title ? { title: message.title } : {}),
|
||||
...(message.displayMode ? { displayMode: message.displayMode } : {}),
|
||||
selectors: message.selectors,
|
||||
rects: message.rects,
|
||||
refreshRects: async () => {
|
||||
const response = await deps.query();
|
||||
return response?.active && response.request?.requestId === message.requestId
|
||||
? response.request.rects
|
||||
: undefined;
|
||||
},
|
||||
onContinue: (note: string) =>
|
||||
deps.finish(message.requestId, "continued", note.trim() ? note : undefined),
|
||||
onCancel: () => deps.finish(message.requestId, "cancelled"),
|
||||
};
|
||||
}
|
||||
@@ -18,6 +18,12 @@ export interface OverlayState {
|
||||
* so 「Agent 正在控制」does not flash between RecordOverlay and teardown.
|
||||
*/
|
||||
suppressControlAfterRecord: boolean;
|
||||
/**
|
||||
* User preference (chrome.storage, toggled from the popup): hide the
|
||||
* control hints — status pill, orange glow, and the input blocker that
|
||||
* comes with them. Not session state; survives overlay resets.
|
||||
*/
|
||||
controlHintsHidden: boolean;
|
||||
}
|
||||
|
||||
type MutableOverlayState = Omit<OverlayState, "controlVisible">;
|
||||
@@ -36,6 +42,7 @@ export class OverlayController {
|
||||
controlMode: "hidden",
|
||||
automationBypassCount: 0,
|
||||
suppressControlAfterRecord: false,
|
||||
controlHintsHidden: false,
|
||||
};
|
||||
|
||||
snapshot(): OverlayState {
|
||||
@@ -129,6 +136,10 @@ export class OverlayController {
|
||||
}
|
||||
}
|
||||
|
||||
setControlHintsHidden(hidden: boolean): void {
|
||||
this.state.controlHintsHidden = hidden;
|
||||
}
|
||||
|
||||
resetAgentOverlays(sessionId: string): HelpRequestData | null {
|
||||
if (this.state.activeSessionId && this.state.activeSessionId !== sessionId) {
|
||||
return null;
|
||||
@@ -152,6 +163,7 @@ export class OverlayController {
|
||||
export function shouldShowAgentControlOverlay(state: OverlayState): boolean {
|
||||
return (
|
||||
state.controlVisible &&
|
||||
!state.controlHintsHidden &&
|
||||
!state.suppressControlAfterRecord &&
|
||||
state.activeHelp === null &&
|
||||
state.activeRecord === null
|
||||
|
||||
@@ -14,3 +14,10 @@
|
||||
:host([data-bsk-overlay-surface][data-bsk-overlay-blocking]) {
|
||||
pointer-events: auto !important;
|
||||
}
|
||||
|
||||
/* Screenshot suppression: the host lives in the main DOM, so hiding it
|
||||
removes the whole shadow subtree from compositing and captured frames
|
||||
only contain page content. */
|
||||
:host([data-bsk-capture-hidden]) {
|
||||
display: none !important;
|
||||
}
|
||||
|
||||
@@ -956,7 +956,7 @@ export function handleRecordContentMessage(
|
||||
capture: RecordCaptureController | null;
|
||||
setActiveRequestId(id: string | null): void;
|
||||
setCapture(capture: RecordCaptureController | null): void;
|
||||
onStart(requestId: string): void;
|
||||
onStart(requestId: string, startedAtMs?: number): void;
|
||||
onStop(): void;
|
||||
},
|
||||
sendResponse?: (response: RecordStartAck | RecordStopAck) => void,
|
||||
@@ -973,7 +973,7 @@ export function handleRecordContentMessage(
|
||||
}),
|
||||
);
|
||||
state.setActiveRequestId(message.requestId);
|
||||
state.onStart(message.requestId);
|
||||
state.onStart(message.requestId, message.startedAtMs);
|
||||
sendResponse?.({ ok: true });
|
||||
return sendResponse !== undefined;
|
||||
}
|
||||
|
||||
@@ -22,6 +22,7 @@ import {
|
||||
} from "@/lib/overlay-bridge";
|
||||
import { POPUP_PORT_NAME, type PopupInbound, type PopupOutbound } from "@/lib/popup-bridge";
|
||||
import { attachSessionsLiveFlag } from "@/lib/sessions-live-flag";
|
||||
import { createDisconnectCleanup } from "@/session-manager/disconnect-cleanup";
|
||||
import { attachSessionEventHandler } from "@/session-manager/event-handler";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import {
|
||||
@@ -137,6 +138,13 @@ export default defineBackground(() => {
|
||||
// stale `true` does not keep waking us on every page load until the
|
||||
// first mutation (review M4/M5 round 3 m-R3-1).
|
||||
void sessionsLive.refresh();
|
||||
const cleanupAfterDisconnect = createDisconnectCleanup({
|
||||
manager: sessions,
|
||||
sessionStopDeps: { cdp },
|
||||
onSessionsChanged: () => {
|
||||
void sessionsLive.syncFromManager();
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({
|
||||
transport,
|
||||
sessions,
|
||||
@@ -267,7 +275,20 @@ export default defineBackground(() => {
|
||||
|
||||
void (async () => {
|
||||
const connectionEnabled = await getConnectionEnabled();
|
||||
await controller.attach(transport, detectBrowserMeta(), connectionEnabled);
|
||||
await controller.attach(transport, detectBrowserMeta(), connectionEnabled, {
|
||||
beforeDisconnect: async () => {
|
||||
const report = await cleanupAfterDisconnect();
|
||||
if (report.failures.length > 0) {
|
||||
console.warn("[browser-skill] session cleanup before disconnect was incomplete", report);
|
||||
}
|
||||
},
|
||||
onDisconnected: async () => {
|
||||
const report = await cleanupAfterDisconnect();
|
||||
if (report.failures.length > 0) {
|
||||
console.warn("[browser-skill] session cleanup after disconnect was incomplete", report);
|
||||
}
|
||||
},
|
||||
});
|
||||
})().catch((err) => {
|
||||
console.error("[browser-skill] controller failed to attach", err);
|
||||
});
|
||||
|
||||
@@ -5,7 +5,9 @@ import { flushSync } from "react-dom";
|
||||
import ReactDOM from "react-dom/client";
|
||||
import { BorrowConfirmationOverlay } from "@/content/BorrowConfirmationOverlay";
|
||||
import { ControlOverlay } from "@/content/ControlOverlay";
|
||||
import { createCaptureSuppressController } from "@/content/capture-suppress";
|
||||
import { HelpRequestOverlay } from "@/content/HelpRequestOverlay";
|
||||
import { createHelpRequestData } from "@/content/help-request";
|
||||
import overlayCss from "@/content/overlay.css?inline";
|
||||
import { OverlayController, shouldShowAgentControlOverlay } from "@/content/overlay-controller";
|
||||
import { RecordOverlay } from "@/content/RecordOverlay";
|
||||
@@ -14,6 +16,11 @@ import {
|
||||
isRecordContentMessage,
|
||||
type RecordCaptureController,
|
||||
} from "@/content/record-capture";
|
||||
import {
|
||||
type CaptureSuppressAck,
|
||||
type CaptureSuppressMessage,
|
||||
isCaptureSuppressMessage,
|
||||
} from "@/lib/capture-suppress-bridge";
|
||||
import {
|
||||
HELP_ACK,
|
||||
HELP_FINISH,
|
||||
@@ -26,6 +33,7 @@ import {
|
||||
isHelpCancelMessage,
|
||||
isHelpRequestMessage,
|
||||
} from "@/lib/help-bridge";
|
||||
import { getControlHintsHidden, STORAGE_KEYS } from "@/lib/instance-id";
|
||||
import {
|
||||
isOverlayAgentOverlayResetMessage,
|
||||
isOverlayAgentStateMessage,
|
||||
@@ -39,6 +47,7 @@ import { sendInterrupt } from "@/lib/overlay-interrupt-client";
|
||||
import {
|
||||
RECORD_FINISH,
|
||||
RECORD_QUERY,
|
||||
type RecordQueryResponse,
|
||||
type RecordStartAck,
|
||||
type RecordStopAck,
|
||||
} from "@/lib/record-bridge";
|
||||
@@ -69,6 +78,16 @@ export default defineContentScript({
|
||||
let hostLossReported = false;
|
||||
let remountInProgress = false;
|
||||
|
||||
// Load the user's control-hints preference up front so an already-active
|
||||
// Agent session does not flash the overlay before the stored value lands.
|
||||
try {
|
||||
overlays.setControlHintsHidden(await getControlHintsHidden());
|
||||
} catch (err) {
|
||||
console.debug("[bsk overlay] control-hints preference read failed", err);
|
||||
}
|
||||
|
||||
const captureSuppress = createCaptureSuppressController(() => overlayHost);
|
||||
|
||||
const ui = await createShadowRootUi(ctx, {
|
||||
name: "browser-skill-overlay",
|
||||
position: "inline",
|
||||
@@ -80,6 +99,8 @@ export default defineContentScript({
|
||||
overlayHost = shadowHost;
|
||||
overlayContainer = container;
|
||||
hostLossReported = false;
|
||||
// A host rebuilt mid-capture must stay hidden until `end` arrives.
|
||||
captureSuppress.onHostMounted(shadowHost);
|
||||
const app = document.createElement("div");
|
||||
app.className = "bsk-overlay-root";
|
||||
container.append(app);
|
||||
@@ -237,12 +258,17 @@ export default defineContentScript({
|
||||
| BorrowCancelMessage
|
||||
| HelpRequestMessage
|
||||
| HelpCancelMessage
|
||||
| CaptureSuppressMessage
|
||||
| OverlayAgentOverlayResetMessage
|
||||
| OverlayAgentStateMessage
|
||||
| OverlayAutomationBypassMessage,
|
||||
_sender: chrome.runtime.MessageSender,
|
||||
sendResponse: (response: BorrowResponseMessage | HelpAckMessage) => void,
|
||||
sendResponse: (response: BorrowResponseMessage | HelpAckMessage | CaptureSuppressAck) => void,
|
||||
) => {
|
||||
if (isCaptureSuppressMessage(message)) {
|
||||
return captureSuppress.handleMessage(message, sendResponse);
|
||||
}
|
||||
|
||||
if (isRecordContentMessage(message)) {
|
||||
const needsAsync = handleRecordContentMessage(
|
||||
message,
|
||||
@@ -255,9 +281,10 @@ export default defineContentScript({
|
||||
setCapture: (capture) => {
|
||||
recordCapture = capture;
|
||||
},
|
||||
onStart: (requestId) => {
|
||||
onStart: (requestId, startedAtMs) => {
|
||||
overlays.setAgentRecordRequest({
|
||||
id: requestId,
|
||||
...(typeof startedAtMs === "number" ? { startedAtMs } : {}),
|
||||
onFinish: () => {
|
||||
void chrome.runtime.sendMessage({
|
||||
type: RECORD_FINISH,
|
||||
@@ -318,16 +345,7 @@ export default defineContentScript({
|
||||
|
||||
if (isHelpRequestMessage(message)) {
|
||||
const helpMsg = message as HelpRequestMessage;
|
||||
const previousHelp = overlays.setAgentHelpRequest({
|
||||
id: helpMsg.requestId,
|
||||
prompt: helpMsg.prompt,
|
||||
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
||||
...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}),
|
||||
selectors: helpMsg.selectors,
|
||||
onContinue: (note: string) =>
|
||||
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
|
||||
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
|
||||
});
|
||||
const previousHelp = mountHelpRequest(helpMsg);
|
||||
if (previousHelp && previousHelp.id !== helpMsg.requestId) {
|
||||
void sendHelpFinish(previousHelp.id, "cancelled");
|
||||
}
|
||||
@@ -380,17 +398,18 @@ export default defineContentScript({
|
||||
});
|
||||
}
|
||||
|
||||
function mountHelpRequest(helpMsg: Omit<HelpRequestMessage, "type">): void {
|
||||
overlays.setAgentHelpRequest({
|
||||
id: helpMsg.requestId,
|
||||
prompt: helpMsg.prompt,
|
||||
...(helpMsg.title ? { title: helpMsg.title } : {}),
|
||||
...(helpMsg.displayMode ? { displayMode: helpMsg.displayMode } : {}),
|
||||
selectors: helpMsg.selectors,
|
||||
onContinue: (note: string) =>
|
||||
void sendHelpFinish(helpMsg.requestId, "continued", note.trim() ? note : undefined),
|
||||
onCancel: () => void sendHelpFinish(helpMsg.requestId, "cancelled"),
|
||||
});
|
||||
function mountHelpRequest(helpMsg: Omit<HelpRequestMessage, "type">) {
|
||||
return overlays.setAgentHelpRequest(
|
||||
createHelpRequestData(helpMsg, {
|
||||
finish: (requestId, outcome, note) => {
|
||||
void sendHelpFinish(requestId, outcome, note);
|
||||
},
|
||||
query: () =>
|
||||
chrome.runtime.sendMessage({ type: HELP_QUERY }) as Promise<
|
||||
HelpQueryResponse | undefined
|
||||
>,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
async function queryActiveHelpWithRetry(): Promise<void> {
|
||||
@@ -415,15 +434,17 @@ export default defineContentScript({
|
||||
try {
|
||||
const recordQuery = (await chrome.runtime.sendMessage({
|
||||
type: RECORD_QUERY,
|
||||
})) as { active?: boolean; requestId?: string } | undefined;
|
||||
})) as RecordQueryResponse | undefined;
|
||||
if (
|
||||
recordQuery?.active &&
|
||||
typeof recordQuery.requestId === "string" &&
|
||||
overlays.snapshot().activeRecord === null
|
||||
) {
|
||||
const requestId = recordQuery.requestId;
|
||||
const startedAtMs = recordQuery.startedAtMs;
|
||||
overlays.setAgentRecordRequest({
|
||||
id: requestId,
|
||||
...(typeof startedAtMs === "number" ? { startedAtMs } : {}),
|
||||
onFinish: () => {
|
||||
void chrome.runtime.sendMessage({
|
||||
type: RECORD_FINISH,
|
||||
@@ -460,6 +481,19 @@ export default defineContentScript({
|
||||
if (event.persisted) void requestOverlayState();
|
||||
};
|
||||
|
||||
// Live-apply popup toggles of the control-hints preference.
|
||||
const onStorageChange = (
|
||||
changes: Record<string, chrome.storage.StorageChange>,
|
||||
areaName: string,
|
||||
) => {
|
||||
if (areaName !== "local") return;
|
||||
const change = changes[STORAGE_KEYS.CONTROL_HINTS_HIDDEN];
|
||||
if (!change) return;
|
||||
overlays.setControlHintsHidden(change.newValue === true);
|
||||
renderAll();
|
||||
};
|
||||
chrome.storage.onChanged.addListener(onStorageChange);
|
||||
|
||||
ui.mount();
|
||||
chrome.runtime.onMessage.addListener(onMessage);
|
||||
void requestOverlayState();
|
||||
@@ -490,6 +524,7 @@ export default defineContentScript({
|
||||
ctx.onInvalidated(() => {
|
||||
hostObserver.disconnect();
|
||||
chrome.runtime.onMessage.removeListener(onMessage);
|
||||
chrome.storage.onChanged.removeListener(onStorageChange);
|
||||
window.removeEventListener("pageshow", onPageShow);
|
||||
// Restore history hooks / remove capture listeners before the CS unloads.
|
||||
recordCapture?.dispose();
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { cleanup, fireEvent, render, screen, waitFor } from "@testing-library/react";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import type { SnapshotInfo } from "@/lib/connection-controller";
|
||||
import { STORAGE_KEYS } from "@/lib/instance-id";
|
||||
import { EXTENSION_VERSION } from "@/transport/handshake";
|
||||
import { App } from "./App";
|
||||
import { useConnectionState } from "./use-connection-state";
|
||||
@@ -244,3 +245,94 @@ describe("App", () => {
|
||||
expect(screen.getByText("连接后可用")).toBeTruthy();
|
||||
});
|
||||
});
|
||||
|
||||
describe("control hints toggle", () => {
|
||||
function stubChromeStorage(initial: Record<string, unknown> = {}) {
|
||||
const store = { ...initial };
|
||||
vi.stubGlobal("chrome", {
|
||||
runtime: { lastError: undefined },
|
||||
storage: {
|
||||
local: {
|
||||
get: (keys: string | string[], cb: (items: Record<string, unknown>) => void) => {
|
||||
const items: Record<string, unknown> = {};
|
||||
for (const k of Array.isArray(keys) ? keys : [keys]) {
|
||||
if (k in store) items[k] = store[k];
|
||||
}
|
||||
cb(items);
|
||||
},
|
||||
set: (items: Record<string, unknown>, cb?: () => void) => {
|
||||
Object.assign(store, items);
|
||||
cb?.();
|
||||
},
|
||||
},
|
||||
onChanged: {
|
||||
addListener: vi.fn(),
|
||||
removeListener: vi.fn(),
|
||||
},
|
||||
},
|
||||
});
|
||||
return store;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it("renders the control hints toggle on when no preference is stored", async () => {
|
||||
stubChromeStorage();
|
||||
|
||||
render(<App />);
|
||||
|
||||
const toggle = await screen.findByRole("switch", { name: "控制提示" });
|
||||
expect(toggle.getAttribute("aria-checked")).toBe("true");
|
||||
});
|
||||
|
||||
it("reflects the stored hidden preference", async () => {
|
||||
stubChromeStorage({ [STORAGE_KEYS.CONTROL_HINTS_HIDDEN]: true });
|
||||
|
||||
render(<App />);
|
||||
|
||||
const toggle = await screen.findByRole("switch", { name: "控制提示" });
|
||||
await waitFor(() => expect(toggle.getAttribute("aria-checked")).toBe("false"));
|
||||
});
|
||||
|
||||
it("persists the hidden preference when the toggle is turned off", async () => {
|
||||
const store = stubChromeStorage();
|
||||
|
||||
render(<App />);
|
||||
|
||||
const toggle = await screen.findByRole("switch", { name: "控制提示" });
|
||||
fireEvent.click(toggle);
|
||||
|
||||
expect(store[STORAGE_KEYS.CONTROL_HINTS_HIDDEN]).toBe(true);
|
||||
expect(toggle.getAttribute("aria-checked")).toBe("false");
|
||||
});
|
||||
|
||||
it("keeps the hint copy in an accessible info tooltip", async () => {
|
||||
stubChromeStorage();
|
||||
|
||||
render(<App />);
|
||||
|
||||
const info = await screen.findByRole("button", { name: "控制提示说明" });
|
||||
expect(info).toBeTruthy();
|
||||
const tooltip = screen.getByRole("tooltip");
|
||||
expect(tooltip.textContent).toBe("Agent 控制页面时显示提示条和橙色闪光。");
|
||||
// Hidden until the info button is hovered or focused.
|
||||
expect(tooltip.className).toContain("opacity-0");
|
||||
});
|
||||
|
||||
it("uses the same switch component and size for both settings rows", async () => {
|
||||
stubChromeStorage();
|
||||
|
||||
render(<App />);
|
||||
|
||||
const hintsToggle = await screen.findByRole("switch", { name: "控制提示" });
|
||||
const connectionToggle = screen.getByRole("switch", { name: "BrowserSkill 连接" });
|
||||
// One shared Switch component, one size — hierarchy comes from copy and
|
||||
// the info icon, not control size. Both rows default to checked, so the
|
||||
// class strings must be identical.
|
||||
expect(hintsToggle.className).toContain("h-5 w-9");
|
||||
expect(hintsToggle.className).toBe(connectionToggle.className);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,17 +1,20 @@
|
||||
import { useTranslation } from "@browser-skill/i18n/react";
|
||||
import { Badge, Button, cn, Input, Label } from "@browser-skill/ui";
|
||||
import { Badge, Button, Input, Label } from "@browser-skill/ui";
|
||||
import {
|
||||
RiApps2Line,
|
||||
RiArrowLeftLine,
|
||||
RiArrowRightSLine,
|
||||
RiCheckLine,
|
||||
RiFileCopyLine,
|
||||
RiInformationLine,
|
||||
} from "@remixicon/react";
|
||||
import { type ChangeEvent, useEffect, useState } from "react";
|
||||
import { PROTOCOL_VERSION } from "@/transport/handshake";
|
||||
import functionIconUrl from "../../../assets/function.svg";
|
||||
import { ConnectionStatusIndicator } from "./connection-status-indicator";
|
||||
import { POPUP_FEATURES, type PopupView } from "./features";
|
||||
import { Switch } from "./switch";
|
||||
import { type PopupStatusState, useConnectionState } from "./use-connection-state";
|
||||
import { useControlHintsHidden } from "./use-control-hints-hidden";
|
||||
|
||||
const STATE_LABEL_KEYS = {
|
||||
disconnected: "popup.stateLabel.disconnected",
|
||||
@@ -37,6 +40,7 @@ function getLogoSrc() {
|
||||
export function App() {
|
||||
const { t } = useTranslation("extension");
|
||||
const { snapshot, statusState, setConnectionEnabled } = useConnectionState();
|
||||
const [controlHintsHidden, setControlHintsHidden] = useControlHintsHidden();
|
||||
const [view, setView] = useState<PopupView>("main");
|
||||
const [copiedInstanceId, setCopiedInstanceId] = useState(false);
|
||||
const [purposeDraft, setPurposeDraft] = useState("");
|
||||
@@ -156,12 +160,17 @@ export function App() {
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="icon"
|
||||
className="size-7 shrink-0 rounded-md"
|
||||
className="group size-8 shrink-0 rounded-md"
|
||||
aria-label={t("popup.launcher.title")}
|
||||
onClick={() => setView("features")}
|
||||
data-slot="popup-launcher"
|
||||
>
|
||||
<RiApps2Line className="size-4" aria-hidden />
|
||||
<img
|
||||
src={functionIconUrl}
|
||||
alt=""
|
||||
className="size-6 opacity-35 transition-opacity group-hover:opacity-100 group-focus-visible:opacity-100"
|
||||
data-slot="popup-launcher-icon"
|
||||
/>
|
||||
</Button>
|
||||
)}
|
||||
</header>
|
||||
@@ -187,26 +196,12 @@ export function App() {
|
||||
>
|
||||
{t(STATE_BADGE_KEYS[statusState])}
|
||||
</Badge>
|
||||
<button
|
||||
type="button"
|
||||
role="switch"
|
||||
aria-checked={snapshot.connectionEnabled}
|
||||
<Switch
|
||||
checked={snapshot.connectionEnabled}
|
||||
onCheckedChange={setConnectionEnabled}
|
||||
aria-label={t("popup.connectionToggleTitle")}
|
||||
data-slot="popup-connection-toggle"
|
||||
className={cn(
|
||||
"relative inline-flex h-5 w-9 shrink-0 cursor-pointer items-center rounded-full border border-transparent transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring/50",
|
||||
snapshot.connectionEnabled ? "bg-primary" : "bg-muted",
|
||||
)}
|
||||
onClick={() => setConnectionEnabled(!snapshot.connectionEnabled)}
|
||||
>
|
||||
<span
|
||||
className={cn(
|
||||
"pointer-events-none block size-4 rounded-full bg-background shadow-sm transition-transform",
|
||||
snapshot.connectionEnabled ? "translate-x-4" : "translate-x-0.5",
|
||||
)}
|
||||
aria-hidden
|
||||
/>
|
||||
</button>
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
{isSkewed && (
|
||||
@@ -222,6 +217,41 @@ export function App() {
|
||||
)}
|
||||
</section>
|
||||
|
||||
<section
|
||||
className="rounded-xl border border-border/80 bg-card/60 px-3 py-2.5"
|
||||
data-slot="popup-control-hints-card"
|
||||
>
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<span className="flex min-w-0 items-center gap-1">
|
||||
<span className="truncate text-sm font-medium">
|
||||
{t("popup.controlHintsToggleTitle")}
|
||||
</span>
|
||||
<span className="group relative inline-flex shrink-0">
|
||||
<button
|
||||
type="button"
|
||||
aria-label={t("popup.controlHintsInfoLabel")}
|
||||
data-slot="popup-control-hints-info"
|
||||
className="flex size-4 items-center justify-center rounded-full text-muted-foreground/70 transition-colors hover:text-muted-foreground focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring/50"
|
||||
>
|
||||
<RiInformationLine className="size-3.5" aria-hidden />
|
||||
</button>
|
||||
<span
|
||||
role="tooltip"
|
||||
className="pointer-events-none absolute bottom-full left-0 z-10 mb-1.5 w-56 whitespace-normal rounded-md bg-foreground/65 px-2 py-1 text-[10px] font-medium leading-snug text-background opacity-0 shadow-md backdrop-blur-sm transition-opacity group-hover:opacity-100 group-focus-within:opacity-100"
|
||||
>
|
||||
{t("popup.controlHintsToggleHint")}
|
||||
</span>
|
||||
</span>
|
||||
</span>
|
||||
<Switch
|
||||
checked={!controlHintsHidden}
|
||||
onCheckedChange={(shown) => setControlHintsHidden(!shown)}
|
||||
aria-label={t("popup.controlHintsToggleTitle")}
|
||||
data-slot="popup-control-hints-toggle"
|
||||
/>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{snapshot.lastError && (
|
||||
<div
|
||||
className="rounded-lg border border-destructive/25 bg-destructive/10 px-3 py-2 text-xs leading-snug text-destructive"
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
import { cleanup, fireEvent, render, screen } from "@testing-library/react";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { Switch } from "./switch";
|
||||
|
||||
describe("Switch", () => {
|
||||
afterEach(() => {
|
||||
cleanup();
|
||||
});
|
||||
|
||||
it("renders switch semantics with the checked state", () => {
|
||||
render(<Switch checked={true} onCheckedChange={vi.fn()} aria-label="开关" />);
|
||||
|
||||
const toggle = screen.getByRole("switch", { name: "开关" });
|
||||
expect(toggle.getAttribute("aria-checked")).toBe("true");
|
||||
expect(toggle.className).toContain("h-5 w-9");
|
||||
expect(toggle.className).toContain("bg-primary");
|
||||
});
|
||||
|
||||
it("calls onCheckedChange with the negated state when clicked", () => {
|
||||
const onCheckedChange = vi.fn();
|
||||
render(<Switch checked={false} onCheckedChange={onCheckedChange} aria-label="开关" />);
|
||||
|
||||
const toggle = screen.getByRole("switch", { name: "开关" });
|
||||
expect(toggle.className).toContain("bg-muted");
|
||||
|
||||
fireEvent.click(toggle);
|
||||
expect(onCheckedChange).toHaveBeenCalledWith(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,43 @@
|
||||
import { cn } from "@browser-skill/ui";
|
||||
|
||||
export interface SwitchProps {
|
||||
checked: boolean;
|
||||
onCheckedChange: (checked: boolean) => void;
|
||||
"aria-label": string;
|
||||
"data-slot"?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The popup's single toggle switch — every settings row uses this same
|
||||
* component and size; hierarchy between primary and secondary settings is
|
||||
* conveyed by copy/iconography, not by control size.
|
||||
*/
|
||||
export function Switch({
|
||||
checked,
|
||||
onCheckedChange,
|
||||
"aria-label": ariaLabel,
|
||||
"data-slot": dataSlot,
|
||||
}: SwitchProps) {
|
||||
return (
|
||||
<button
|
||||
type="button"
|
||||
role="switch"
|
||||
aria-checked={checked}
|
||||
aria-label={ariaLabel}
|
||||
data-slot={dataSlot}
|
||||
className={cn(
|
||||
"relative inline-flex h-5 w-9 shrink-0 cursor-pointer items-center rounded-full border border-transparent transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-ring/50",
|
||||
checked ? "bg-primary" : "bg-muted",
|
||||
)}
|
||||
onClick={() => onCheckedChange(!checked)}
|
||||
>
|
||||
<span
|
||||
className={cn(
|
||||
"pointer-events-none block size-4 rounded-full bg-background shadow-sm transition-transform",
|
||||
checked ? "translate-x-4" : "translate-x-0.5",
|
||||
)}
|
||||
aria-hidden
|
||||
/>
|
||||
</button>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import { useEffect, useState } from "react";
|
||||
import { getControlHintsHidden, STORAGE_KEYS, setControlHintsHidden } from "@/lib/instance-id";
|
||||
|
||||
/**
|
||||
* Popup-side view of the "hide control hints" preference in
|
||||
* `chrome.storage.local`. Defaults to shown when storage is unavailable
|
||||
* (e.g. unit tests) so the popup still renders; writes go straight to
|
||||
* storage and content scripts pick them up via `storage.onChanged`.
|
||||
*/
|
||||
export function useControlHintsHidden(): [boolean, (hidden: boolean) => void] {
|
||||
const [hidden, setHidden] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (typeof chrome === "undefined" || !chrome.storage?.local) return undefined;
|
||||
let cancelled = false;
|
||||
getControlHintsHidden()
|
||||
.then((value) => {
|
||||
if (!cancelled) setHidden(value);
|
||||
})
|
||||
.catch((err) => {
|
||||
console.debug("[browser-skill] control-hints preference read failed", err);
|
||||
});
|
||||
const onChanged = (changes: Record<string, chrome.storage.StorageChange>, areaName: string) => {
|
||||
if (areaName !== "local") return;
|
||||
const change = changes[STORAGE_KEYS.CONTROL_HINTS_HIDDEN];
|
||||
if (change) setHidden(change.newValue === true);
|
||||
};
|
||||
chrome.storage.onChanged.addListener(onChanged);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
chrome.storage.onChanged.removeListener(onChanged);
|
||||
};
|
||||
}, []);
|
||||
|
||||
const update = (value: boolean) => {
|
||||
setHidden(value);
|
||||
if (typeof chrome === "undefined" || !chrome.storage?.local) return;
|
||||
setControlHintsHidden(value).catch((err) => {
|
||||
console.debug("[browser-skill] control-hints preference write failed", err);
|
||||
});
|
||||
};
|
||||
|
||||
return [hidden, update];
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
CAPTURE_SUPPRESS,
|
||||
type CaptureSuppressMessage,
|
||||
withOverlaysHiddenForCapture,
|
||||
} from "../capture-suppress-bridge";
|
||||
|
||||
function recordingSendToTab(events: string[]) {
|
||||
return vi.fn(async (_tabId: number, message: CaptureSuppressMessage) => {
|
||||
events.push(message.phase);
|
||||
return { type: CAPTURE_SUPPRESS, ok: true };
|
||||
});
|
||||
}
|
||||
|
||||
describe("withOverlaysHiddenForCapture", () => {
|
||||
it("sends begin → fn → end and returns fn's result", async () => {
|
||||
const events: string[] = [];
|
||||
const sendToTab = recordingSendToTab(events);
|
||||
|
||||
const result = await withOverlaysHiddenForCapture(
|
||||
7,
|
||||
async () => {
|
||||
events.push("capture");
|
||||
return "shot";
|
||||
},
|
||||
sendToTab,
|
||||
);
|
||||
|
||||
expect(result).toBe("shot");
|
||||
expect(events).toEqual(["begin", "capture", "end"]);
|
||||
expect(sendToTab).toHaveBeenCalledWith(7, { type: CAPTURE_SUPPRESS, phase: "begin" });
|
||||
expect(sendToTab).toHaveBeenCalledWith(7, { type: CAPTURE_SUPPRESS, phase: "end" });
|
||||
});
|
||||
|
||||
it("still sends end when fn throws, then rethrows", async () => {
|
||||
const events: string[] = [];
|
||||
const sendToTab = recordingSendToTab(events);
|
||||
|
||||
await expect(
|
||||
withOverlaysHiddenForCapture(
|
||||
7,
|
||||
async () => {
|
||||
events.push("capture");
|
||||
throw new Error("capture exploded");
|
||||
},
|
||||
sendToTab,
|
||||
),
|
||||
).rejects.toThrow("capture exploded");
|
||||
expect(events).toEqual(["begin", "capture", "end"]);
|
||||
});
|
||||
|
||||
it("captures directly when the tab has no content script", async () => {
|
||||
const sendToTab = vi.fn(async () => {
|
||||
throw new Error("Could not establish connection. Receiving end does not exist.");
|
||||
});
|
||||
const fn = vi.fn(async () => "shot");
|
||||
|
||||
const result = await withOverlaysHiddenForCapture(7, fn, sendToTab);
|
||||
|
||||
expect(result).toBe("shot");
|
||||
expect(fn).toHaveBeenCalledTimes(1);
|
||||
// begin failed → no matching end may be sent.
|
||||
expect(sendToTab).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("swallows end failures so the capture result stands", async () => {
|
||||
const sendToTab = vi.fn(async (_tabId: number, message: CaptureSuppressMessage) => {
|
||||
if (message.phase === "end") throw new Error("tab navigated away");
|
||||
return { type: CAPTURE_SUPPRESS, ok: true };
|
||||
});
|
||||
|
||||
const result = await withOverlaysHiddenForCapture(7, async () => "shot", sendToTab);
|
||||
|
||||
expect(result).toBe("shot");
|
||||
expect(sendToTab).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
});
|
||||
@@ -1,7 +1,7 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { MIN_COMPATIBLE_PROTOCOL } from "../../transport/handshake";
|
||||
import type { ConnectionStateHandler, Transport } from "../../transport/transport";
|
||||
import type { ConnectionState, HandshakeResult } from "../../transport/types";
|
||||
import type { ConnectionStateHandler, FrameHandler, Transport } from "../../transport/transport";
|
||||
import type { ConnectionState, HandshakeResult, ProtocolFrame } from "../../transport/types";
|
||||
import { __testing__, ConnectionController } from "../connection-controller";
|
||||
|
||||
vi.mock("../instance-id", () => ({
|
||||
@@ -97,6 +97,7 @@ describe("computeConnectedState (protocol-based compat)", () => {
|
||||
function makeMockTransport(initialState: ConnectionState = "disconnected") {
|
||||
let state = initialState;
|
||||
const stateHandlers = new Set<ConnectionStateHandler>();
|
||||
const messageHandlers = new Set<FrameHandler>();
|
||||
const transport = {
|
||||
get state() {
|
||||
return state;
|
||||
@@ -110,7 +111,10 @@ function makeMockTransport(initialState: ConnectionState = "disconnected") {
|
||||
for (const h of stateHandlers) h("disconnected");
|
||||
}),
|
||||
send: vi.fn(),
|
||||
onMessage: vi.fn(() => ({ dispose: () => {} })),
|
||||
onMessage: vi.fn((handler: FrameHandler) => {
|
||||
messageHandlers.add(handler);
|
||||
return { dispose: () => messageHandlers.delete(handler) };
|
||||
}),
|
||||
onConnectionStateChange: vi.fn((handler: ConnectionStateHandler) => {
|
||||
stateHandlers.add(handler);
|
||||
return { dispose: () => stateHandlers.delete(handler) };
|
||||
@@ -119,6 +123,9 @@ function makeMockTransport(initialState: ConnectionState = "disconnected") {
|
||||
state = next;
|
||||
for (const h of stateHandlers) h(next);
|
||||
},
|
||||
emitMessage(frame: ProtocolFrame) {
|
||||
for (const handler of messageHandlers) handler(frame);
|
||||
},
|
||||
};
|
||||
return transport as typeof transport & Transport;
|
||||
}
|
||||
@@ -128,6 +135,10 @@ describe("ConnectionController connectionEnabled", () => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("does not connect on attach when connection is disabled", async () => {
|
||||
const controller = new ConnectionController();
|
||||
const transport = makeMockTransport();
|
||||
@@ -152,6 +163,36 @@ describe("ConnectionController connectionEnabled", () => {
|
||||
expect(controller.snapshot().lastError).toBeNull();
|
||||
});
|
||||
|
||||
it("runs safe session cleanup before an intentional disconnect", async () => {
|
||||
const controller = new ConnectionController();
|
||||
const transport = makeMockTransport();
|
||||
const order: string[] = [];
|
||||
transport.disconnect.mockImplementation(async () => {
|
||||
order.push("disconnect");
|
||||
});
|
||||
await controller.attach(transport, { name: "Chrome", version: "120" }, true, {
|
||||
beforeDisconnect: async () => {
|
||||
order.push("cleanup");
|
||||
},
|
||||
});
|
||||
|
||||
await controller.setConnectionEnabled(false);
|
||||
|
||||
expect(order).toEqual(["cleanup", "disconnect"]);
|
||||
});
|
||||
|
||||
it("runs session cleanup when the transport disconnects unexpectedly", async () => {
|
||||
const controller = new ConnectionController();
|
||||
const transport = makeMockTransport();
|
||||
const onDisconnected = vi.fn(async () => {});
|
||||
await controller.attach(transport, { name: "Chrome", version: "120" }, true, {
|
||||
onDisconnected,
|
||||
});
|
||||
|
||||
transport.emitState("disconnected");
|
||||
await vi.waitFor(() => expect(onDisconnected).toHaveBeenCalledTimes(1));
|
||||
});
|
||||
|
||||
it("reconnects when connection is re-enabled", async () => {
|
||||
const controller = new ConnectionController();
|
||||
const transport = makeMockTransport();
|
||||
@@ -171,4 +212,41 @@ describe("ConnectionController connectionEnabled", () => {
|
||||
|
||||
expect(controller.snapshot().state).toBe("disconnected");
|
||||
});
|
||||
|
||||
it("disconnects and retries when handshake fails while the socket is still open", async () => {
|
||||
vi.useFakeTimers();
|
||||
const controller = new ConnectionController();
|
||||
const transport = makeMockTransport();
|
||||
await controller.attach(transport, { name: "Chrome", version: "120" }, true);
|
||||
const request = transport.send.mock.calls[0]?.[0] as { id: string };
|
||||
|
||||
transport.emitMessage({
|
||||
id: request.id,
|
||||
error: { code: "protocol_error", message: "bad handshake" },
|
||||
});
|
||||
await vi.waitFor(() => expect(transport.disconnect).toHaveBeenCalledTimes(1));
|
||||
expect(controller.snapshot().state).toBe("disconnected");
|
||||
|
||||
await vi.advanceTimersByTimeAsync(1_000);
|
||||
expect(transport.connect).toHaveBeenCalledTimes(2);
|
||||
});
|
||||
|
||||
it("binds each handshake to the connection that initiated it", async () => {
|
||||
const controller = new ConnectionController();
|
||||
const transport = makeMockTransport();
|
||||
await controller.attach(transport, { name: "Chrome", version: "120" }, true);
|
||||
const first = transport.send.mock.calls[0]?.[0] as { id: string };
|
||||
|
||||
transport.emitState("disconnected");
|
||||
transport.emitState("connected");
|
||||
const second = transport.send.mock.calls[1]?.[0] as { id: string };
|
||||
expect(second.id).not.toBe(first.id);
|
||||
|
||||
transport.emitMessage({ id: first.id, result: handshake("1.0", "1.0") });
|
||||
await Promise.resolve();
|
||||
expect(controller.snapshot().state).not.toBe("connected");
|
||||
|
||||
transport.emitMessage({ id: second.id, result: handshake("1.0", "1.0") });
|
||||
await vi.waitFor(() => expect(controller.snapshot().state).toBe("connected"));
|
||||
});
|
||||
});
|
||||
|
||||
@@ -32,6 +32,7 @@ describe("help-bridge", () => {
|
||||
prompt: "log in",
|
||||
displayMode: "compact",
|
||||
selectors: ["#login"],
|
||||
rects: [{ top: 10, left: 20, width: 100, height: 40 }],
|
||||
timeoutMs: 1000,
|
||||
}),
|
||||
).toBe(true);
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
getConnectionEnabled,
|
||||
getControlHintsHidden,
|
||||
getLabel,
|
||||
getOrCreateInstanceId,
|
||||
STORAGE_KEYS,
|
||||
setConnectionEnabled,
|
||||
setControlHintsHidden,
|
||||
setLabel,
|
||||
} from "../instance-id";
|
||||
|
||||
@@ -93,4 +95,26 @@ describe("instance-id", () => {
|
||||
expect(store[STORAGE_KEYS.CONNECTION_ENABLED]).toBe(false);
|
||||
expect(await getConnectionEnabled(backend)).toBe(false);
|
||||
});
|
||||
|
||||
it("getControlHintsHidden returns false when storage is empty", async () => {
|
||||
const { backend } = fakeStorage();
|
||||
expect(await getControlHintsHidden(backend)).toBe(false);
|
||||
});
|
||||
|
||||
it("getControlHintsHidden returns persisted boolean values", async () => {
|
||||
const { backend } = fakeStorage({ [STORAGE_KEYS.CONTROL_HINTS_HIDDEN]: true });
|
||||
expect(await getControlHintsHidden(backend)).toBe(true);
|
||||
});
|
||||
|
||||
it("getControlHintsHidden treats non-boolean stored values as shown", async () => {
|
||||
const { backend } = fakeStorage({ [STORAGE_KEYS.CONTROL_HINTS_HIDDEN]: "true" });
|
||||
expect(await getControlHintsHidden(backend)).toBe(false);
|
||||
});
|
||||
|
||||
it("setControlHintsHidden persists the value retrievable by getControlHintsHidden", async () => {
|
||||
const { backend, store } = fakeStorage();
|
||||
await setControlHintsHidden(true, backend);
|
||||
expect(store[STORAGE_KEYS.CONTROL_HINTS_HIDDEN]).toBe(true);
|
||||
expect(await getControlHintsHidden(backend)).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -104,6 +104,27 @@ describe("reduceTraceSteps", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps committed empty fill steps", () => {
|
||||
const { steps } = reduceTraceSteps(
|
||||
[
|
||||
{
|
||||
op: "fill",
|
||||
target: { tag: "input", role: "textbox", name: "Search query" },
|
||||
value: "",
|
||||
page_url: "https://example.com/search",
|
||||
},
|
||||
],
|
||||
"https://example.com/search",
|
||||
);
|
||||
|
||||
expect(steps).toEqual([
|
||||
expect.objectContaining({
|
||||
op: "fill",
|
||||
value: "",
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("maps select navigated_to onto effect.navigated_to (page id)", () => {
|
||||
const { pages, steps } = reduceTraceSteps(
|
||||
[
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
/**
|
||||
* Wire protocol and background-side helper for hiding the extension's
|
||||
* in-page overlay while a screenshot is captured, so captured frames only
|
||||
* contain the page itself.
|
||||
*
|
||||
* The background wraps every capture in `withOverlaysHiddenForCapture`,
|
||||
* which sends `begin` (the content script hides the overlay host and only
|
||||
* acks once the compositor has produced an overlay-free frame) and always
|
||||
* follows up with `end`, even when the capture throws. Tabs without the
|
||||
* content script (chrome://, the Web Store, ...) host no overlay, so a
|
||||
* failed `begin` falls through to capturing directly.
|
||||
*/
|
||||
|
||||
export const CAPTURE_SUPPRESS = "bsk/capture-suppress";
|
||||
|
||||
export type CaptureSuppressPhase = "begin" | "end";
|
||||
|
||||
export interface CaptureSuppressMessage {
|
||||
type: typeof CAPTURE_SUPPRESS;
|
||||
phase: CaptureSuppressPhase;
|
||||
}
|
||||
|
||||
export interface CaptureSuppressAck {
|
||||
type: typeof CAPTURE_SUPPRESS;
|
||||
ok: true;
|
||||
}
|
||||
|
||||
export function isCaptureSuppressMessage(msg: unknown): msg is CaptureSuppressMessage {
|
||||
if (typeof msg !== "object" || msg === null) return false;
|
||||
const m = msg as Record<string, unknown>;
|
||||
return m.type === CAPTURE_SUPPRESS && (m.phase === "begin" || m.phase === "end");
|
||||
}
|
||||
|
||||
/** Minimal `chrome.tabs.sendMessage` surface so tests can inject a fake. */
|
||||
export type CaptureSuppressSendToTab = (
|
||||
tabId: number,
|
||||
message: CaptureSuppressMessage,
|
||||
) => Promise<unknown>;
|
||||
|
||||
const defaultSendToTab: CaptureSuppressSendToTab = (tabId, message) =>
|
||||
chrome.tabs.sendMessage(tabId, message);
|
||||
|
||||
/**
|
||||
* Run `fn` with the target tab's overlay hidden. `end` is sent from a
|
||||
* `finally` so the overlay always reappears, and a missing content script
|
||||
* (or any other `begin` failure) simply skips suppression — there is no
|
||||
* overlay to hide in that tab.
|
||||
*/
|
||||
export async function withOverlaysHiddenForCapture<T>(
|
||||
tabId: number,
|
||||
fn: () => Promise<T>,
|
||||
sendToTab: CaptureSuppressSendToTab = defaultSendToTab,
|
||||
): Promise<T> {
|
||||
let began = false;
|
||||
try {
|
||||
await sendToTab(tabId, { type: CAPTURE_SUPPRESS, phase: "begin" });
|
||||
began = true;
|
||||
} catch {
|
||||
// No content script in the target tab → no overlay can leak into the
|
||||
// capture; proceed without suppression.
|
||||
}
|
||||
try {
|
||||
return await fn();
|
||||
} finally {
|
||||
if (began) {
|
||||
await sendToTab(tabId, { type: CAPTURE_SUPPRESS, phase: "end" }).catch((err) => {
|
||||
console.debug("[bsk capture] overlay restore failed", err);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -9,6 +9,8 @@ import type { ConnectionState, HandshakeResult } from "../transport/types";
|
||||
import { getLabel, getOrCreateInstanceId } from "./instance-id";
|
||||
import { compareProtocol, parseProtocolMajor } from "./semver";
|
||||
|
||||
const HANDSHAKE_RETRY_DELAY_MS = 1_000;
|
||||
|
||||
export interface SnapshotInfo {
|
||||
state: ConnectionState;
|
||||
instanceId: string;
|
||||
@@ -21,6 +23,13 @@ export interface SnapshotInfo {
|
||||
|
||||
type Listener = (s: SnapshotInfo) => void;
|
||||
|
||||
export interface ConnectionLifecycleHooks {
|
||||
/** Safe local teardown that must finish before an intentional disconnect. */
|
||||
beforeDisconnect?: () => void | Promise<void>;
|
||||
/** Best-effort teardown after an unexpected transport loss. */
|
||||
onDisconnected?: () => void | Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Orchestrates Transport + handshake lifecycle for the background SW.
|
||||
*
|
||||
@@ -36,7 +45,12 @@ export class ConnectionController {
|
||||
private lastError: string | null = null;
|
||||
private connectionEnabled = true;
|
||||
private listeners = new Set<Listener>();
|
||||
private handshakeInFlight = false;
|
||||
private lifecycleHooks: ConnectionLifecycleHooks = {};
|
||||
private disconnectRecovery: Promise<void> | null = null;
|
||||
private connectionGeneration = 0;
|
||||
private handshakeAbort: AbortController | null = null;
|
||||
private handshakeRetryTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private suppressDisconnectRecovery = false;
|
||||
|
||||
get isConnectionEnabled(): boolean {
|
||||
return this.connectionEnabled;
|
||||
@@ -68,20 +82,36 @@ export class ConnectionController {
|
||||
transport: Transport,
|
||||
browser: { name: string; version: string },
|
||||
connectionEnabled = true,
|
||||
lifecycleHooks: ConnectionLifecycleHooks = {},
|
||||
): Promise<void> {
|
||||
this.transport = transport;
|
||||
this.connectionEnabled = connectionEnabled;
|
||||
this.lifecycleHooks = lifecycleHooks;
|
||||
this.instanceId = await getOrCreateInstanceId();
|
||||
this.label = await getLabel();
|
||||
|
||||
transport.onConnectionStateChange((s) => {
|
||||
if (!this.connectionEnabled) return;
|
||||
if (s === "connected") {
|
||||
void this.runHandshake(browser);
|
||||
if (s === "disconnected") {
|
||||
// An in-flight handshake belongs to the dead connection — cancel it so
|
||||
// a late resolution can never clobber the next attempt.
|
||||
this.cancelHandshake();
|
||||
this.handshake = null;
|
||||
if (this.connectionEnabled) {
|
||||
this.setState("disconnected");
|
||||
if (this.suppressDisconnectRecovery) {
|
||||
// Deliberate disconnect from runHandshake (rejected peer or failed
|
||||
// handshake bounce); the caller owns the reconnect policy there.
|
||||
this.suppressDisconnectRecovery = false;
|
||||
} else {
|
||||
void this.recoverFromDisconnect();
|
||||
}
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (s === "disconnected") {
|
||||
this.handshake = null;
|
||||
if (!this.connectionEnabled) return;
|
||||
if (s === "connected") {
|
||||
this.startHandshake(browser);
|
||||
return;
|
||||
}
|
||||
this.setState(s);
|
||||
});
|
||||
@@ -126,41 +156,69 @@ export class ConnectionController {
|
||||
this.fire();
|
||||
}
|
||||
|
||||
private async runHandshake(browser: { name: string; version: string }): Promise<void> {
|
||||
private startHandshake(browser: { name: string; version: string }): void {
|
||||
this.cancelHandshake();
|
||||
const generation = ++this.connectionGeneration;
|
||||
const abort = new AbortController();
|
||||
this.handshakeAbort = abort;
|
||||
void this.runHandshake(browser, generation, abort.signal);
|
||||
}
|
||||
|
||||
private async runHandshake(
|
||||
browser: { name: string; version: string },
|
||||
generation: number,
|
||||
signal: AbortSignal,
|
||||
): Promise<void> {
|
||||
if (!this.transport) return;
|
||||
if (!this.connectionEnabled) return;
|
||||
if (this.handshakeInFlight) return;
|
||||
this.handshakeInFlight = true;
|
||||
this.setState("connecting");
|
||||
try {
|
||||
const outcome = await performHandshake(this.transport, {
|
||||
instanceId: this.instanceId,
|
||||
browser,
|
||||
label: this.label,
|
||||
});
|
||||
const outcome = await performHandshake(
|
||||
this.transport,
|
||||
{
|
||||
instanceId: this.instanceId,
|
||||
browser,
|
||||
label: this.label,
|
||||
},
|
||||
{ signal },
|
||||
);
|
||||
if (generation !== this.connectionGeneration || signal.aborted) return;
|
||||
this.handshake = outcome.result;
|
||||
const verdict = computeConnectedState(outcome.result);
|
||||
if (verdict.kind === "rejected") {
|
||||
this.lastError = `version_too_old: ${verdict.reason}`;
|
||||
this.handshake = null;
|
||||
// Rejected peers must not be auto-retried: mark this disconnect as
|
||||
// deliberate so the listener skips unexpected-loss recovery.
|
||||
this.suppressDisconnectRecovery = true;
|
||||
await this.transport.disconnect().catch(() => {});
|
||||
this.setState("disconnected");
|
||||
return;
|
||||
}
|
||||
this.clearHandshakeRetry();
|
||||
this.lastError = null;
|
||||
this.setState(verdict.kind);
|
||||
} catch (err) {
|
||||
if (generation !== this.connectionGeneration || signal.aborted || isAbortError(err)) return;
|
||||
this.handshake = null;
|
||||
this.lastError = err instanceof Error ? err.message : String(err);
|
||||
// Bounce the half-open socket deliberately; the retry timer below owns
|
||||
// the reconnect, so skip unexpected-loss recovery for this disconnect.
|
||||
this.suppressDisconnectRecovery = true;
|
||||
await this.transport.disconnect().catch(() => {});
|
||||
this.setState("disconnected");
|
||||
this.scheduleHandshakeRetry(browser);
|
||||
} finally {
|
||||
this.handshakeInFlight = false;
|
||||
if (generation === this.connectionGeneration) this.handshakeAbort = null;
|
||||
}
|
||||
}
|
||||
|
||||
private async applyDisabledState(): Promise<void> {
|
||||
this.cancelHandshake();
|
||||
this.clearHandshakeRetry();
|
||||
this.handshake = null;
|
||||
this.lastError = null;
|
||||
await this.lifecycleHooks.beforeDisconnect?.();
|
||||
if (this.transport) {
|
||||
await this.transport.disconnect().catch(() => {});
|
||||
}
|
||||
@@ -169,6 +227,60 @@ export class ConnectionController {
|
||||
if (was === "disconnected") this.fire();
|
||||
}
|
||||
|
||||
private recoverFromDisconnect(): Promise<void> {
|
||||
if (this.disconnectRecovery) return this.disconnectRecovery;
|
||||
const recovery = (async () => {
|
||||
// WSTransport schedules its reconnect timer immediately after notifying
|
||||
// state listeners. Yield once, then disconnect explicitly so that timer
|
||||
// is cancelled before local session teardown begins.
|
||||
await Promise.resolve();
|
||||
if (!this.connectionEnabled || !this.transport) return;
|
||||
// Another path already reconnected while we yielded — nothing to do.
|
||||
if (this.currentState !== "disconnected") return;
|
||||
await this.transport.disconnect().catch(() => {});
|
||||
try {
|
||||
await this.lifecycleHooks.onDisconnected?.();
|
||||
} catch (err) {
|
||||
console.warn("[browser-skill] session cleanup after disconnect failed", err);
|
||||
}
|
||||
if (!this.connectionEnabled) return;
|
||||
try {
|
||||
await this.transport.connect();
|
||||
} catch (err) {
|
||||
this.lastError = err instanceof Error ? err.message : String(err);
|
||||
this.setState("disconnected");
|
||||
}
|
||||
})();
|
||||
this.disconnectRecovery = recovery.finally(() => {
|
||||
this.disconnectRecovery = null;
|
||||
});
|
||||
return this.disconnectRecovery;
|
||||
}
|
||||
|
||||
private cancelHandshake(): void {
|
||||
this.connectionGeneration += 1;
|
||||
this.handshakeAbort?.abort();
|
||||
this.handshakeAbort = null;
|
||||
}
|
||||
|
||||
private scheduleHandshakeRetry(browser: { name: string; version: string }): void {
|
||||
if (this.handshakeRetryTimer || !this.connectionEnabled) return;
|
||||
this.handshakeRetryTimer = setTimeout(() => {
|
||||
this.handshakeRetryTimer = null;
|
||||
if (!this.connectionEnabled || !this.transport) return;
|
||||
void this.transport.connect().catch((err) => {
|
||||
this.lastError = err instanceof Error ? err.message : String(err);
|
||||
this.setState("disconnected");
|
||||
});
|
||||
}, HANDSHAKE_RETRY_DELAY_MS);
|
||||
}
|
||||
|
||||
private clearHandshakeRetry(): void {
|
||||
if (!this.handshakeRetryTimer) return;
|
||||
clearTimeout(this.handshakeRetryTimer);
|
||||
this.handshakeRetryTimer = null;
|
||||
}
|
||||
|
||||
private setState(next: ConnectionState): void {
|
||||
if (this.currentState === next) return;
|
||||
this.currentState = next;
|
||||
@@ -187,6 +299,12 @@ export class ConnectionController {
|
||||
}
|
||||
}
|
||||
|
||||
function isAbortError(err: unknown): boolean {
|
||||
return (
|
||||
typeof err === "object" && err !== null && (err as { name?: string }).name === "AbortError"
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Verdict of the symmetric post-handshake compat check.
|
||||
*/
|
||||
|
||||
@@ -16,6 +16,13 @@ export const HELP_ACK = "bsk-help-ack";
|
||||
export const HELP_QUERY = "bsk-help-query";
|
||||
export const HELP_FINISH = "bsk-help-finish";
|
||||
|
||||
export interface HelpHighlightRect {
|
||||
top: number;
|
||||
left: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface HelpRequestMessage {
|
||||
type: typeof HELP_REQUEST;
|
||||
requestId: string;
|
||||
@@ -26,6 +33,8 @@ export interface HelpRequestMessage {
|
||||
displayMode?: "full" | "compact";
|
||||
/** CSS selectors to scroll to + flash-highlight (may be empty). */
|
||||
selectors: string[];
|
||||
/** Top-viewport rectangles for targets that cannot be represented by a root selector. */
|
||||
rects?: HelpHighlightRect[];
|
||||
timeoutMs: number;
|
||||
}
|
||||
|
||||
@@ -74,6 +83,16 @@ export function isHelpRequestMessage(msg: unknown): msg is HelpRequestMessage {
|
||||
(m.displayMode === undefined || m.displayMode === "full" || m.displayMode === "compact") &&
|
||||
Array.isArray(m.selectors) &&
|
||||
m.selectors.every((selector) => typeof selector === "string") &&
|
||||
(m.rects === undefined ||
|
||||
(Array.isArray(m.rects) &&
|
||||
m.rects.every(
|
||||
(rect) =>
|
||||
typeof rect === "object" &&
|
||||
rect !== null &&
|
||||
["top", "left", "width", "height"].every((key) =>
|
||||
Number.isFinite((rect as Record<string, unknown>)[key]),
|
||||
),
|
||||
))) &&
|
||||
typeof m.timeoutMs === "number"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
const STORAGE_KEY = "bsk_instance_id";
|
||||
const LABEL_STORAGE_KEY = "bh_label";
|
||||
const CONNECTION_ENABLED_KEY = "bh_connection_enabled";
|
||||
const CONTROL_HINTS_HIDDEN_KEY = "bsk_control_hints_hidden";
|
||||
|
||||
export interface StorageBackend {
|
||||
get(keys: string | string[]): Promise<Record<string, unknown>>;
|
||||
@@ -105,8 +106,30 @@ export async function setConnectionEnabled(
|
||||
await storage.set({ [CONNECTION_ENABLED_KEY]: enabled });
|
||||
}
|
||||
|
||||
/**
|
||||
* User preference for the in-page control hints (status pill + orange glow
|
||||
* shown while the Agent controls a tab). Defaults to shown when unset or
|
||||
* non-boolean; when hidden the whole control overlay — including its input
|
||||
* blocker — is skipped so the page looks and behaves normally.
|
||||
*/
|
||||
export async function getControlHintsHidden(
|
||||
storage: StorageBackend = defaultStorage(),
|
||||
): Promise<boolean> {
|
||||
const items = await storage.get(CONTROL_HINTS_HIDDEN_KEY);
|
||||
const raw = items[CONTROL_HINTS_HIDDEN_KEY];
|
||||
return typeof raw === "boolean" ? raw : false;
|
||||
}
|
||||
|
||||
export async function setControlHintsHidden(
|
||||
hidden: boolean,
|
||||
storage: StorageBackend = defaultStorage(),
|
||||
): Promise<void> {
|
||||
await storage.set({ [CONTROL_HINTS_HIDDEN_KEY]: hidden });
|
||||
}
|
||||
|
||||
export const STORAGE_KEYS = {
|
||||
INSTANCE_ID: STORAGE_KEY,
|
||||
LABEL: LABEL_STORAGE_KEY,
|
||||
CONNECTION_ENABLED: CONNECTION_ENABLED_KEY,
|
||||
CONTROL_HINTS_HIDDEN: CONTROL_HINTS_HIDDEN_KEY,
|
||||
} as const;
|
||||
|
||||
@@ -25,11 +25,19 @@ export interface RecordQueryMessage {
|
||||
export interface RecordQueryResponse {
|
||||
active: boolean;
|
||||
requestId?: string;
|
||||
/** Epoch ms when the recording began; see `RecordStartMessage.startedAtMs`. */
|
||||
startedAtMs?: number;
|
||||
}
|
||||
|
||||
export interface RecordStartMessage {
|
||||
type: typeof RECORD_START;
|
||||
requestId: string;
|
||||
/**
|
||||
* Epoch ms when the whole recording began, not when this tab was armed.
|
||||
* The overlay timer must span the session, so it survives navigations and
|
||||
* content-script remounts instead of restarting per page.
|
||||
*/
|
||||
startedAtMs?: number;
|
||||
}
|
||||
|
||||
export interface RecordStepPayload {
|
||||
|
||||
@@ -18,7 +18,6 @@ export function shouldRecordPress(
|
||||
}
|
||||
|
||||
function shouldIncludeDraft(step: DraftTraceStep): boolean {
|
||||
if (step.op === "fill" && !(step.value ?? "").trim() && !step.redacted) return false;
|
||||
if (step.op === "press" && !shouldRecordPress(step.key, step.modifiers)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -44,3 +44,53 @@ describe("chromeAgentWindowApi.ensureActiveTab", () => {
|
||||
expect(update).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe("chromeAgentWindowApi.create", () => {
|
||||
const create = vi.fn();
|
||||
|
||||
beforeEach(() => {
|
||||
vi.stubGlobal("chrome", {
|
||||
windows: { create },
|
||||
});
|
||||
create.mockReset();
|
||||
create.mockResolvedValue({ id: 100 });
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
it("focuses Agent Windows by default", async () => {
|
||||
await chromeAgentWindowApi.create(AGENT_WINDOW_HOME);
|
||||
|
||||
expect(create).toHaveBeenCalledWith({
|
||||
type: "normal",
|
||||
focused: true,
|
||||
url: AGENT_WINDOW_HOME,
|
||||
});
|
||||
});
|
||||
|
||||
it("can create an Agent Window without stealing focus", async () => {
|
||||
await chromeAgentWindowApi.create(AGENT_WINDOW_HOME, { focused: false });
|
||||
|
||||
expect(create).toHaveBeenCalledWith({
|
||||
type: "normal",
|
||||
focused: false,
|
||||
url: AGENT_WINDOW_HOME,
|
||||
});
|
||||
});
|
||||
|
||||
it("passes an optional window size through the options object", async () => {
|
||||
await chromeAgentWindowApi.create(AGENT_WINDOW_HOME, {
|
||||
size: { width: 1280, height: 800 },
|
||||
});
|
||||
|
||||
expect(create).toHaveBeenCalledWith({
|
||||
type: "normal",
|
||||
focused: true,
|
||||
url: AGENT_WINDOW_HOME,
|
||||
width: 1280,
|
||||
height: 800,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { createDisconnectCleanup } from "../disconnect-cleanup";
|
||||
import { SessionManager } from "../manager";
|
||||
|
||||
describe("disconnect session cleanup", () => {
|
||||
it("stops every local session through the safe session-stop path", async () => {
|
||||
const remove = vi.fn(async () => {});
|
||||
let nextWindowId = 100;
|
||||
const manager = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => nextWindowId++),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
remove,
|
||||
},
|
||||
});
|
||||
await manager.start("aa11");
|
||||
await manager.start("bb22");
|
||||
const detachSession = vi.fn(async () => {});
|
||||
const onSessionsChanged = vi.fn();
|
||||
const cleanup = createDisconnectCleanup({
|
||||
manager,
|
||||
sessionStopDeps: { cdp: { detachSession } },
|
||||
onSessionsChanged,
|
||||
});
|
||||
|
||||
const report = await cleanup();
|
||||
|
||||
expect(report).toEqual({ stoppedSessionIds: ["aa11", "bb22"], failures: [] });
|
||||
expect(manager.list()).toEqual([]);
|
||||
expect(remove).toHaveBeenCalledTimes(2);
|
||||
expect(detachSession).toHaveBeenCalledWith("aa11");
|
||||
expect(detachSession).toHaveBeenCalledWith("bb22");
|
||||
expect(onSessionsChanged).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("coalesces overlapping disconnect notifications", async () => {
|
||||
let releaseRemove: () => void = () => {};
|
||||
const removeGate = new Promise<void>((resolve) => {
|
||||
releaseRemove = resolve;
|
||||
});
|
||||
const manager = new SessionManager({
|
||||
agentWindow: {
|
||||
create: vi.fn(async () => 100),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
remove: vi.fn(() => removeGate),
|
||||
},
|
||||
});
|
||||
await manager.start("aa11");
|
||||
const cleanup = createDisconnectCleanup({ manager });
|
||||
|
||||
const first = cleanup();
|
||||
const second = cleanup();
|
||||
expect(second).toBe(first);
|
||||
releaseRemove();
|
||||
|
||||
await expect(first).resolves.toEqual({ stoppedSessionIds: ["aa11"], failures: [] });
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,5 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { AgentWindowApi } from "../agent-window";
|
||||
import type { AgentWindowApi, AgentWindowCreateOptions } from "../agent-window";
|
||||
import { SessionManager } from "../manager";
|
||||
|
||||
function fakeAgentWindow(): AgentWindowApi & {
|
||||
@@ -8,7 +8,7 @@ function fakeAgentWindow(): AgentWindowApi & {
|
||||
ensureActiveTabMock: ReturnType<typeof vi.fn>;
|
||||
} {
|
||||
let nextId = 100;
|
||||
const createMock = vi.fn(async (_url: string) => {
|
||||
const createMock = vi.fn(async (_url: string, _opts?: AgentWindowCreateOptions) => {
|
||||
const id = nextId++;
|
||||
return id;
|
||||
});
|
||||
@@ -30,7 +30,7 @@ describe("SessionManager", () => {
|
||||
const sm = new SessionManager({ agentWindow: aw, now: () => 1700000000000 });
|
||||
const ctx = await sm.start("aa11");
|
||||
expect(aw.createMock).toHaveBeenCalledOnce();
|
||||
expect(aw.createMock).toHaveBeenCalledWith("about:blank");
|
||||
expect(aw.createMock).toHaveBeenCalledWith("about:blank", {});
|
||||
expect(aw.ensureActiveTabMock).toHaveBeenCalledOnce();
|
||||
expect(aw.ensureActiveTabMock).toHaveBeenCalledWith(100, "about:blank");
|
||||
expect(ctx.sessionId).toBe("aa11");
|
||||
@@ -42,11 +42,22 @@ describe("SessionManager", () => {
|
||||
it("forwards an optional window size when starting a session", async () => {
|
||||
const aw = fakeAgentWindow();
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const ctx = await sm.start("aa11", { width: 1280, height: 800 });
|
||||
expect(aw.createMock).toHaveBeenCalledWith("about:blank", { width: 1280, height: 800 });
|
||||
const ctx = await sm.start("aa11", { size: { width: 1280, height: 800 } });
|
||||
expect(aw.createMock).toHaveBeenCalledWith("about:blank", {
|
||||
size: { width: 1280, height: 800 },
|
||||
});
|
||||
expect(ctx.agentWindowId).toBe(100);
|
||||
});
|
||||
|
||||
it("forwards an explicit unfocused start to the Agent Window", async () => {
|
||||
const aw = fakeAgentWindow();
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
|
||||
await sm.start("aa11", { focused: false });
|
||||
|
||||
expect(aw.createMock).toHaveBeenCalledWith("about:blank", { focused: false });
|
||||
});
|
||||
|
||||
it("indexes the session by sessionId and agent window id", async () => {
|
||||
const aw = fakeAgentWindow();
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
@@ -64,6 +75,52 @@ describe("SessionManager", () => {
|
||||
await expect(sm.start("aa11")).rejects.toThrow(/already exists/);
|
||||
});
|
||||
|
||||
it("removes a newly created Agent Window when startup is aborted", async () => {
|
||||
const aw = fakeAgentWindow();
|
||||
let resolveCreate: (windowId: number) => void = () => {};
|
||||
aw.createMock.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise<number>((resolve) => {
|
||||
resolveCreate = resolve;
|
||||
}),
|
||||
);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const controller = new AbortController();
|
||||
const pending = sm.start("aa11", { signal: controller.signal });
|
||||
|
||||
controller.abort();
|
||||
resolveCreate(777);
|
||||
|
||||
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
|
||||
expect(aw.removeMock).toHaveBeenCalledWith(777);
|
||||
expect(sm.has("aa11")).toBe(false);
|
||||
});
|
||||
|
||||
it("removes an incomplete Agent Window when active-tab setup fails", async () => {
|
||||
const aw = fakeAgentWindow();
|
||||
aw.ensureActiveTabMock.mockRejectedValueOnce(new Error("tab setup failed"));
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
|
||||
await expect(sm.start("aa11")).rejects.toThrow("tab setup failed");
|
||||
|
||||
expect(aw.removeMock).toHaveBeenCalledWith(100);
|
||||
expect(sm.has("aa11")).toBe(false);
|
||||
});
|
||||
|
||||
it("surfaces the orphan Agent Window id when startup cleanup fails", async () => {
|
||||
const aw = fakeAgentWindow();
|
||||
aw.ensureActiveTabMock.mockRejectedValueOnce(new Error("tab setup failed"));
|
||||
aw.removeMock.mockRejectedValueOnce(new Error("window removal denied"));
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
|
||||
await expect(sm.start("aa11")).rejects.toMatchObject({
|
||||
name: "SessionStartCleanupError",
|
||||
windowId: 100,
|
||||
message: expect.stringMatching(/cleanup of Agent Window 100 failed.*window removal denied/),
|
||||
});
|
||||
expect(sm.has("aa11")).toBe(false);
|
||||
});
|
||||
|
||||
it("stop() closes the Agent Window and forgets the session", async () => {
|
||||
const aw = fakeAgentWindow();
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
|
||||
@@ -33,4 +33,19 @@ describe("RefStore", () => {
|
||||
expect(s.resolve("@e11")).toBe(11);
|
||||
expect(s.size()).toBe(2);
|
||||
});
|
||||
|
||||
it("preserves the child CDP session as part of ref identity", () => {
|
||||
const s = new RefStore();
|
||||
s.set("e1", 42, {
|
||||
tabId: 7,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
expect(s.resolveEntry("e1")).toMatchObject({
|
||||
backendNodeId: 42,
|
||||
tabId: 7,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
*/
|
||||
|
||||
export interface AgentWindowApi {
|
||||
create(url: string, size?: { width: number; height: number }): Promise<number>;
|
||||
create(url: string, opts?: AgentWindowCreateOptions): Promise<number>;
|
||||
remove(windowId: number): Promise<void>;
|
||||
/**
|
||||
* Guarantee the Agent Window has an active, CDP-navigable tab.
|
||||
@@ -18,16 +18,24 @@ export interface AgentWindowApi {
|
||||
ensureActiveTab(windowId: number, url: string): Promise<void>;
|
||||
}
|
||||
|
||||
/** Creation hints for a new Agent Window. */
|
||||
export interface AgentWindowCreateOptions {
|
||||
/** Optional outer size in CSS pixels. */
|
||||
size?: { width: number; height: number };
|
||||
/** Defaults to true so existing sessions keep visible Agent Windows. */
|
||||
focused?: boolean;
|
||||
}
|
||||
|
||||
/** Initial tab URL for every new session's Agent Window. */
|
||||
export const AGENT_WINDOW_HOME = "about:blank";
|
||||
|
||||
export const chromeAgentWindowApi: AgentWindowApi = {
|
||||
async create(url: string, size?: { width: number; height: number }): Promise<number> {
|
||||
async create(url: string, opts: AgentWindowCreateOptions = {}): Promise<number> {
|
||||
const win = await chrome.windows.create({
|
||||
type: "normal",
|
||||
focused: true,
|
||||
focused: opts.focused ?? true,
|
||||
url,
|
||||
...(size ? { width: size.width, height: size.height } : {}),
|
||||
...(opts.size ? { width: opts.size.width, height: opts.size.height } : {}),
|
||||
});
|
||||
if (typeof win?.id !== "number") {
|
||||
throw new Error("[bh] chrome.windows.create returned no window id");
|
||||
@@ -35,12 +43,11 @@ export const chromeAgentWindowApi: AgentWindowApi = {
|
||||
return win.id;
|
||||
},
|
||||
async remove(windowId: number): Promise<void> {
|
||||
try {
|
||||
await chrome.windows.remove(windowId);
|
||||
} catch (err) {
|
||||
// Window may have been closed by the user already; ignore.
|
||||
console.debug("[bh] chrome.windows.remove failed", err);
|
||||
}
|
||||
// Callers decide whether a missing/failed removal is benign. In
|
||||
// particular, transactional session-start cleanup must be able to
|
||||
// surface a window it could not remove instead of reporting a false
|
||||
// cancellation success while the Agent Window remains open.
|
||||
await chrome.windows.remove(windowId);
|
||||
},
|
||||
async ensureActiveTab(windowId: number, url: string): Promise<void> {
|
||||
const tabs = await chrome.tabs.query({ windowId });
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
import { handleSessionStop, type SessionStopDeps } from "@/tools/session";
|
||||
import type { SessionManager } from "./manager";
|
||||
|
||||
export interface DisconnectCleanupFailure {
|
||||
sessionId: string;
|
||||
message: string;
|
||||
}
|
||||
|
||||
export interface DisconnectCleanupReport {
|
||||
stoppedSessionIds: string[];
|
||||
failures: DisconnectCleanupFailure[];
|
||||
}
|
||||
|
||||
export interface DisconnectCleanupOptions {
|
||||
manager: SessionManager;
|
||||
sessionStopDeps?: SessionStopDeps;
|
||||
onSessionsChanged?: () => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a coalesced cleanup operation for transport loss.
|
||||
*
|
||||
* The daemon purges its registry when the extension socket disappears. To
|
||||
* keep the extension-side mirror consistent, every local session must follow
|
||||
* the normal safe stop path as well: return borrowed tabs, clear refs, detach
|
||||
* CDP, and only then close the Agent Window.
|
||||
*/
|
||||
export function createDisconnectCleanup(options: DisconnectCleanupOptions) {
|
||||
let inFlight: Promise<DisconnectCleanupReport> | null = null;
|
||||
|
||||
return (): Promise<DisconnectCleanupReport> => {
|
||||
if (inFlight) return inFlight;
|
||||
inFlight = cleanupSessions(options).finally(() => {
|
||||
inFlight = null;
|
||||
});
|
||||
return inFlight;
|
||||
};
|
||||
}
|
||||
|
||||
async function cleanupSessions(
|
||||
options: DisconnectCleanupOptions,
|
||||
): Promise<DisconnectCleanupReport> {
|
||||
const stoppedSessionIds: string[] = [];
|
||||
const failures: DisconnectCleanupFailure[] = [];
|
||||
|
||||
for (const ctx of options.manager.list()) {
|
||||
try {
|
||||
const result = await handleSessionStop(
|
||||
options.manager,
|
||||
{ session_id: ctx.sessionId },
|
||||
options.sessionStopDeps,
|
||||
);
|
||||
if ("code" in result) {
|
||||
failures.push({ sessionId: ctx.sessionId, message: result.message });
|
||||
continue;
|
||||
}
|
||||
if (result.return_failures?.length) {
|
||||
failures.push({
|
||||
sessionId: ctx.sessionId,
|
||||
message: result.return_failures.map((failure) => failure.message).join("; "),
|
||||
});
|
||||
continue;
|
||||
}
|
||||
stoppedSessionIds.push(ctx.sessionId);
|
||||
} catch (err) {
|
||||
failures.push({
|
||||
sessionId: ctx.sessionId,
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
options.onSessionsChanged?.();
|
||||
return { stoppedSessionIds, failures };
|
||||
}
|
||||
@@ -25,6 +25,46 @@ export interface SessionManagerOptions {
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
/** Options for starting a session's Agent Window. */
|
||||
export interface SessionStartOptions {
|
||||
/** Optional Agent Window outer size in CSS pixels. */
|
||||
size?: { width: number; height: number };
|
||||
/** Defaults to true so existing clients keep visible Agent Windows. */
|
||||
focused?: boolean;
|
||||
/** Cancellation for the transactional Agent Window startup sequence. */
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export class SessionStartCleanupError extends Error {
|
||||
readonly windowId: number;
|
||||
readonly startupError: unknown;
|
||||
readonly cleanupError: unknown;
|
||||
|
||||
constructor(windowId: number, startupError: unknown, cleanupError: unknown) {
|
||||
const startupMessage =
|
||||
startupError instanceof Error ? startupError.message : String(startupError);
|
||||
const cleanupMessage =
|
||||
cleanupError instanceof Error ? cleanupError.message : String(cleanupError);
|
||||
super(
|
||||
`session_start failed (${startupMessage}) and cleanup of Agent Window ${windowId} failed: ${cleanupMessage}`,
|
||||
);
|
||||
this.name = "SessionStartCleanupError";
|
||||
this.windowId = windowId;
|
||||
this.startupError = startupError;
|
||||
this.cleanupError = cleanupError;
|
||||
}
|
||||
}
|
||||
|
||||
function sessionStartAbortError(): Error {
|
||||
const error = new Error("session_start aborted");
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
function throwIfSessionStartAborted(signal: AbortSignal | undefined): void {
|
||||
if (signal?.aborted) throw sessionStartAbortError();
|
||||
}
|
||||
|
||||
/**
|
||||
* Owner of all live agent sessions inside the extension.
|
||||
*
|
||||
@@ -128,29 +168,40 @@ export class SessionManager {
|
||||
* Returns the created window id so callers can echo it back to the
|
||||
* daemon in the `tool.session_start` reply.
|
||||
*/
|
||||
async start(
|
||||
sessionId: string,
|
||||
size?: { width: number; height: number },
|
||||
): Promise<SessionContext> {
|
||||
async start(sessionId: string, opts: SessionStartOptions = {}): Promise<SessionContext> {
|
||||
if (this.sessions.has(sessionId)) {
|
||||
throw new Error(`[bh] session ${sessionId} already exists`);
|
||||
}
|
||||
// Only pass `size` when given so the no-size call shape (and its
|
||||
// chrome.windows.create payload) stays exactly as before.
|
||||
const windowId = size
|
||||
? await this.agentWindow.create(AGENT_WINDOW_HOME, size)
|
||||
: await this.agentWindow.create(AGENT_WINDOW_HOME);
|
||||
await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
|
||||
const ctx: SessionContext = {
|
||||
sessionId,
|
||||
agentWindowId: windowId,
|
||||
refStore: new RefStore(),
|
||||
borrowedTabs: new Map(),
|
||||
createdAtMs: this.now(),
|
||||
};
|
||||
this.sessions.set(sessionId, ctx);
|
||||
this.windowIndex.set(windowId, sessionId);
|
||||
return ctx;
|
||||
throwIfSessionStartAborted(opts.signal);
|
||||
|
||||
let windowId: number | null = null;
|
||||
try {
|
||||
const { signal: _signal, ...createOptions } = opts;
|
||||
windowId = await this.agentWindow.create(AGENT_WINDOW_HOME, createOptions);
|
||||
throwIfSessionStartAborted(opts.signal);
|
||||
await this.agentWindow.ensureActiveTab(windowId, AGENT_WINDOW_HOME);
|
||||
throwIfSessionStartAborted(opts.signal);
|
||||
|
||||
const ctx: SessionContext = {
|
||||
sessionId,
|
||||
agentWindowId: windowId,
|
||||
refStore: new RefStore(),
|
||||
borrowedTabs: new Map(),
|
||||
createdAtMs: this.now(),
|
||||
};
|
||||
this.sessions.set(sessionId, ctx);
|
||||
this.windowIndex.set(windowId, sessionId);
|
||||
return ctx;
|
||||
} catch (startupError) {
|
||||
if (windowId !== null) {
|
||||
try {
|
||||
await this.agentWindow.remove(windowId);
|
||||
} catch (cleanupError) {
|
||||
throw new SessionStartCleanupError(windowId, startupError, cleanupError);
|
||||
}
|
||||
}
|
||||
throw startupError;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
/**
|
||||
* Per-session map from `@e<N>` snapshot refs to CDP `backendNodeId`.
|
||||
* Per-session map from `@e<N>` snapshot refs to a CDP node address.
|
||||
*
|
||||
* Each fresh `tool.snapshot` resets the store: M6 will call
|
||||
* `replace(...)` with the new ref → backendNodeId pairs. Tools like
|
||||
* `tool.click` consume the store via `resolve("@e1")`.
|
||||
* `replace(...)` with the new ref → node address pairs. A node address
|
||||
* includes the owning flat CDP session/frame when the element lives in
|
||||
* an OOPIF; tools resolve live geometry from that identity at call time.
|
||||
*
|
||||
* Refs are session-scoped (§7): looking up a ref in the wrong session
|
||||
* returns `null`, never silently leaks. Storing values in different
|
||||
@@ -15,10 +16,19 @@ export type BackendNodeId = number;
|
||||
export interface RefEntry {
|
||||
backendNodeId: BackendNodeId;
|
||||
tabId: number | null;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
generation: number;
|
||||
}
|
||||
|
||||
type RefInput = BackendNodeId | { backendNodeId: BackendNodeId; tabId: number };
|
||||
export type RefInput =
|
||||
| BackendNodeId
|
||||
| {
|
||||
backendNodeId: BackendNodeId;
|
||||
tabId: number;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
};
|
||||
|
||||
export class RefStore {
|
||||
private readonly map = new Map<string, RefEntry>();
|
||||
@@ -44,7 +54,7 @@ export class RefStore {
|
||||
}
|
||||
|
||||
/**
|
||||
* Replace the entire store with a new ref → backendNodeId mapping.
|
||||
* Replace the entire store with a new ref → CDP node identity mapping.
|
||||
* Used after every fresh `tool.snapshot`.
|
||||
*/
|
||||
replace(entries: Iterable<readonly [string, RefInput]>): void {
|
||||
@@ -53,10 +63,20 @@ export class RefStore {
|
||||
for (const [ref, input] of entries) this.map.set(normaliseRef(ref), this.entry(input));
|
||||
}
|
||||
|
||||
set(ref: string, id: BackendNodeId, opts: { tabId?: number } = {}): void {
|
||||
set(
|
||||
ref: string,
|
||||
id: BackendNodeId,
|
||||
opts: {
|
||||
tabId?: number;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
} = {},
|
||||
): void {
|
||||
this.map.set(normaliseRef(ref), {
|
||||
backendNodeId: id,
|
||||
tabId: opts.tabId ?? null,
|
||||
...(opts.frameId ? { frameId: opts.frameId } : {}),
|
||||
...(opts.cdpSessionId ? { cdpSessionId: opts.cdpSessionId } : {}),
|
||||
generation: this.generation,
|
||||
});
|
||||
}
|
||||
@@ -80,6 +100,8 @@ export class RefStore {
|
||||
return {
|
||||
backendNodeId: input.backendNodeId,
|
||||
tabId: input.tabId,
|
||||
...(input.frameId ? { frameId: input.frameId } : {}),
|
||||
...(input.cdpSessionId ? { cdpSessionId: input.cdpSessionId } : {}),
|
||||
generation: this.generation,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -612,4 +612,14 @@ describe("isInjectableContentScriptUrl", () => {
|
||||
).toBe(false);
|
||||
expect(isInjectableContentScriptUrl("https://chromewebstore.google.com/")).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects the Edge Add-ons store", () => {
|
||||
expect(
|
||||
isInjectableContentScriptUrl(
|
||||
"https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg",
|
||||
),
|
||||
).toBe(false);
|
||||
// Non-store paths on the same host stay injectable.
|
||||
expect(isInjectableContentScriptUrl("https://microsoftedge.microsoft.com/")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -69,6 +69,25 @@ describe("ToolDispatcher", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("forwards an unfocused session start to the Agent Window", async () => {
|
||||
const { transport, deliver } = fakeTransport();
|
||||
const create = vi.fn(async () => 4242);
|
||||
const sessions = new SessionManager({
|
||||
agentWindow: {
|
||||
create,
|
||||
remove: vi.fn(),
|
||||
ensureActiveTab: vi.fn(async () => {}),
|
||||
},
|
||||
});
|
||||
const dispatcher = new ToolDispatcher({ transport, sessions });
|
||||
dispatcher.start();
|
||||
|
||||
deliver(makeRequest("tool.session_start", { session_id: "aa11", focused: false }));
|
||||
await flushMicrotasks();
|
||||
|
||||
expect(create).toHaveBeenCalledWith("about:blank", { focused: false });
|
||||
});
|
||||
|
||||
it("routes tool.session_stop and replies with empty result", async () => {
|
||||
const { transport, sent, deliver } = fakeTransport();
|
||||
const sessions = new SessionManager({
|
||||
@@ -658,23 +677,31 @@ describe("ToolDispatcher", () => {
|
||||
await flushMicrotasks();
|
||||
expect(ac?.signal.aborted).toBe(true);
|
||||
|
||||
// Cancel ack arrived synchronously; the slow tool replies with
|
||||
// `cancelled` once the dispatcher's race observes the abort.
|
||||
// Cancel ack arrives synchronously, but the original RPC must not reply
|
||||
// until the in-progress window creation has completed and been rolled back.
|
||||
const ack = sent.find(
|
||||
(m) =>
|
||||
typeof (m as { id?: string }).id === "string" && (m as { id: string }).id === "cancel-1",
|
||||
);
|
||||
expect(ack).toEqual({ id: "cancel-1", result: { cancelled: true } });
|
||||
|
||||
expect(
|
||||
sent.find(
|
||||
(m) => typeof (m as { id?: string }).id === "string" && (m as { id: string }).id === "r-1",
|
||||
),
|
||||
).toBeUndefined();
|
||||
expect(dispatcher.inflightAbortControllers.has("r-1")).toBe(true);
|
||||
|
||||
resolveCreate(9999);
|
||||
await flushMicrotasks();
|
||||
|
||||
const slow = sent.find(
|
||||
(m) => typeof (m as { id?: string }).id === "string" && (m as { id: string }).id === "r-1",
|
||||
);
|
||||
expect(slow).toMatchObject({ id: "r-1", error: { code: "cancelled" } });
|
||||
expect(dispatcher.inflightAbortControllers.has("r-1")).toBe(false);
|
||||
|
||||
// Drain the dangling create promise so vitest does not warn.
|
||||
resolveCreate(9999);
|
||||
await flushMicrotasks();
|
||||
expect(sessions.has("aa44")).toBe(false);
|
||||
expect(sessions.list()).toEqual([]);
|
||||
});
|
||||
|
||||
it("cancel for an unknown rpc_id replies with cancelled=false", async () => {
|
||||
|
||||
@@ -1,24 +1,54 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { nodeBoundingRect, quadBoundingRect } from "../element-geometry";
|
||||
import { nodeContentRegion } from "../element-geometry";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
describe("quadBoundingRect", () => {
|
||||
it("computes axis-aligned bounds from an 8-double quad", () => {
|
||||
expect(quadBoundingRect([10, 20, 110, 20, 110, 60, 10, 60])).toEqual({
|
||||
x: 10,
|
||||
y: 20,
|
||||
width: 100,
|
||||
height: 40,
|
||||
describe("nodeContentRegion", () => {
|
||||
it("rejects invalid content quads before using the box model fallback", async () => {
|
||||
const send = async (_tabId: number, method: string) => {
|
||||
if (method === "DOM.getContentQuads") return { quads: [[0, 0, 0, 0, 0, 0, 0, 0]] };
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [5, 10, 25, 10, 25, 30, 5, 30] } };
|
||||
}
|
||||
throw new Error(`unexpected CDP call ${method}`);
|
||||
};
|
||||
|
||||
await expect(nodeContentRegion({ send: send as CdpRunner["send"] }, 7, 555)).resolves.toEqual([
|
||||
[
|
||||
{ x: 5, y: 10 },
|
||||
{ x: 25, y: 10 },
|
||||
{ x: 25, y: 30 },
|
||||
{ x: 5, y: 30 },
|
||||
],
|
||||
]);
|
||||
});
|
||||
|
||||
it("preserves every visible content fragment", async () => {
|
||||
const send = async () => ({
|
||||
quads: [
|
||||
[10, 20, 60, 20, 60, 40, 10, 40],
|
||||
[15, 50, 90, 50, 90, 80, 15, 80],
|
||||
],
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as CdpRunner["send"],
|
||||
};
|
||||
|
||||
await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([
|
||||
[
|
||||
{ x: 10, y: 20 },
|
||||
{ x: 60, y: 20 },
|
||||
{ x: 60, y: 40 },
|
||||
{ x: 10, y: 40 },
|
||||
],
|
||||
[
|
||||
{ x: 15, y: 50 },
|
||||
{ x: 90, y: 50 },
|
||||
{ x: 90, y: 80 },
|
||||
{ x: 15, y: 80 },
|
||||
],
|
||||
]);
|
||||
});
|
||||
|
||||
it("returns null for degenerate quads", () => {
|
||||
expect(quadBoundingRect([0, 0, 0, 0, 0, 0, 0, 0])).toBeNull();
|
||||
expect(quadBoundingRect([1, 2, 3])).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("nodeBoundingRect", () => {
|
||||
it("falls back to visible descendant bounds for zero-size containers", async () => {
|
||||
const calls: Array<{ method: string; params?: object }> = [];
|
||||
const send = async (_tabId: number, method: string, params?: object) => {
|
||||
@@ -38,12 +68,14 @@ describe("nodeBoundingRect", () => {
|
||||
};
|
||||
const cdp = { send: send as CdpRunner["send"] };
|
||||
|
||||
await expect(nodeBoundingRect(cdp, 7, 555)).resolves.toEqual({
|
||||
x: 242,
|
||||
y: 468,
|
||||
width: 769,
|
||||
height: 180,
|
||||
});
|
||||
await expect(nodeContentRegion(cdp, 7, 555)).resolves.toEqual([
|
||||
[
|
||||
{ x: 242, y: 468 },
|
||||
{ x: 1011, y: 468 },
|
||||
{ x: 1011, y: 648 },
|
||||
{ x: 242, y: 648 },
|
||||
],
|
||||
]);
|
||||
expect(calls.map((c) => c.method)).toEqual([
|
||||
"DOM.getContentQuads",
|
||||
"DOM.getBoxModel",
|
||||
|
||||
@@ -0,0 +1,286 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { resolveFrameProjection, resolveNodeGeometry } from "../frame-geometry";
|
||||
import {
|
||||
clipPolygon,
|
||||
polygonArea,
|
||||
polygonBounds,
|
||||
projectAndClipRegion,
|
||||
projectUnitPoint,
|
||||
type Quad,
|
||||
rectPolygon,
|
||||
regionBounds,
|
||||
} from "../geometry";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
describe("frame geometry projection", () => {
|
||||
it("keeps region bounds separate from polygon area", () => {
|
||||
const region = [
|
||||
rectPolygon({ x: 0, y: 0, w: 10, h: 10 }),
|
||||
rectPolygon({ x: 20, y: 5, w: 5, h: 5 }),
|
||||
];
|
||||
expect(regionBounds(region)).toEqual({ x: 0, y: 0, width: 25, height: 10 });
|
||||
expect(polygonArea(region[0])).toBe(100);
|
||||
});
|
||||
|
||||
it("maps through the iframe content box instead of its border box", () => {
|
||||
const contentQuad: Quad = [
|
||||
{ x: 204, y: 306 },
|
||||
{ x: 604, y: 306 },
|
||||
{ x: 604, y: 506 },
|
||||
{ x: 204, y: 506 },
|
||||
];
|
||||
const projected = projectAndClipRegion(
|
||||
[rectPolygon({ x: 10, y: 20, w: 100, h: 40 })],
|
||||
[{ sourceViewport: { width: 200, height: 100 }, destinationQuad: contentQuad }],
|
||||
{ width: 1000, height: 800 },
|
||||
);
|
||||
expect(polygonBounds(projected.flat())).toEqual({ x: 224, y: 346, width: 200, height: 80 });
|
||||
});
|
||||
|
||||
it("uses a projective mapping for perspective-transformed iframe quads", () => {
|
||||
const quad: Quad = [
|
||||
{ x: 0, y: 0 },
|
||||
{ x: 200, y: 20 },
|
||||
{ x: 180, y: 140 },
|
||||
{ x: 20, y: 100 },
|
||||
];
|
||||
expect(projectUnitPoint({ x: 0, y: 0 }, quad)).toEqual(quad[0]);
|
||||
expect(projectUnitPoint({ x: 1, y: 0 }, quad)).toEqual(quad[1]);
|
||||
expect(projectUnitPoint({ x: 1, y: 1 }, quad)).toEqual(quad[2]);
|
||||
expect(projectUnitPoint({ x: 0, y: 1 }, quad)).toEqual(quad[3]);
|
||||
});
|
||||
|
||||
it("clips against the iframe polygon rather than its bounding box", () => {
|
||||
const frameQuad: Quad = [
|
||||
{ x: 50, y: 0 },
|
||||
{ x: 100, y: 50 },
|
||||
{ x: 50, y: 100 },
|
||||
{ x: 0, y: 50 },
|
||||
];
|
||||
expect(clipPolygon(rectPolygon({ x: 0, y: 0, w: 10, h: 10 }), frameQuad)).toEqual([]);
|
||||
});
|
||||
|
||||
it("does not reapply same-target frame transforms around an OOPIF", async () => {
|
||||
const send = vi.fn(async (_tabId, method, params) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||
}
|
||||
if (method === "DOM.getBoxModel") {
|
||||
const backendNodeId = (params as { backendNodeId?: number }).backendNodeId;
|
||||
return backendNodeId === 20
|
||||
? { model: { content: [110, 70, 310, 70, 310, 170, 110, 170] } }
|
||||
: { model: { content: [100, 50, 400, 50, 400, 250, 100, 250] } };
|
||||
}
|
||||
throw new Error(`unexpected root command ${method}`);
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: send as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
throw new Error(`unexpected child command ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
};
|
||||
const projection = await resolveFrameProjection(
|
||||
cdp,
|
||||
{
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "same-process-parent",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4 },
|
||||
},
|
||||
{
|
||||
frameId: "oopif",
|
||||
parentFrameId: "same-process-parent",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 4, sessionId: "oopif-session" },
|
||||
},
|
||||
],
|
||||
},
|
||||
"oopif",
|
||||
);
|
||||
|
||||
expect(projection?.edges).toEqual([
|
||||
{
|
||||
sourceViewport: { width: 200, height: 100 },
|
||||
destinationQuad: [
|
||||
{ x: 110, y: 70 },
|
||||
{ x: 310, y: 70 },
|
||||
{ x: 310, y: 170 },
|
||||
{ x: 110, y: 170 },
|
||||
],
|
||||
destinationClips: [
|
||||
[
|
||||
{ x: 100, y: 50 },
|
||||
{ x: 400, y: 50 },
|
||||
{ x: 400, y: 250 },
|
||||
{ x: 100, y: 250 },
|
||||
],
|
||||
],
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("clips live OOPIF geometry at every frame boundary", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||
}
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||
}
|
||||
throw new Error(`unexpected root command ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[-50, -20, 250, -20, 250, 120, -50, 120]] };
|
||||
}
|
||||
throw new Error(`unexpected child command ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const geometry = await resolveNodeGeometry(cdp, 4, {
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
frameId: "child",
|
||||
backendNodeId: 101,
|
||||
});
|
||||
|
||||
expect(geometry).toMatchObject({
|
||||
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
||||
actionPoint: { x: 200, y: 150 },
|
||||
});
|
||||
});
|
||||
|
||||
it("reuses one frame graph while scrolling and projecting live geometry", async () => {
|
||||
const getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
}));
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||
}
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||
}
|
||||
throw new Error(`unexpected root command ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||
}
|
||||
throw new Error(`unexpected child command ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
getFrameGraph,
|
||||
};
|
||||
|
||||
await expect(
|
||||
resolveNodeGeometry(
|
||||
cdp,
|
||||
4,
|
||||
{
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
frameId: "child",
|
||||
backendNodeId: 101,
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
),
|
||||
).resolves.not.toMatchObject({ code: expect.any(String) });
|
||||
expect(getFrameGraph).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("clips live geometry to same-target iframe ancestors", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 800, clientHeight: 600 } };
|
||||
}
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [100, 100, 300, 100, 300, 200, 100, 200] } };
|
||||
}
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[50, 50, 350, 50, 350, 250, 50, 250]] };
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4 },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const geometry = await resolveNodeGeometry(cdp, 4, {
|
||||
target: { tabId: 4 },
|
||||
frameId: "child",
|
||||
backendNodeId: 101,
|
||||
});
|
||||
|
||||
expect(geometry).toMatchObject({
|
||||
topBounds: { x: 100, y: 100, width: 200, height: 100 },
|
||||
actionPoint: { x: 200, y: 150 },
|
||||
});
|
||||
});
|
||||
|
||||
it("fails closed when an OOPIF frame graph is unavailable", async () => {
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(),
|
||||
sendToTarget: vi.fn(async (_target, method) => {
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[0, 0, 20, 0, 20, 20, 0, 20]] };
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}) as CdpRunner["sendToTarget"],
|
||||
};
|
||||
|
||||
await expect(
|
||||
resolveNodeGeometry(cdp, 4, {
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
frameId: "child",
|
||||
backendNodeId: 101,
|
||||
}),
|
||||
).resolves.toMatchObject({ code: "cdp_failed" });
|
||||
});
|
||||
});
|
||||
@@ -1,5 +1,6 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import { RefStore } from "@/session-manager/ref-store";
|
||||
import type { RequestHelpParams } from "@/transport/types";
|
||||
import { handleRequestHelp, type RequestHelpDeps, resetHelpLifecycleForTests } from "../human-loop";
|
||||
|
||||
@@ -50,11 +51,10 @@ function installHelpLifecycleChrome() {
|
||||
}
|
||||
|
||||
function fakeManager(sessionId: string, agentWindowId: number, tabId: number) {
|
||||
const refStore = new RefStore();
|
||||
const mgr = {
|
||||
get: (id: string) =>
|
||||
id === sessionId
|
||||
? { sessionId, agentWindowId, refStore: { resolve: () => null }, borrowedTabs: new Map() }
|
||||
: null,
|
||||
id === sessionId ? { sessionId, agentWindowId, refStore, borrowedTabs: new Map() } : null,
|
||||
findByWindowId: (wid: number) => (wid === agentWindowId ? { sessionId } : null),
|
||||
} as unknown as SessionManager;
|
||||
return mgr;
|
||||
@@ -339,26 +339,23 @@ describe("handleRequestHelp", () => {
|
||||
|
||||
ac.abort();
|
||||
await expect(pending).resolves.toMatchObject({ code: "cancelled" });
|
||||
expect(cdpSend).toHaveBeenCalledWith(
|
||||
5,
|
||||
"DOM.querySelectorAll",
|
||||
expect.objectContaining({ selector: "[data-bsk-help]" }),
|
||||
);
|
||||
expect(cdpSend).toHaveBeenCalledWith(
|
||||
6,
|
||||
"DOM.querySelectorAll",
|
||||
expect.objectContaining({ selector: "[data-bsk-help]" }),
|
||||
expect(cdpSend).not.toHaveBeenCalledWith(
|
||||
expect.anything(),
|
||||
"DOM.removeAttribute",
|
||||
expect.anything(),
|
||||
);
|
||||
});
|
||||
|
||||
it("tags ref targets via CDP and reports them matched", async () => {
|
||||
it("resolves ref targets to explicit viewport rectangles without mutating the page", async () => {
|
||||
const refStore = new RefStore();
|
||||
refStore.set("e1", 42, { tabId: 5 });
|
||||
const mgr = {
|
||||
get: (id: string) =>
|
||||
id === "abcd"
|
||||
? {
|
||||
sessionId: "abcd",
|
||||
agentWindowId: 99,
|
||||
refStore: { resolve: () => 42 },
|
||||
refStore,
|
||||
borrowedTabs: new Map(),
|
||||
}
|
||||
: null,
|
||||
@@ -366,7 +363,17 @@ describe("handleRequestHelp", () => {
|
||||
} as unknown as SessionManager;
|
||||
const deps = baseDeps({
|
||||
cdp: {
|
||||
send: vi.fn(async () => ({ object: { objectId: "obj-1" } })),
|
||||
send: vi.fn(async (_tabId, method) => {
|
||||
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
throw new Error(`unexpected ${method}`);
|
||||
}),
|
||||
} as unknown as RequestHelpDeps["cdp"],
|
||||
});
|
||||
const res = await handleRequestHelp(
|
||||
@@ -375,14 +382,97 @@ describe("handleRequestHelp", () => {
|
||||
deps,
|
||||
);
|
||||
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
|
||||
expect(sentMsg.selectors).toContain('[data-bsk-help="0"]');
|
||||
expect(sentMsg.selectors).toEqual([]);
|
||||
expect(sentMsg.rects).toEqual([{ top: 20, left: 10, width: 100, height: 40 }]);
|
||||
expect(res).toMatchObject({
|
||||
outcome: "continued",
|
||||
tab_id: 5,
|
||||
resolved_targets: [{ matched: true, ref: "@e1" }],
|
||||
resolved_targets: [
|
||||
{
|
||||
matched: true,
|
||||
ref: "@e1",
|
||||
},
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
it("projects an OOPIF ref through its live content quad for the help overlay", async () => {
|
||||
const mgr = {
|
||||
get: (id: string) =>
|
||||
id === "abcd"
|
||||
? {
|
||||
sessionId: "abcd",
|
||||
agentWindowId: 99,
|
||||
refStore: {
|
||||
resolveEntry: () => ({
|
||||
backendNodeId: 42,
|
||||
tabId: 5,
|
||||
frameId: "child",
|
||||
cdpSessionId: "child-session",
|
||||
generation: 1,
|
||||
}),
|
||||
},
|
||||
borrowedTabs: new Map(),
|
||||
}
|
||||
: null,
|
||||
findByWindowId: (wid: number) => (wid === 99 ? { sessionId: "abcd" } : null),
|
||||
} as unknown as SessionManager;
|
||||
const send = vi.fn(async (_tabId, method) => {
|
||||
if (method === "DOM.getDocument") return { root: { nodeId: 1 } };
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getBoxModel") {
|
||||
return { model: { content: [204, 306, 604, 306, 604, 506, 204, 506] } };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
throw new Error(`unexpected root ${method}`);
|
||||
});
|
||||
const sendToTarget = vi.fn(async (_target, method) => {
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
throw new Error(`unexpected child ${method}`);
|
||||
});
|
||||
const deps = baseDeps({
|
||||
cdp: {
|
||||
send,
|
||||
sendToTarget,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 5 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 99,
|
||||
target: { tabId: 5, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
} as unknown as NonNullable<RequestHelpDeps["cdp"]>,
|
||||
});
|
||||
|
||||
const res = await handleRequestHelp(
|
||||
mgr,
|
||||
baseParams({ tab_id: 5, targets: [{ ref: "@e1" }] }),
|
||||
deps,
|
||||
);
|
||||
|
||||
const sentMsg = (deps.sendToTab as ReturnType<typeof vi.fn>).mock.calls[0][1];
|
||||
expect(sentMsg.rects).toEqual([{ top: 346, left: 224, width: 200, height: 80 }]);
|
||||
expect(res).toMatchObject({ resolved_targets: [{ matched: true, ref: "@e1" }] });
|
||||
expect(sendToTarget).toHaveBeenCalledWith(
|
||||
{ tabId: 5, sessionId: "child-session" },
|
||||
"DOM.getContentQuads",
|
||||
{ backendNodeId: 42 },
|
||||
);
|
||||
});
|
||||
|
||||
it("reports ref target unmatched when ref does not resolve", async () => {
|
||||
const deps = baseDeps();
|
||||
const res = await handleRequestHelp(
|
||||
@@ -398,16 +488,15 @@ describe("handleRequestHelp", () => {
|
||||
});
|
||||
|
||||
it("reports ref target unmatched when ref is for another tab", async () => {
|
||||
const refStore = new RefStore();
|
||||
refStore.set("e1", 42, { tabId: 4 });
|
||||
const mgr = {
|
||||
get: (id: string) =>
|
||||
id === "abcd"
|
||||
? {
|
||||
sessionId: "abcd",
|
||||
agentWindowId: 99,
|
||||
refStore: {
|
||||
resolve: (ref: string, opts: { tabId?: number }) =>
|
||||
ref === "e1" && opts.tabId === 4 ? 42 : null,
|
||||
},
|
||||
refStore,
|
||||
borrowedTabs: new Map(),
|
||||
}
|
||||
: null,
|
||||
|
||||
@@ -30,6 +30,9 @@ function makeFakeCdp(handlers: Record<string, (params: unknown) => unknown>) {
|
||||
const sendImpl = async (tabId: number, method: string, params?: object) => {
|
||||
sent.push({ tabId, method, params });
|
||||
const h = handlers[method];
|
||||
if (!h && method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
if (!h) throw new Error(`unexpected CDP call ${method}`);
|
||||
return h(params);
|
||||
};
|
||||
@@ -164,6 +167,63 @@ describe("handleClick", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("resolves frame refs in their CDP session and dispatches input in top coordinates", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
const fake = makeFakeCdp({
|
||||
"Input.dispatchMouseEvent": () => ({}),
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.getBoxModel": () => ({
|
||||
model: { content: [204, 306, 604, 306, 604, 506, 204, 506] },
|
||||
}),
|
||||
});
|
||||
fake.cdp.getFrameGraph = vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child-frame",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 99,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
}));
|
||||
const targetCalls: Array<{ sessionId?: string; method: string }> = [];
|
||||
fake.cdp.sendToTarget = vi.fn(async (target, method) => {
|
||||
targetCalls.push({ sessionId: target.sessionId, method });
|
||||
if (method === "DOM.scrollIntoViewIfNeeded") return {};
|
||||
if (method === "DOM.getContentQuads") {
|
||||
return { quads: [[10, 20, 110, 20, 110, 60, 10, 60]] };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 200, clientHeight: 100 } };
|
||||
}
|
||||
throw new Error(`unexpected child CDP call ${method}`);
|
||||
}) as CdpRunner["sendToTarget"];
|
||||
|
||||
const res = await handleClick(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e3" },
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
// The iframe content box starts at (204, 306) after its border and is scaled 2x.
|
||||
expect(res).toMatchObject({ x: 324, y: 386 });
|
||||
expect(targetCalls).toEqual([
|
||||
{ sessionId: "child-session", method: "DOM.scrollIntoViewIfNeeded" },
|
||||
{ sessionId: "child-session", method: "DOM.getContentQuads" },
|
||||
{ sessionId: "child-session", method: "Page.getLayoutMetrics" },
|
||||
]);
|
||||
expect(fake.sent.filter((call) => call.method === "Input.dispatchMouseEvent")).toHaveLength(3);
|
||||
});
|
||||
|
||||
it("enables overlay bypass before mouse events when overlay blocks the click point", async () => {
|
||||
const order: string[] = [];
|
||||
const bypassOverlay = vi.fn(async (_tabId: number, enabled: boolean) => {
|
||||
|
||||
@@ -49,6 +49,8 @@ function makeFakeCdp(opts?: {
|
||||
beforeLoaderId?: string;
|
||||
readyState?: string;
|
||||
readyStateSequence?: string[];
|
||||
/** Make the readyState probe's `Runtime.evaluate` throw (returns null). */
|
||||
evaluateThrows?: boolean;
|
||||
}) {
|
||||
const readyStates = opts?.readyStateSequence ?? (opts?.readyState ? [opts.readyState] : []);
|
||||
let readyStateIdx = 0;
|
||||
@@ -94,7 +96,10 @@ function makeFakeCdp(opts?: {
|
||||
return {};
|
||||
},
|
||||
"Runtime.enable": () => ({}),
|
||||
"Runtime.evaluate": () => ({ result: { value: nextReadyState() } }),
|
||||
"Runtime.evaluate": () => {
|
||||
if (opts?.evaluateThrows) throw new Error("Cannot execute in a destroyed execution context");
|
||||
return { result: { value: nextReadyState() } };
|
||||
},
|
||||
"Page.getNavigationHistory": () => ({
|
||||
currentIndex: opts?.historyIndex ?? 1,
|
||||
entries: opts?.historyEntries ?? [
|
||||
@@ -168,8 +173,13 @@ function makeFakeCdp(opts?: {
|
||||
const payload = { name, frameId, loaderId };
|
||||
for (const listener of [...events]) listener({ tabId: 4 }, "Page.lifecycleEvent", payload);
|
||||
},
|
||||
fireFrameNavigated(frameId = opts?.navigateFrameId ?? "frame-1") {
|
||||
const payload = { frame: { id: frameId, url: opts?.finalUrl ?? "https://example.com/" } };
|
||||
fireFrameNavigated(
|
||||
frameId = opts?.navigateFrameId ?? "frame-1",
|
||||
loaderId = opts?.navigateLoaderId ?? "loader-after",
|
||||
) {
|
||||
const payload = {
|
||||
frame: { id: frameId, loaderId, url: opts?.finalUrl ?? "https://example.com/" },
|
||||
};
|
||||
for (const listener of [...events]) listener({ tabId: 4 }, "Page.frameNavigated", payload);
|
||||
},
|
||||
listeners: events,
|
||||
@@ -457,6 +467,34 @@ describe("handleNavigate", () => {
|
||||
expect(res.reached).toBe("commit");
|
||||
expect(fake.listeners.length).toBe(0);
|
||||
});
|
||||
|
||||
it("resolves wait_until=commit from Page.frameNavigated when the readyState probe fails", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
// Cross-origin-style navigation: the readyState probe can't read the
|
||||
// (already destroyed) old execution context, so commit must come from
|
||||
// the Page.frameNavigated event. Before the fix this timed out because
|
||||
// the handler dropped the frame's loaderId and the loader guard then
|
||||
// rejected the event.
|
||||
const fake = makeFakeCdp({ evaluateThrows: true });
|
||||
const navP = handleNavigate(
|
||||
sm,
|
||||
{
|
||||
session_id: "aa11",
|
||||
url: "https://example.com/",
|
||||
wait_until: "commit",
|
||||
timeout_ms: 1_000,
|
||||
},
|
||||
{ cdp: fake.cdp, tabsApi: fake.tabsApi },
|
||||
);
|
||||
|
||||
await new Promise((r) => setTimeout(r, 5));
|
||||
fake.fireFrameNavigated("frame-1", "loader-after");
|
||||
const res = await navP;
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.reached).toBe("commit");
|
||||
expect(fake.listeners.length).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe("handleNavigateBack / Forward", () => {
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
import { renderVom } from "@browser-skill/vom";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { CAPTURE_SUPPRESS, type CaptureSuppressMessage } from "@/lib/capture-suppress-bridge";
|
||||
import { OVERLAY_HOST_MARKER_ATTR, OVERLAY_HOST_NAME } from "@/lib/overlay-bridge";
|
||||
import { SessionManager } from "@/session-manager/manager";
|
||||
import type { CdpRunner } from "@/tools/shared";
|
||||
import {
|
||||
buildFrameVomScene,
|
||||
buildVomScene,
|
||||
type CdpAxNode,
|
||||
captureVomObservation,
|
||||
handleGetHtml,
|
||||
handleObserve,
|
||||
handleScreenshot,
|
||||
@@ -61,6 +64,9 @@ function makeFakeCdp(handlers: Record<string, (params?: object) => unknown>) {
|
||||
const send = vi.fn(async (_tabId: number, method: string, params?: object) => {
|
||||
sent.push({ method, params });
|
||||
const handler = handlers[method];
|
||||
if (!handler && method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1280, clientHeight: 720 } };
|
||||
}
|
||||
if (!handler) throw new Error(`unexpected CDP call ${method}`);
|
||||
return handler(params);
|
||||
});
|
||||
@@ -195,6 +201,71 @@ describe("handleScreenshot", () => {
|
||||
expect(res).toMatchObject({ code: "cdp_failed", message: /captureVisibleTab refused/ });
|
||||
});
|
||||
|
||||
it("falls back to CDP Page.captureScreenshot when captureVisibleTab fails", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const capture = vi.fn(async () => {
|
||||
throw new Error("Failed to capture tab: image readback failed");
|
||||
});
|
||||
const { cdp, sent } = makeFakeCdp({
|
||||
"Page.captureScreenshot": () => ({ data: TINY_PNG }),
|
||||
});
|
||||
const res = await handleScreenshot(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
makeScreenshotDeps({ cdp, captureVisibleTab: capture }),
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.tab_id).toBe(7);
|
||||
expect(res.image_base64).toBe(TINY_PNG);
|
||||
expect(res.format).toBe("png");
|
||||
expect(capture).toHaveBeenCalledTimes(1);
|
||||
const fallbackCall = sent.find((c) => c.method === "Page.captureScreenshot");
|
||||
expect(fallbackCall?.params).toEqual({ format: "png", fromSurface: true });
|
||||
});
|
||||
|
||||
it("reports screenshot_capture_failed when both capture paths fail", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const capture = vi.fn(async () => {
|
||||
throw new Error("Failed to capture tab: image readback failed");
|
||||
});
|
||||
const { cdp } = makeFakeCdp({
|
||||
"Page.captureScreenshot": () => {
|
||||
throw new Error("debugger detached");
|
||||
},
|
||||
});
|
||||
const res = await handleScreenshot(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
makeScreenshotDeps({ cdp, captureVisibleTab: capture }),
|
||||
);
|
||||
expect(res).toMatchObject({
|
||||
code: "cdp_failed",
|
||||
data: { reason: "screenshot_capture_failed" },
|
||||
});
|
||||
const message = (res as { message?: string }).message ?? "";
|
||||
expect(message).toContain("image readback failed");
|
||||
expect(message).toContain("debugger detached");
|
||||
});
|
||||
|
||||
it("falls back to CDP when captureVisibleTab returns and CDP yields data", async () => {
|
||||
// Sanity: a successful primary path never touches CDP even when a
|
||||
// runner is available.
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const capture = vi.fn(async () => `data:image/png;base64,${TINY_PNG}`);
|
||||
const { cdp, sent } = makeFakeCdp({});
|
||||
const res = await handleScreenshot(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
makeScreenshotDeps({ cdp, captureVisibleTab: capture }),
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.image_base64).toBe(TINY_PNG);
|
||||
expect(sent.find((c) => c.method === "Page.captureScreenshot")).toBeUndefined();
|
||||
});
|
||||
|
||||
it("captures a clipped PNG when ref is given", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
@@ -345,11 +416,389 @@ describe("handleScreenshot", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("handleScreenshot overlay suppression", () => {
|
||||
/**
|
||||
* Fake background→content bridge that mirrors the content script's
|
||||
* contract: `begin` hides the overlay, `end` restores it. Records the
|
||||
* phase order so tests can assert begin → capture → end.
|
||||
*/
|
||||
function fakeSuppressBridge(events: string[]) {
|
||||
let overlayHidden = false;
|
||||
const sendToTab = vi.fn(async (_tabId: number, message: CaptureSuppressMessage) => {
|
||||
events.push(message.phase);
|
||||
overlayHidden = message.phase === "begin";
|
||||
return { type: CAPTURE_SUPPRESS, ok: true };
|
||||
});
|
||||
return { sendToTab, isHidden: () => overlayHidden };
|
||||
}
|
||||
|
||||
it("hides the overlay around a visible-tab capture", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const events: string[] = [];
|
||||
const bridge = fakeSuppressBridge(events);
|
||||
const capture = vi.fn(async (_w: number) => {
|
||||
events.push("capture");
|
||||
// The overlay must already be hidden when the capture runs.
|
||||
expect(bridge.isHidden()).toBe(true);
|
||||
return `data:image/png;base64,${TINY_PNG}`;
|
||||
});
|
||||
const res = await handleScreenshot(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ ...makeScreenshotDeps({ captureVisibleTab: capture }), sendToTab: bridge.sendToTab },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.image_base64).toBe(TINY_PNG);
|
||||
expect(events).toEqual(["begin", "capture", "end"]);
|
||||
expect(bridge.isHidden()).toBe(false);
|
||||
});
|
||||
|
||||
it("hides the overlay around a CDP element capture", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e5", 999, { tabId: 7 });
|
||||
const events: string[] = [];
|
||||
const bridge = fakeSuppressBridge(events);
|
||||
const { cdp } = makeFakeCdp({
|
||||
"DOM.scrollIntoViewIfNeeded": () => ({}),
|
||||
"DOM.getContentQuads": () => ({ quads: [[10, 20, 110, 20, 110, 60, 10, 60]] }),
|
||||
"Page.captureScreenshot": () => {
|
||||
events.push("capture");
|
||||
expect(bridge.isHidden()).toBe(true);
|
||||
return { data: TINY_PNG };
|
||||
},
|
||||
});
|
||||
const res = await handleScreenshot(
|
||||
sm,
|
||||
{ session_id: "aa11", ref: "@e5", tab_id: 7 },
|
||||
{
|
||||
...makeScreenshotDeps({
|
||||
cdp,
|
||||
get: vi.fn(async () => ({ id: 7, windowId: 100, active: false }) as chrome.tabs.Tab),
|
||||
query: vi.fn(),
|
||||
captureVisibleTab: vi.fn(),
|
||||
}),
|
||||
sendToTab: bridge.sendToTab,
|
||||
},
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.image_base64).toBe(TINY_PNG);
|
||||
expect(events).toEqual(["begin", "capture", "end"]);
|
||||
expect(bridge.isHidden()).toBe(false);
|
||||
});
|
||||
|
||||
it("still captures when the tab has no content script to ack", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
const sendToTab = vi.fn(async () => {
|
||||
throw new Error("Could not establish connection. Receiving end does not exist.");
|
||||
});
|
||||
const capture = vi.fn(async (_w: number) => `data:image/png;base64,${TINY_PNG}`);
|
||||
const res = await handleScreenshot(
|
||||
sm,
|
||||
{ session_id: "aa11" },
|
||||
{ ...makeScreenshotDeps({ captureVisibleTab: capture }), sendToTab },
|
||||
);
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.image_base64).toBe(TINY_PNG);
|
||||
expect(capture).toHaveBeenCalledTimes(1);
|
||||
// The failed begin must not be followed by an end.
|
||||
expect(sendToTab).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// buildVomScene
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe("buildVomScene", () => {
|
||||
it("merges multiple sibling and nested frames without mixing their targets", () => {
|
||||
const node = (
|
||||
backendNodeId: number,
|
||||
parentBackendNodeId: number | null,
|
||||
frameId: string,
|
||||
tag: string,
|
||||
attrs: Record<string, string> = {},
|
||||
): CapturedNode => ({
|
||||
backendNodeId,
|
||||
parentBackendNodeId,
|
||||
frameId,
|
||||
tag,
|
||||
attrs,
|
||||
rect: { x: 0, y: 0, w: 100, h: 30 },
|
||||
localRect: { x: 0, y: 0, w: 100, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
});
|
||||
const mainNodes = [
|
||||
node(1, null, "main", "body"),
|
||||
node(10, 1, "main", "iframe", { title: "First" }),
|
||||
node(20, 1, "main", "iframe", { title: "Second" }),
|
||||
];
|
||||
const firstNodes = [
|
||||
node(100, null, "first", "body"),
|
||||
node(101, 100, "first", "button", { "aria-label": "First action" }),
|
||||
];
|
||||
const secondNodes = [
|
||||
node(200, null, "second", "body"),
|
||||
node(201, 200, "second", "h2"),
|
||||
node(210, 200, "second", "iframe", { title: "Nested" }),
|
||||
];
|
||||
const nestedNodes = [
|
||||
node(300, null, "nested", "body"),
|
||||
node(301, 300, "nested", "input", { placeholder: "Nested field" }),
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
viewport: { width: 1200, height: 800 },
|
||||
nodes: mainNodes,
|
||||
iframeNodes: new Map([
|
||||
[10, firstNodes],
|
||||
[20, secondNodes],
|
||||
[210, nestedNodes],
|
||||
]),
|
||||
frameNodes: new Map([
|
||||
["main", mainNodes],
|
||||
["first", firstNodes],
|
||||
["second", secondNodes],
|
||||
["nested", nestedNodes],
|
||||
]),
|
||||
frameOwnerBackendNodeIds: new Map([
|
||||
["first", 10],
|
||||
["second", 20],
|
||||
["nested", 210],
|
||||
]),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
const axNode = (
|
||||
nodeId: string,
|
||||
backendDOMNodeId: number,
|
||||
role: string,
|
||||
parentId?: string,
|
||||
name?: string,
|
||||
): CdpAxNode => ({
|
||||
nodeId,
|
||||
backendDOMNodeId,
|
||||
role: { type: "role", value: role },
|
||||
...(parentId ? { parentId } : {}),
|
||||
...(name ? { name: { type: "computedString", value: name } } : {}),
|
||||
});
|
||||
const result = buildFrameVomScene(
|
||||
[
|
||||
{
|
||||
frameId: "main",
|
||||
contextScopeId: "main",
|
||||
target: { tabId: 7 },
|
||||
domNodes: mainNodes,
|
||||
axNodes: [
|
||||
axNode("root", 1, "RootWebArea"),
|
||||
axNode("frame-1", 10, "Iframe", "root", "First"),
|
||||
axNode("frame-2", 20, "Iframe", "root", "Second"),
|
||||
],
|
||||
},
|
||||
{
|
||||
frameId: "first",
|
||||
contextScopeId: "first",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 7, sessionId: "session-first" },
|
||||
domNodes: firstNodes,
|
||||
axNodes: [
|
||||
axNode("root", 100, "RootWebArea"),
|
||||
axNode("button", 101, "button", "root", "First action"),
|
||||
],
|
||||
},
|
||||
{
|
||||
frameId: "second",
|
||||
contextScopeId: "second",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 7, sessionId: "session-second" },
|
||||
domNodes: secondNodes,
|
||||
axNodes: [
|
||||
axNode("root", 200, "RootWebArea"),
|
||||
axNode("heading", 201, "heading", "root", "Second heading"),
|
||||
axNode("nested-frame", 210, "Iframe", "root", "Nested"),
|
||||
],
|
||||
},
|
||||
{
|
||||
frameId: "nested",
|
||||
contextScopeId: "nested",
|
||||
parentFrameId: "second",
|
||||
ownerBackendNodeId: 210,
|
||||
target: { tabId: 7, sessionId: "session-nested" },
|
||||
domNodes: nestedNodes,
|
||||
axNodes: [
|
||||
axNode("root", 300, "RootWebArea"),
|
||||
axNode("input", 301, "textbox", "root", "Nested field"),
|
||||
],
|
||||
},
|
||||
],
|
||||
captured,
|
||||
);
|
||||
|
||||
const rendered = renderVom(result);
|
||||
expect(rendered.text).toContain('Iframe "First"');
|
||||
expect(rendered.text).toContain('@e1 button "First action"');
|
||||
expect(rendered.text).toContain('heading "Second heading"');
|
||||
expect(rendered.text).toContain('Iframe "Nested"');
|
||||
expect(rendered.text).toContain('textbox "Nested field"');
|
||||
expect(rendered.refs.find((ref) => ref.backendNodeId === 101)?.frameId).toBe("first");
|
||||
expect(rendered.refs.find((ref) => ref.backendNodeId === 301)?.frameId).toBe("nested");
|
||||
|
||||
const firstAction = result.nodes.find((item) => item.backendNodeId === 101);
|
||||
const nestedInput = result.nodes.find((item) => item.backendNodeId === 301);
|
||||
expect(firstAction).toMatchObject({ frameId: "first", contextScopeId: "first" });
|
||||
expect(nestedInput).toMatchObject({ frameId: "nested", contextScopeId: "nested" });
|
||||
expect(nestedInput?.parentId).toBe(result.nodes.find((item) => item.backendNodeId === 210)?.id);
|
||||
});
|
||||
|
||||
it("does not place a child document at the page root when its frame boundary is unresolved", () => {
|
||||
const childNode: CapturedNode = {
|
||||
backendNodeId: 101,
|
||||
parentBackendNodeId: null,
|
||||
frameId: "child",
|
||||
tag: "button",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 100, h: 30 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
};
|
||||
const captured: CapturedViewModel = {
|
||||
viewport: { width: 800, height: 600 },
|
||||
nodes: [],
|
||||
iframeNodes: new Map(),
|
||||
frameNodes: new Map([["child", [childNode]]]),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
|
||||
const result = buildFrameVomScene(
|
||||
[
|
||||
{
|
||||
frameId: "main",
|
||||
contextScopeId: "main",
|
||||
target: { tabId: 7 },
|
||||
domNodes: [],
|
||||
axNodes: [],
|
||||
},
|
||||
{
|
||||
frameId: "child",
|
||||
contextScopeId: "child",
|
||||
parentFrameId: "main",
|
||||
target: { tabId: 7 },
|
||||
domNodes: [childNode],
|
||||
axNodes: [
|
||||
{
|
||||
nodeId: "button",
|
||||
frameId: "child",
|
||||
backendDOMNodeId: 101,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Child action" },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
captured,
|
||||
);
|
||||
|
||||
expect(result.nodes).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps AX-only frame content attached to its iframe boundary", () => {
|
||||
const mainNodes: CapturedNode[] = [
|
||||
{
|
||||
backendNodeId: 1,
|
||||
parentBackendNodeId: null,
|
||||
frameId: "main",
|
||||
tag: "body",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 800, h: 600 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
{
|
||||
backendNodeId: 10,
|
||||
parentBackendNodeId: 1,
|
||||
frameId: "main",
|
||||
tag: "iframe",
|
||||
attrs: { title: "Remote" },
|
||||
rect: { x: 100, y: 100, w: 400, h: 300 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
},
|
||||
];
|
||||
const captured: CapturedViewModel = {
|
||||
viewport: { width: 800, height: 600 },
|
||||
nodes: mainNodes,
|
||||
iframeNodes: new Map(),
|
||||
frameNodes: new Map([
|
||||
["main", mainNodes],
|
||||
["child", []],
|
||||
]),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
|
||||
const result = buildFrameVomScene(
|
||||
[
|
||||
{
|
||||
frameId: "main",
|
||||
contextScopeId: "main",
|
||||
target: { tabId: 7 },
|
||||
domNodes: mainNodes,
|
||||
axNodes: [
|
||||
{
|
||||
nodeId: "main-root",
|
||||
backendDOMNodeId: 1,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
},
|
||||
{
|
||||
nodeId: "frame-owner",
|
||||
parentId: "main-root",
|
||||
backendDOMNodeId: 10,
|
||||
role: { type: "role", value: "Iframe" },
|
||||
name: { type: "computedString", value: "Remote" },
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
frameId: "child",
|
||||
contextScopeId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 7, sessionId: "child-session" },
|
||||
domNodes: [],
|
||||
axNodes: [
|
||||
{
|
||||
nodeId: "child-root",
|
||||
backendDOMNodeId: 100,
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
},
|
||||
{
|
||||
nodeId: "child-button",
|
||||
parentId: "child-root",
|
||||
backendDOMNodeId: 101,
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "AX fallback action" },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
captured,
|
||||
);
|
||||
|
||||
const childButton = result.nodes.find((node) => node.backendNodeId === 101);
|
||||
expect(childButton).toMatchObject({ frameId: "child", contextScopeId: "child" });
|
||||
expect(childButton?.parentId).toBe(result.nodes.find((node) => node.backendNodeId === 10)?.id);
|
||||
expect(renderVom(result).text).toContain('@e1 button "AX fallback action"');
|
||||
});
|
||||
|
||||
it("joins AX semantics with captured geometry by backendDOMNodeId", () => {
|
||||
const axNodes: CdpAxNode[] = [
|
||||
{
|
||||
@@ -658,6 +1107,7 @@ describe("buildVomScene", () => {
|
||||
parentId: "2",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "x", value: "输入密码" },
|
||||
value: { value: "iframe-secret" },
|
||||
backendDOMNodeId: 202,
|
||||
properties: [{ name: "inputType", value: { value: "password" } }],
|
||||
},
|
||||
@@ -712,6 +1162,10 @@ describe("buildVomScene", () => {
|
||||
|
||||
expect(passwordNodes).toHaveLength(1);
|
||||
expect(passwordNodes[0]).toEqual(expect.objectContaining({ id: 202, sensitive: true }));
|
||||
expect(passwordNodes[0].value).toBeUndefined();
|
||||
const rendered = renderVom(buildVomScene(axNodes, captured)).text;
|
||||
expect(rendered).toContain('textbox "输入密码" [filled] ="•••"');
|
||||
expect(rendered).not.toContain("iframe-secret");
|
||||
});
|
||||
|
||||
it("keeps unnamed iframe controls but skips unnamed iframe links", () => {
|
||||
@@ -876,7 +1330,7 @@ describe("buildVomScene", () => {
|
||||
expect(rendered).toContain(' @e2 textbox "验证码"');
|
||||
});
|
||||
|
||||
it("does not mark non-password inputs sensitive from password-like labels", () => {
|
||||
it("marks current-password autocomplete as sensitive even for text inputs", () => {
|
||||
const axNodes: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "1",
|
||||
@@ -911,7 +1365,7 @@ describe("buildVomScene", () => {
|
||||
expect(buildVomScene(axNodes, captured).nodes[0]).toEqual(
|
||||
expect.objectContaining({
|
||||
id: 10,
|
||||
sensitive: false,
|
||||
sensitive: true,
|
||||
}),
|
||||
);
|
||||
});
|
||||
@@ -1206,7 +1660,9 @@ describe("buildVomScene", () => {
|
||||
);
|
||||
const rendered = renderVom(scene);
|
||||
expect(rendered.text).toContain('@e1 button "close"');
|
||||
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
|
||||
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
|
||||
{ ref: "e1", backendNodeId: 20 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not promote a clickable container that wraps a real interactive control", () => {
|
||||
@@ -1350,7 +1806,9 @@ describe("buildVomScene", () => {
|
||||
expect(scene.nodes.find((n) => n.id === 30)?.role).toBe("generic");
|
||||
const rendered = renderVom(scene);
|
||||
expect(rendered.text).toContain('@e1 button "收藏"');
|
||||
expect(rendered.refs).toEqual([{ ref: "e1", backendNodeId: 20 }]);
|
||||
expect(rendered.refs.map(({ ref, backendNodeId }) => ({ ref, backendNodeId }))).toEqual([
|
||||
{ ref: "e1", backendNodeId: 20 },
|
||||
]);
|
||||
});
|
||||
|
||||
it("builds active scope blocks from active aria-controls relationships", () => {
|
||||
@@ -2299,6 +2757,64 @@ describe("handleSnapshot", () => {
|
||||
expect(ctx.refStore.resolve("e1")).toBeNull();
|
||||
});
|
||||
|
||||
it("keeps the previous RefStore when cancellation lands during DOM capture", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e1", 999, { tabId: 4 });
|
||||
const controller = new AbortController();
|
||||
let resolveCapture: (value: unknown) => void = () => {};
|
||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") {
|
||||
return {
|
||||
nodes: [
|
||||
{
|
||||
nodeId: "new",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "New" },
|
||||
backendDOMNodeId: 123,
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return new Promise((resolve) => {
|
||||
resolveCapture = resolve;
|
||||
});
|
||||
}
|
||||
throw new Error(`unexpected CDP method ${method}`);
|
||||
});
|
||||
const deps = {
|
||||
cdp: {
|
||||
send: send as unknown as <T = unknown>(
|
||||
tabId: number,
|
||||
method: string,
|
||||
params?: object,
|
||||
) => Promise<T>,
|
||||
trackSessionTab: vi.fn(),
|
||||
},
|
||||
tabsApi: {
|
||||
get: vi.fn(),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
};
|
||||
|
||||
const pending = handleSnapshot(sm, { session_id: "aa11" }, deps, controller.signal);
|
||||
await vi.waitFor(() =>
|
||||
expect(send).toHaveBeenCalledWith(4, "DOMSnapshot.captureSnapshot", expect.any(Object)),
|
||||
);
|
||||
controller.abort();
|
||||
resolveCapture({ strings: [], documents: [] });
|
||||
|
||||
await expect(pending).resolves.toMatchObject({ code: "cancelled" });
|
||||
expect(ctx.refStore.resolve("e1", { tabId: 4 })).toBe(999);
|
||||
expect(ctx.refStore.resolve("e2")).toBeNull();
|
||||
});
|
||||
|
||||
it("surfaces CDP failures as cdp_failed", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
@@ -2393,10 +2909,360 @@ describe("handleSnapshot", () => {
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function secretsSnapshotReply(secrets: {
|
||||
email: string;
|
||||
password: string;
|
||||
otp: string;
|
||||
card: string;
|
||||
}) {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
"input",
|
||||
"position",
|
||||
"static",
|
||||
"pointer-events",
|
||||
"auto",
|
||||
"type",
|
||||
"text",
|
||||
"password",
|
||||
"autocomplete",
|
||||
"one-time-code",
|
||||
"cc-number",
|
||||
"value",
|
||||
secrets.email,
|
||||
secrets.password,
|
||||
secrets.otp,
|
||||
secrets.card,
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
const style = [i("static"), i("auto"), i("auto")];
|
||||
return {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1, 1, 1, 1],
|
||||
nodeName: [i("html"), i("body"), i("input"), i("input"), i("input"), i("input")],
|
||||
backendNodeId: [10, 11, 12, 13, 14, 15],
|
||||
attributes: [
|
||||
[],
|
||||
[],
|
||||
[i("type"), i("text"), i("value"), i(secrets.email)],
|
||||
[i("type"), i("password"), i("value"), i(secrets.password)],
|
||||
[
|
||||
i("type"),
|
||||
i("text"),
|
||||
i("autocomplete"),
|
||||
i("one-time-code"),
|
||||
i("value"),
|
||||
i(secrets.otp),
|
||||
],
|
||||
[
|
||||
i("type"),
|
||||
i("text"),
|
||||
i("autocomplete"),
|
||||
i("cc-number"),
|
||||
i("value"),
|
||||
i(secrets.card),
|
||||
],
|
||||
],
|
||||
inputValue: {
|
||||
index: [2, 3, 4, 5],
|
||||
value: [i(secrets.email), i(secrets.password), i(secrets.otp), i(secrets.card)],
|
||||
},
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 3, 4, 5],
|
||||
styles: [style, style, style, style, style],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[10, 10, 200, 32],
|
||||
[10, 50, 200, 32],
|
||||
[10, 90, 200, 32],
|
||||
[10, 130, 200, 32],
|
||||
],
|
||||
paintOrders: [0, 1, 2, 3, 4],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
const VP_METRICS = {
|
||||
cssLayoutViewport: { clientWidth: 1000, clientHeight: 800, pageX: 0, pageY: 0 },
|
||||
};
|
||||
|
||||
function makeFrameAwareDeps() {
|
||||
const strings = ["html", "body", "iframe", "button", "title", "Remote", "static", "auto"];
|
||||
const i = (value: string) => strings.indexOf(value);
|
||||
const styles = [i("static"), i("auto"), i("auto")];
|
||||
const snapshot = {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
frameId: "main",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("iframe")],
|
||||
backendNodeId: [10, 11, 12],
|
||||
attributes: [[], [], [i("title"), i("Remote")]],
|
||||
contentDocumentIndex: { index: [2], value: [1] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1, 2],
|
||||
styles: [styles, styles, styles],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[0, 0, 1000, 800],
|
||||
[100, 100, 400, 300],
|
||||
],
|
||||
paintOrders: [0, 0, 1],
|
||||
},
|
||||
},
|
||||
{
|
||||
frameId: "child",
|
||||
nodes: {
|
||||
parentIndex: [-1, 0, 1],
|
||||
nodeName: [i("html"), i("body"), i("button")],
|
||||
backendNodeId: [20, 21, 22],
|
||||
attributes: [[], [], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1, 2],
|
||||
styles: [styles, styles, styles],
|
||||
bounds: [
|
||||
[0, 0, 400, 300],
|
||||
[0, 0, 400, 300],
|
||||
[20, 30, 120, 40],
|
||||
],
|
||||
paintOrders: [0, 0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
const mainAx: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "main-root",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 11,
|
||||
childIds: ["frame-owner"],
|
||||
},
|
||||
{
|
||||
nodeId: "frame-owner",
|
||||
parentId: "main-root",
|
||||
role: { type: "role", value: "Iframe" },
|
||||
name: { type: "computedString", value: "Remote" },
|
||||
backendDOMNodeId: 12,
|
||||
},
|
||||
];
|
||||
const childAx: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "child-root",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 21,
|
||||
childIds: ["child-button"],
|
||||
},
|
||||
{
|
||||
nodeId: "child-button",
|
||||
parentId: "child-root",
|
||||
role: { type: "role", value: "button" },
|
||||
name: { type: "computedString", value: "Frame action" },
|
||||
backendDOMNodeId: 22,
|
||||
},
|
||||
];
|
||||
const send = vi.fn(async (_tabId: number, method: string) => {
|
||||
if (method === "Page.getLayoutMetrics") return VP_METRICS;
|
||||
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") return snapshot;
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: mainAx };
|
||||
throw new Error(`unexpected root CDP method ${method}`);
|
||||
});
|
||||
const sendToTarget = vi.fn(async (_target, method: string) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: childAx };
|
||||
throw new Error(`unexpected child CDP method ${method}`);
|
||||
});
|
||||
const cdp = {
|
||||
send: send as unknown as CdpRunner["send"],
|
||||
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 12,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
trackSessionTab: vi.fn(),
|
||||
} satisfies CdpRunner;
|
||||
return {
|
||||
cdp,
|
||||
tabsApi: {
|
||||
get: vi.fn(
|
||||
async (tabId: number) => ({ id: tabId, windowId: 100, active: true }) as chrome.tabs.Tab,
|
||||
),
|
||||
query: vi.fn(async () => [{ id: 4, windowId: 100, active: true } as chrome.tabs.Tab]),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
it("exposes the frame-aware observation result for recording", async () => {
|
||||
const ax: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "root",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 11,
|
||||
childIds: ["password"],
|
||||
},
|
||||
{
|
||||
nodeId: "password",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Password" },
|
||||
value: { value: "hunter2" },
|
||||
backendDOMNodeId: 13,
|
||||
},
|
||||
];
|
||||
const deps = makeOverlayDeps(ax, loginSnapshotReply(), VP_METRICS);
|
||||
|
||||
const result = await captureVomObservation(deps.cdp, 4, "https://example.com", {
|
||||
redactValues: true,
|
||||
});
|
||||
|
||||
expect(result.text).not.toContain("hunter2");
|
||||
expect(result.text).toContain("•••");
|
||||
expect(result.refs[0]).toMatchObject({
|
||||
backendNodeId: 13,
|
||||
role: "textbox",
|
||||
name: "Password",
|
||||
line: expect.any(Number),
|
||||
});
|
||||
expect(result.rootFrameId).toBe("root");
|
||||
expect(result).not.toHaveProperty("frameDocuments");
|
||||
expect(result.frames).toEqual([{ frameId: "root", target: { tabId: 4 } }]);
|
||||
expect(result.matchNodes.find((node) => node.backendNodeId === 13)).toMatchObject({
|
||||
frameId: "root",
|
||||
backendNodeId: 13,
|
||||
tag: "input",
|
||||
rect: { x: 400, y: 300, w: 200, h: 40 },
|
||||
localRect: { x: 400, y: 300, w: 200, h: 40 },
|
||||
});
|
||||
});
|
||||
|
||||
it("does not leak form secrets through the record-safe observation payload", async () => {
|
||||
const secrets = {
|
||||
email: "user@example.com",
|
||||
password: "hunter2",
|
||||
otp: "847291",
|
||||
card: "4111111111111111",
|
||||
};
|
||||
const ax: CdpAxNode[] = [
|
||||
{
|
||||
nodeId: "root",
|
||||
role: { type: "role", value: "RootWebArea" },
|
||||
backendDOMNodeId: 11,
|
||||
childIds: ["email", "password", "otp", "card"],
|
||||
},
|
||||
{
|
||||
nodeId: "email",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Email" },
|
||||
value: { value: secrets.email },
|
||||
backendDOMNodeId: 12,
|
||||
},
|
||||
{
|
||||
nodeId: "password",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Password" },
|
||||
value: { value: secrets.password },
|
||||
backendDOMNodeId: 13,
|
||||
},
|
||||
{
|
||||
nodeId: "otp",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Code" },
|
||||
value: { value: secrets.otp },
|
||||
backendDOMNodeId: 14,
|
||||
},
|
||||
{
|
||||
nodeId: "card",
|
||||
parentId: "root",
|
||||
role: { type: "role", value: "textbox" },
|
||||
name: { type: "computedString", value: "Card" },
|
||||
value: { value: secrets.card },
|
||||
backendDOMNodeId: 15,
|
||||
},
|
||||
];
|
||||
const deps = makeOverlayDeps(ax, secretsSnapshotReply(secrets), VP_METRICS);
|
||||
|
||||
const result = await captureVomObservation(deps.cdp, 4, "https://example.com", {
|
||||
redactValues: true,
|
||||
});
|
||||
const dumped = JSON.stringify(result);
|
||||
|
||||
expect(result).not.toHaveProperty("frameDocuments");
|
||||
expect(dumped).not.toContain(secrets.email);
|
||||
expect(dumped).not.toContain(secrets.password);
|
||||
expect(dumped).not.toContain(secrets.otp);
|
||||
expect(dumped).not.toContain(secrets.card);
|
||||
expect(dumped).not.toContain("formValue");
|
||||
expect(dumped).not.toContain("formDefaultValue");
|
||||
expect(dumped).not.toContain("axNodes");
|
||||
expect(dumped).not.toContain("domNodes");
|
||||
expect(dumped).not.toContain("attrs");
|
||||
expect(result.text).toContain("•••");
|
||||
});
|
||||
|
||||
it("keeps frames and rendered refs on the same frame identity", async () => {
|
||||
const deps = makeFrameAwareDeps();
|
||||
|
||||
const result = await captureVomObservation(deps.cdp, 4, "https://example.com");
|
||||
|
||||
expect(result.rootFrameId).toBe("main");
|
||||
expect(result.frames).toHaveLength(2);
|
||||
expect(result.frames.find((frame) => frame.frameId === "child")).toEqual({
|
||||
frameId: "child",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 12,
|
||||
target: { tabId: 4, sessionId: "child-session" },
|
||||
});
|
||||
expect(result.matchNodes.find((node) => node.backendNodeId === 22)).toMatchObject({
|
||||
frameId: "child",
|
||||
tag: "button",
|
||||
localRect: { x: 20, y: 30, w: 120, h: 40 },
|
||||
});
|
||||
expect(result.refs.find((ref) => ref.backendNodeId === 22)).toMatchObject({
|
||||
frameId: "child",
|
||||
name: "Frame action",
|
||||
});
|
||||
});
|
||||
|
||||
it("stores iframe refs with their owning CDP session", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
const deps = makeFrameAwareDeps();
|
||||
|
||||
const result = await handleSnapshot(sm, { session_id: "aa11" }, deps);
|
||||
|
||||
if ("code" in result) throw new Error(`unexpected error: ${JSON.stringify(result)}`);
|
||||
expect(result.text).toContain('@e1 button "Frame action"');
|
||||
const frameRef = [...ctx.refStore.entries()].find(([, entry]) => entry.backendNodeId === 22);
|
||||
expect(frameRef?.[1]).toMatchObject({
|
||||
tabId: 4,
|
||||
frameId: "child",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
});
|
||||
|
||||
it("renders a blocking login overlay as the focused top layer", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
await sm.start("aa11");
|
||||
@@ -2907,6 +3773,32 @@ describe("handleGetHtml", () => {
|
||||
expect(deps.send).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("reads a frame ref through its child CDP session", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e7", 4242, {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
const deps = makeDeps({});
|
||||
const sendToTarget = vi.fn(async (target, method, params) => {
|
||||
expect(target).toEqual({ tabId: 4, sessionId: "child-session" });
|
||||
expect(method).toBe("DOM.getOuterHTML");
|
||||
expect(params).toEqual({ backendNodeId: 4242 });
|
||||
return { outerHTML: "<button>frame</button>" };
|
||||
});
|
||||
(deps.cdp as CdpRunner).sendToTarget = sendToTarget as unknown as NonNullable<
|
||||
CdpRunner["sendToTarget"]
|
||||
>;
|
||||
|
||||
const res = await handleGetHtml(sm, { session_id: "aa11", ref: "@e7" }, deps);
|
||||
|
||||
if ("code" in res) throw new Error(`unexpected error: ${JSON.stringify(res)}`);
|
||||
expect(res.html).toBe("<button>frame</button>");
|
||||
expect(deps.send).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("returns not_found when a ref belongs to another tab", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
|
||||
@@ -60,6 +60,7 @@ function makeApis(
|
||||
}),
|
||||
getLastFocused: vi.fn(async () => ({ id: 500 }) as chrome.windows.Window),
|
||||
create: vi.fn(async () => ({ id: 999 }) as chrome.windows.Window),
|
||||
remove: vi.fn(async () => {}),
|
||||
};
|
||||
return { tabs, windows };
|
||||
}
|
||||
|
||||
@@ -46,6 +46,25 @@ describe("lookupSnapshotRef", () => {
|
||||
});
|
||||
|
||||
describe("resolveSnapshotRef", () => {
|
||||
it("preserves the frame and child session needed to route iframe refs", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.refStore.set("e3", 1234, {
|
||||
tabId: 4,
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
});
|
||||
|
||||
const expected = {
|
||||
backendNodeId: 1234,
|
||||
refKey: "e3",
|
||||
frameId: "child-frame",
|
||||
cdpSessionId: "child-session",
|
||||
};
|
||||
expect(lookupSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
||||
expect(resolveSnapshotRef(ctx, "@e3", 4)).toEqual(expected);
|
||||
});
|
||||
|
||||
it("returns not_found for unknown ref", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
|
||||
@@ -179,6 +179,7 @@ function makeWindowsApi(
|
||||
get: ReturnType<typeof vi.fn>;
|
||||
lastFocused: ReturnType<typeof vi.fn>;
|
||||
create: ReturnType<typeof vi.fn>;
|
||||
remove: ReturnType<typeof vi.fn>;
|
||||
};
|
||||
} {
|
||||
const get = vi.fn(async (windowId: number) => {
|
||||
@@ -197,7 +198,13 @@ function makeWindowsApi(
|
||||
const id = opts?.createWindowId ?? 999;
|
||||
return { id } as chrome.windows.Window;
|
||||
});
|
||||
return { api: { get, getLastFocused: lastFocused, create }, spies: { get, lastFocused, create } };
|
||||
const remove = vi.fn(async (windowId: number) => {
|
||||
state.windowsClosed.add(windowId);
|
||||
});
|
||||
return {
|
||||
api: { get, getLastFocused: lastFocused, create, remove },
|
||||
spies: { get, lastFocused, create, remove },
|
||||
};
|
||||
}
|
||||
|
||||
describe("handleTabCreate", () => {
|
||||
@@ -601,6 +608,86 @@ describe("handleTabReturn", () => {
|
||||
expect(winSpies.create).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
it("closes a newly created fallback window when cancellation wins before the move", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
|
||||
const state: FakeTabState = {
|
||||
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
|
||||
nextTabId: 50,
|
||||
windowsClosed: new Set([200]),
|
||||
};
|
||||
const { api, spies } = makeTabMutationApi(state);
|
||||
const { api: windowsApi, spies: winSpies } = makeWindowsApi(state, {
|
||||
lastFocused: 100,
|
||||
createWindowId: 777,
|
||||
});
|
||||
let resolveCreate: (window: chrome.windows.Window) => void = () => {};
|
||||
winSpies.create.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
resolveCreate = resolve;
|
||||
}),
|
||||
);
|
||||
const controller = new AbortController();
|
||||
|
||||
const pending = handleTabReturn(
|
||||
sm,
|
||||
{ session_id: "aa11", tab_id: 7 },
|
||||
{ tabs: api, windows: windowsApi, signal: controller.signal },
|
||||
);
|
||||
await vi.waitFor(() => expect(winSpies.create).toHaveBeenCalledOnce());
|
||||
controller.abort();
|
||||
resolveCreate({ id: 777 } as chrome.windows.Window);
|
||||
|
||||
await expect(pending).resolves.toMatchObject({ code: "cancelled" });
|
||||
expect(winSpies.remove).toHaveBeenCalledWith(777);
|
||||
expect(spies.move).not.toHaveBeenCalled();
|
||||
expect(ctx.borrowedTabs.has(7)).toBe(true);
|
||||
});
|
||||
|
||||
it("surfaces the fallback window id when cancellation cleanup fails", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
ctx.borrowedTabs.set(7, { tabId: 7, originalWindowId: 200, originalIndex: 4 });
|
||||
const state: FakeTabState = {
|
||||
tabs: new Map([[7, { id: 7, windowId: 100 } as chrome.tabs.Tab]]),
|
||||
nextTabId: 50,
|
||||
windowsClosed: new Set([200]),
|
||||
};
|
||||
const { api, spies } = makeTabMutationApi(state);
|
||||
const { api: windowsApi, spies: winSpies } = makeWindowsApi(state, {
|
||||
lastFocused: 100,
|
||||
createWindowId: 777,
|
||||
});
|
||||
let resolveCreate: (window: chrome.windows.Window) => void = () => {};
|
||||
winSpies.create.mockImplementationOnce(
|
||||
() =>
|
||||
new Promise((resolve) => {
|
||||
resolveCreate = resolve;
|
||||
}),
|
||||
);
|
||||
winSpies.remove.mockRejectedValueOnce(new Error("window removal denied"));
|
||||
const controller = new AbortController();
|
||||
|
||||
const pending = handleTabReturn(
|
||||
sm,
|
||||
{ session_id: "aa11", tab_id: 7 },
|
||||
{ tabs: api, windows: windowsApi, signal: controller.signal },
|
||||
);
|
||||
await vi.waitFor(() => expect(winSpies.create).toHaveBeenCalledOnce());
|
||||
controller.abort();
|
||||
resolveCreate({ id: 777 } as chrome.windows.Window);
|
||||
|
||||
await expect(pending).resolves.toMatchObject({
|
||||
code: "protocol_error",
|
||||
data: { reason: "cleanup_failed", resource_type: "window", resource_id: 777 },
|
||||
message: expect.stringMatching(/fallback window 777.*window removal denied/),
|
||||
});
|
||||
expect(spies.move).not.toHaveBeenCalled();
|
||||
expect(ctx.borrowedTabs.has(7)).toBe(true);
|
||||
});
|
||||
|
||||
it("falls back to a new window when getLastFocused only returns the Agent Window", async () => {
|
||||
const sm = new SessionManager({ agentWindow: fakeAgentWindow([100]) });
|
||||
const ctx = await sm.start("aa11");
|
||||
|
||||
@@ -107,7 +107,7 @@ describe("handleSessionStart window size", () => {
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const result = await handleSessionStart(sm, { session_id: "aa11", width: 1280, height: 800 });
|
||||
expect(result).toEqual({ agent_window_id: 100 });
|
||||
expect(aw.create).toHaveBeenCalledWith("about:blank", { width: 1280, height: 800 });
|
||||
expect(aw.create).toHaveBeenCalledWith("about:blank", { size: { width: 1280, height: 800 } });
|
||||
});
|
||||
|
||||
it("creates the window without size when width/height are omitted", async () => {
|
||||
@@ -115,7 +115,15 @@ describe("handleSessionStart window size", () => {
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const result = await handleSessionStart(sm, { session_id: "aa11" });
|
||||
expect(result).toEqual({ agent_window_id: 100 });
|
||||
expect(aw.create).toHaveBeenCalledWith("about:blank");
|
||||
expect(aw.create).toHaveBeenCalledWith("about:blank", {});
|
||||
});
|
||||
|
||||
it("forwards focused: false to the Agent Window", async () => {
|
||||
const aw = fakeAgentWindow([100]);
|
||||
const sm = new SessionManager({ agentWindow: aw });
|
||||
const result = await handleSessionStart(sm, { session_id: "aa11", focused: false });
|
||||
expect(result).toEqual({ agent_window_id: 100 });
|
||||
expect(aw.create).toHaveBeenCalledWith("about:blank", { focused: false });
|
||||
});
|
||||
|
||||
it("rejects a lone width without height", async () => {
|
||||
|
||||
@@ -250,9 +250,10 @@ const DEFAULT_NOTIFICATION_COPY: BorrowNotificationCopy = {
|
||||
// URL injection-eligibility test
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const CHROME_WEB_STORE_RES = [
|
||||
const EXTENSION_STORE_RES = [
|
||||
/^https:\/\/chrome\.google\.com\/webstore/i,
|
||||
/^https:\/\/chromewebstore\.google\.com/i,
|
||||
/^https:\/\/microsoftedge\.microsoft\.com\/addons/i,
|
||||
];
|
||||
|
||||
/**
|
||||
@@ -264,14 +265,15 @@ const CHROME_WEB_STORE_RES = [
|
||||
* - `ftp://` no longer hosts content scripts reliably in modern Chrome.
|
||||
* - `about:` / `chrome:` / `chrome-extension:` / `edge:` / `devtools:` /
|
||||
* `view-source:` / `data:` / `blob:` are blocked by the platform.
|
||||
* - The Chrome Web Store (both legacy and new domains) is also blocked.
|
||||
* - Extension storefronts are blocked by their own browser: the Chrome Web
|
||||
* Store (legacy and new domains) under Chrome, and Edge Add-ons under Edge.
|
||||
*
|
||||
* Mirrors the implicit scheme list documented in
|
||||
* https://developer.chrome.com/docs/extensions/develop/concepts/match-patterns.
|
||||
*/
|
||||
export function isInjectableContentScriptUrl(url: string | undefined): boolean {
|
||||
if (!url) return false;
|
||||
for (const re of CHROME_WEB_STORE_RES) {
|
||||
for (const re of EXTENSION_STORE_RES) {
|
||||
if (re.test(url)) return false;
|
||||
}
|
||||
return /^https?:\/\//i.test(url);
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
export {
|
||||
type CaptureVomObservationOptions,
|
||||
captureVomObservation,
|
||||
} from "./observation";
|
||||
export {
|
||||
type CaptureVomFrame,
|
||||
type CaptureVomMatchNode,
|
||||
type CaptureVomObservationResult,
|
||||
} from "./vom/record-safe-observation";
|
||||
@@ -27,19 +27,23 @@ export async function handleConsole(
|
||||
manager: SessionManager,
|
||||
params: ConsoleParams,
|
||||
deps: ConsoleDeps = defaultConsoleDeps(),
|
||||
signal?: AbortSignal,
|
||||
): Promise<ConsoleResult | RpcError> {
|
||||
if (signal?.aborted) return { code: "cancelled", message: "console aborted" };
|
||||
const ctxOrErr = lookupSession(manager, params, "console");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const bounds = parseBufferedReadBounds(params);
|
||||
if (isRpcError(bounds)) return bounds;
|
||||
const target = await resolveTargetTab(manager, ctxOrErr, params.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
if (signal?.aborted) return { code: "cancelled", message: "console aborted" };
|
||||
if (!deps.cdp.ensureConsoleCapture || !deps.cdp.consoleEntriesSince) {
|
||||
return { code: "cdp_failed", message: "console capture requires CDP console support" };
|
||||
}
|
||||
|
||||
try {
|
||||
await deps.cdp.ensureConsoleCapture(target.tabId);
|
||||
if (signal?.aborted) return { code: "cancelled", message: "console aborted" };
|
||||
deps.cdp.trackSessionTab?.(ctxOrErr.sessionId, target.tabId);
|
||||
return deps.cdp.consoleEntriesSince(
|
||||
target.tabId,
|
||||
|
||||
@@ -125,11 +125,10 @@ export interface DispatcherDeps {
|
||||
* `AbortController` keyed by its wire `id` in
|
||||
* [`inflightAbortControllers`]. When the daemon pushes a `cancel`
|
||||
* request the dispatcher trips the matching controller; tool
|
||||
* handlers that already accept a `signal` (waits, navigation,
|
||||
* interaction, evaluate, tabs) react in line, and the dispatcher
|
||||
* additionally races the in-flight invocation against the abort
|
||||
* promise so handlers without explicit signal plumbing still respond
|
||||
* promptly with `cancelled`.
|
||||
* handlers observe that signal between awaited operations. The
|
||||
* original RPC remains pending until its handler has stopped or
|
||||
* completed compensation; only the separate cancel acknowledgement
|
||||
* takes the fast path.
|
||||
*/
|
||||
export class ToolDispatcher {
|
||||
private readonly transport: Transport;
|
||||
@@ -221,7 +220,7 @@ export class ToolDispatcher {
|
||||
try {
|
||||
const sessionId = sessionIdForBrowserControlMethod(req);
|
||||
if (sessionId) this.onBrowserControlResumed?.(sessionId);
|
||||
const result = await Promise.race([this.invoke(req, ac.signal), abortPromise(ac.signal)]);
|
||||
const result = await this.invoke(req, ac.signal);
|
||||
if (isRpcError(result)) {
|
||||
body = { id: req.id, error: result };
|
||||
} else {
|
||||
@@ -286,7 +285,7 @@ export class ToolDispatcher {
|
||||
private async invoke(req: RequestFrame, signal: AbortSignal): Promise<unknown | RpcError> {
|
||||
switch (req.method) {
|
||||
case "tool.session_start":
|
||||
return handleSessionStart(this.sessions, req.params as SessionStartParams);
|
||||
return handleSessionStart(this.sessions, req.params as SessionStartParams, { signal });
|
||||
case "tool.session_stop": {
|
||||
await this.releaseHoverLatch((req.params as SessionStopParams).session_id);
|
||||
return handleSessionStop(this.sessions, req.params as SessionStopParams, {
|
||||
@@ -294,29 +293,36 @@ export class ToolDispatcher {
|
||||
});
|
||||
}
|
||||
case "tool.tab_list":
|
||||
return handleTabList(this.sessions, req.params as TabListParams);
|
||||
return handleTabList(this.sessions, req.params as TabListParams, chromeTabsApi, signal);
|
||||
case "tool.tab_create":
|
||||
return handleTabCreate(this.sessions, req.params as TabCreateParams);
|
||||
return handleTabCreate(this.sessions, req.params as TabCreateParams, { signal });
|
||||
case "tool.tab_close":
|
||||
return this.withHoverReleaseForRequest(req.params as TabCloseParams, () =>
|
||||
handleTabClose(this.sessions, req.params as TabCloseParams),
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as TabCloseParams,
|
||||
() => handleTabClose(this.sessions, req.params as TabCloseParams, { signal }),
|
||||
signal,
|
||||
);
|
||||
case "tool.tab_select":
|
||||
return handleTabSelect(this.sessions, req.params as TabSelectParams);
|
||||
return handleTabSelect(this.sessions, req.params as TabSelectParams, { signal });
|
||||
case "tool.tab_borrow":
|
||||
return handleTabBorrow(this.sessions, req.params as TabBorrowParams, {
|
||||
signal,
|
||||
approveBorrow: this.approveBorrow,
|
||||
});
|
||||
case "tool.tab_return":
|
||||
return handleTabReturn(this.sessions, req.params as TabReturnParams);
|
||||
return handleTabReturn(this.sessions, req.params as TabReturnParams, { signal });
|
||||
case "tool.window_resize":
|
||||
return handleWindowResize(this.sessions, req.params as WindowResizeParams);
|
||||
return handleWindowResize(
|
||||
this.sessions,
|
||||
req.params as WindowResizeParams,
|
||||
undefined,
|
||||
signal,
|
||||
);
|
||||
case "tool.emulate":
|
||||
return handleEmulate(
|
||||
this.sessions,
|
||||
req.params as EmulateParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi } : undefined,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
);
|
||||
case "tool.screenshot":
|
||||
return handleScreenshot(
|
||||
@@ -325,43 +331,57 @@ export class ToolDispatcher {
|
||||
this.cdp
|
||||
? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi, captureApi: chromeTabsCaptureApi }
|
||||
: undefined,
|
||||
signal,
|
||||
);
|
||||
case "tool.console":
|
||||
return handleConsole(
|
||||
this.sessions,
|
||||
req.params as ConsoleParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi } : undefined,
|
||||
signal,
|
||||
);
|
||||
case "tool.network":
|
||||
return handleNetwork(
|
||||
this.sessions,
|
||||
req.params as NetworkParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi } : undefined,
|
||||
signal,
|
||||
);
|
||||
case "tool.snapshot":
|
||||
return this.withHoverReassert(req.params as SnapshotParams, () =>
|
||||
handleSnapshot(
|
||||
this.sessions,
|
||||
req.params as SnapshotParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||
),
|
||||
return this.withHoverReassert(
|
||||
req.params as SnapshotParams,
|
||||
() =>
|
||||
handleSnapshot(
|
||||
this.sessions,
|
||||
req.params as SnapshotParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||
signal,
|
||||
),
|
||||
{},
|
||||
signal,
|
||||
);
|
||||
case "tool.observe": {
|
||||
const params = req.params as ObserveParams;
|
||||
const hoverScope = await this.resolveHoverLatchScope(params);
|
||||
return this.withHoverReassert(params, () =>
|
||||
handleObserve(
|
||||
this.sessions,
|
||||
params,
|
||||
this.cdp
|
||||
? {
|
||||
cdp: this.cdp,
|
||||
tabsApi: chromeTabsCaptureApi,
|
||||
conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope),
|
||||
hoverProbeBypassOverlay: bypassOverlay,
|
||||
}
|
||||
: undefined,
|
||||
),
|
||||
throwIfDispatchAborted(signal);
|
||||
return this.withHoverReassert(
|
||||
params,
|
||||
() =>
|
||||
handleObserve(
|
||||
this.sessions,
|
||||
params,
|
||||
this.cdp
|
||||
? {
|
||||
cdp: this.cdp,
|
||||
tabsApi: chromeTabsCaptureApi,
|
||||
conditionalSurfaceProbe: !this.hasHoverLatchForScope(hoverScope),
|
||||
hoverProbeBypassOverlay: bypassOverlay,
|
||||
}
|
||||
: undefined,
|
||||
signal,
|
||||
),
|
||||
{},
|
||||
signal,
|
||||
);
|
||||
}
|
||||
case "tool.get_html":
|
||||
@@ -369,38 +389,51 @@ export class ToolDispatcher {
|
||||
this.sessions,
|
||||
req.params as GetHtmlParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsCaptureApi } : undefined,
|
||||
signal,
|
||||
);
|
||||
case "tool.navigate":
|
||||
return this.withHoverReleaseForRequest(req.params as NavigateParams, () =>
|
||||
handleNavigate(
|
||||
this.sessions,
|
||||
req.params as NavigateParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as NavigateParams,
|
||||
() =>
|
||||
handleNavigate(
|
||||
this.sessions,
|
||||
req.params as NavigateParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
signal,
|
||||
);
|
||||
case "tool.navigate_back":
|
||||
return this.withHoverReleaseForRequest(req.params as NavigateBackParams, () =>
|
||||
handleNavigateBack(
|
||||
this.sessions,
|
||||
req.params as NavigateBackParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as NavigateBackParams,
|
||||
() =>
|
||||
handleNavigateBack(
|
||||
this.sessions,
|
||||
req.params as NavigateBackParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
signal,
|
||||
);
|
||||
case "tool.navigate_forward":
|
||||
return this.withHoverReleaseForRequest(req.params as NavigateForwardParams, () =>
|
||||
handleNavigateForward(
|
||||
this.sessions,
|
||||
req.params as NavigateForwardParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as NavigateForwardParams,
|
||||
() =>
|
||||
handleNavigateForward(
|
||||
this.sessions,
|
||||
req.params as NavigateForwardParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
signal,
|
||||
);
|
||||
case "tool.reload":
|
||||
return this.withHoverReleaseForRequest(req.params as ReloadParams, () =>
|
||||
handleReload(
|
||||
this.sessions,
|
||||
req.params as ReloadParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as ReloadParams,
|
||||
() =>
|
||||
handleReload(
|
||||
this.sessions,
|
||||
req.params as ReloadParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
signal,
|
||||
);
|
||||
case "tool.click":
|
||||
return this.withHoverReassert(
|
||||
@@ -419,6 +452,7 @@ export class ToolDispatcher {
|
||||
: undefined,
|
||||
),
|
||||
{ releaseAfter: true },
|
||||
signal,
|
||||
);
|
||||
case "tool.hover": {
|
||||
const result = await handleHover(
|
||||
@@ -438,28 +472,37 @@ export class ToolDispatcher {
|
||||
return this.rememberHover((req.params as HoverParams).session_id, result);
|
||||
}
|
||||
case "tool.fill":
|
||||
return this.withHoverReleaseForRequest(req.params as FillParams, () =>
|
||||
handleFill(
|
||||
this.sessions,
|
||||
req.params as FillParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as FillParams,
|
||||
() =>
|
||||
handleFill(
|
||||
this.sessions,
|
||||
req.params as FillParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
signal,
|
||||
);
|
||||
case "tool.press":
|
||||
return this.withHoverReleaseForRequest(req.params as PressParams, () =>
|
||||
handlePress(
|
||||
this.sessions,
|
||||
req.params as PressParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as PressParams,
|
||||
() =>
|
||||
handlePress(
|
||||
this.sessions,
|
||||
req.params as PressParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
signal,
|
||||
);
|
||||
case "tool.select":
|
||||
return this.withHoverReleaseForRequest(req.params as SelectParams, () =>
|
||||
handleSelect(
|
||||
this.sessions,
|
||||
req.params as SelectParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
return this.withHoverReleaseForRequest(
|
||||
req.params as SelectParams,
|
||||
() =>
|
||||
handleSelect(
|
||||
this.sessions,
|
||||
req.params as SelectParams,
|
||||
this.cdp ? { cdp: this.cdp, tabsApi: chromeTabsApi, signal } : undefined,
|
||||
),
|
||||
signal,
|
||||
);
|
||||
case "tool.evaluate":
|
||||
return handleEvaluate(
|
||||
@@ -566,9 +609,13 @@ export class ToolDispatcher {
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
options: { releaseAfter?: boolean } = {},
|
||||
signal?: AbortSignal,
|
||||
): Promise<T> {
|
||||
throwIfDispatchAborted(signal);
|
||||
const scope = await this.resolveHoverLatchScope(params);
|
||||
throwIfDispatchAborted(signal);
|
||||
await this.reassertHover(scope);
|
||||
throwIfDispatchAborted(signal);
|
||||
try {
|
||||
return await work();
|
||||
} finally {
|
||||
@@ -581,9 +628,13 @@ export class ToolDispatcher {
|
||||
private async withHoverReleaseForRequest<T>(
|
||||
params: { session_id: string; tab_id?: number },
|
||||
work: () => Promise<T>,
|
||||
signal?: AbortSignal,
|
||||
): Promise<T> {
|
||||
throwIfDispatchAborted(signal);
|
||||
const scope = await this.resolveHoverLatchScope(params);
|
||||
throwIfDispatchAborted(signal);
|
||||
await this.releaseHoverLatch(scope.session_id, scope.tab_id);
|
||||
throwIfDispatchAborted(signal);
|
||||
return work();
|
||||
}
|
||||
|
||||
@@ -693,41 +744,14 @@ async function bypassOverlay(tabId: number, enabled: boolean): Promise<void> {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves never; rejects with `AbortLikeError` as soon as the signal
|
||||
* fires (or immediately if it is already aborted). Used by the
|
||||
* dispatcher to race the tool invocation so handlers without explicit
|
||||
* signal plumbing still surface a `cancelled` reply promptly.
|
||||
*/
|
||||
function abortPromise(signal: AbortSignal): Promise<never> {
|
||||
return new Promise<never>((_, reject) => {
|
||||
if (signal.aborted) {
|
||||
reject(new AbortLikeError());
|
||||
return;
|
||||
}
|
||||
signal.addEventListener(
|
||||
"abort",
|
||||
() => {
|
||||
reject(new AbortLikeError());
|
||||
},
|
||||
{ once: true },
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Sentinel error class so [`isAbortLikeError`] can recognise our own
|
||||
* race-rejection without confusing it with a real CDP failure.
|
||||
*/
|
||||
class AbortLikeError extends Error {
|
||||
constructor() {
|
||||
super("rpc aborted by daemon cancel");
|
||||
this.name = "BhAbortError";
|
||||
}
|
||||
function throwIfDispatchAborted(signal: AbortSignal | undefined): void {
|
||||
if (!signal?.aborted) return;
|
||||
const error = new Error("rpc aborted by daemon cancel");
|
||||
error.name = "AbortError";
|
||||
throw error;
|
||||
}
|
||||
|
||||
function isAbortLikeError(err: unknown): boolean {
|
||||
if (err instanceof AbortLikeError) return true;
|
||||
if (err instanceof DOMException && err.name === "AbortError") return true;
|
||||
if (typeof err === "object" && err !== null && (err as { name?: string }).name === "AbortError") {
|
||||
return true;
|
||||
|
||||
@@ -1,9 +1,15 @@
|
||||
// Shared CDP helpers for scrolling nodes into view and computing
|
||||
// viewport-space bounding rectangles. Used by interaction tools (click
|
||||
// centre) and observation tools (element screenshot clip).
|
||||
// Resolve and scroll nodes inside one CDP target. Cross-frame projection
|
||||
// belongs to frame-geometry.ts.
|
||||
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { rpcError } from "./errors";
|
||||
import {
|
||||
parseCdpQuad,
|
||||
polygonBounds,
|
||||
type Region,
|
||||
rectPolygon,
|
||||
type ViewportRect,
|
||||
} from "./geometry";
|
||||
import { type CdpRunner, isRpcError } from "./shared";
|
||||
|
||||
const ELEMENT_NOT_VISIBLE_MESSAGE =
|
||||
@@ -13,45 +19,6 @@ function elementNotVisibleError(): RpcError {
|
||||
return rpcError("permission_denied", "element_not_visible", ELEMENT_NOT_VISIBLE_MESSAGE);
|
||||
}
|
||||
|
||||
/** Viewport-space axis-aligned bounding box for CDP `Page.captureScreenshot` clip. */
|
||||
export interface ViewportRect {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the axis-aligned bounding box of an 8-double CDP content
|
||||
* quad / box-model polygon. Exported for unit tests.
|
||||
*/
|
||||
export function quadBoundingRect(quad: number[]): ViewportRect | null {
|
||||
if (quad.length !== 8) return null;
|
||||
const xs = [quad[0], quad[2], quad[4], quad[6]];
|
||||
const ys = [quad[1], quad[3], quad[5], quad[7]];
|
||||
const minX = Math.min(...xs);
|
||||
const maxX = Math.max(...xs);
|
||||
const minY = Math.min(...ys);
|
||||
const maxY = Math.max(...ys);
|
||||
const width = maxX - minX;
|
||||
const height = maxY - minY;
|
||||
if (width <= 0 || height <= 0) return null;
|
||||
return { x: minX, y: minY, width, height };
|
||||
}
|
||||
|
||||
function rectToQuad(rect: ViewportRect): number[] {
|
||||
return [
|
||||
rect.x,
|
||||
rect.y,
|
||||
rect.x + rect.width,
|
||||
rect.y,
|
||||
rect.x + rect.width,
|
||||
rect.y + rect.height,
|
||||
rect.x,
|
||||
rect.y + rect.height,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve `backendNodeId` → CDP `objectId` so we can invoke
|
||||
* `Runtime.callFunctionOn` against the live JS object.
|
||||
@@ -112,48 +79,24 @@ export async function scrollNodeIntoView(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Viewport-space bounding box for `backendNodeId`. Prefers
|
||||
* `DOM.getContentQuads`, then `DOM.getBoxModel`, then the union of
|
||||
* visible descendant client rects for zero-size overflow containers.
|
||||
*/
|
||||
export async function nodeBoundingRect(
|
||||
function validCdpRegion(quads: number[][] | undefined): Region {
|
||||
return (quads ?? []).flatMap((raw) => {
|
||||
const polygon = parseCdpQuad(raw);
|
||||
return polygon && polygonBounds(polygon) ? [polygon] : [];
|
||||
});
|
||||
}
|
||||
|
||||
async function resolveVisibleContentRegion(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
backendNodeId: number,
|
||||
): Promise<ViewportRect | RpcError> {
|
||||
const polygon = await visibleContentPolygon(cdp, tabId, backendNodeId);
|
||||
if (isRpcError(polygon)) return polygon;
|
||||
// visibleContentPolygon only returns quads with positive area.
|
||||
return quadBoundingRect(polygon)!;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the centroid of a CDP content quad. Quads are reported as
|
||||
* 8 doubles in clockwise order: (x1, y1, x2, y2, x3, y3, x4, y4).
|
||||
*/
|
||||
export function quadCentre(quad: number[]): { x: number; y: number } {
|
||||
const x = (quad[0] + quad[4]) / 2;
|
||||
const y = (quad[1] + quad[5]) / 2;
|
||||
return { x, y };
|
||||
}
|
||||
|
||||
/** Centre of a `DOM.getBoxModel` `content` polygon (also 8 doubles). */
|
||||
export function boxCentre(box: number[]): { x: number; y: number } {
|
||||
return quadCentre(box);
|
||||
}
|
||||
|
||||
async function visibleContentPolygon(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
backendNodeId: number,
|
||||
): Promise<number[] | RpcError> {
|
||||
): Promise<Region | RpcError> {
|
||||
try {
|
||||
const quads = await cdp.send<{ quads?: number[][] }>(tabId, "DOM.getContentQuads", {
|
||||
backendNodeId,
|
||||
});
|
||||
const quad = quads.quads?.find((q) => q.length === 8 && quadBoundingRect(q) !== null);
|
||||
if (quad) return quad;
|
||||
const visible = validCdpRegion(quads.quads);
|
||||
if (visible.length > 0) return visible;
|
||||
} catch (err) {
|
||||
console.debug("[bsk element-geometry] getContentQuads failed", err);
|
||||
}
|
||||
@@ -162,15 +105,34 @@ async function visibleContentPolygon(
|
||||
backendNodeId,
|
||||
});
|
||||
const content = box.model?.content;
|
||||
if (content && content.length === 8 && quadBoundingRect(content)) return content;
|
||||
const fallback = validCdpRegion(content ? [content] : undefined);
|
||||
if (fallback.length > 0) return fallback;
|
||||
} catch (err) {
|
||||
console.debug("[bsk element-geometry] getBoxModel failed", err);
|
||||
}
|
||||
const descendantRect = await descendantBoundingRect(cdp, tabId, backendNodeId);
|
||||
if (descendantRect) return rectToQuad(descendantRect);
|
||||
if (descendantRect) {
|
||||
return [
|
||||
rectPolygon({
|
||||
x: descendantRect.x,
|
||||
y: descendantRect.y,
|
||||
w: descendantRect.width,
|
||||
h: descendantRect.height,
|
||||
}),
|
||||
];
|
||||
}
|
||||
return elementNotVisibleError();
|
||||
}
|
||||
|
||||
/** Resolve every visible content region in the node's target-local viewport. */
|
||||
export async function nodeContentRegion(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
backendNodeId: number,
|
||||
): Promise<Region | RpcError> {
|
||||
return resolveVisibleContentRegion(cdp, tabId, backendNodeId);
|
||||
}
|
||||
|
||||
async function descendantBoundingRect(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
@@ -241,20 +203,3 @@ function parseViewportRect(value: unknown): ViewportRect | null {
|
||||
if (![x, y, width, height].every(Number.isFinite) || width <= 0 || height <= 0) return null;
|
||||
return { x, y, width, height };
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the click point for `backendNodeId`. Prefers
|
||||
* `DOM.getContentQuads` (richer for rotated / transformed elements);
|
||||
* falls back to `DOM.getBoxModel`, then visible descendant bounds.
|
||||
* Returns `permission_denied` with "element not visible" when every
|
||||
* geometry path comes back empty / fails.
|
||||
*/
|
||||
export async function nodeCentre(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
backendNodeId: number,
|
||||
): Promise<{ x: number; y: number } | RpcError> {
|
||||
const polygon = await visibleContentPolygon(cdp, tabId, backendNodeId);
|
||||
if (isRpcError(polygon)) return polygon;
|
||||
return quadCentre(polygon);
|
||||
}
|
||||
|
||||
@@ -44,6 +44,7 @@ export interface EmulateCdpRunner {
|
||||
export interface EmulateDeps {
|
||||
cdp: EmulateCdpRunner;
|
||||
tabsApi: ChromeTabsApi;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
function defaultEmulateDeps(): EmulateDeps {
|
||||
@@ -210,11 +211,13 @@ export async function handleEmulate(
|
||||
params: EmulateParams,
|
||||
deps: EmulateDeps = defaultEmulateDeps(),
|
||||
): Promise<EmulateResult | RpcError> {
|
||||
if (deps.signal?.aborted) return { code: "cancelled", message: "emulate aborted" };
|
||||
const ctxOrErr = lookupSession(manager, params, "emulate");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
const target = await resolveTargetTab(manager, ctx, params?.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
if (deps.signal?.aborted) return { code: "cancelled", message: "emulate aborted" };
|
||||
const denied = enforceAgentWindow(ctx, target, "emulate");
|
||||
if (denied) return denied;
|
||||
|
||||
@@ -223,9 +226,16 @@ export async function handleEmulate(
|
||||
return invalidParams("emulate off cannot be combined with overrides");
|
||||
}
|
||||
try {
|
||||
if (deps.signal?.aborted) return { code: "cancelled", message: "emulate aborted" };
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await deps.cdp.clearDeviceMetricsOverride(target.tabId);
|
||||
if (deps.signal?.aborted) {
|
||||
return { code: "cancelled", message: "emulate aborted after clearing device metrics" };
|
||||
}
|
||||
await deps.cdp.setTouchEmulationEnabled(target.tabId, false);
|
||||
if (deps.signal?.aborted) {
|
||||
return { code: "cancelled", message: "emulate aborted after clearing touch emulation" };
|
||||
}
|
||||
// An empty userAgent string clears the override (CDP convention).
|
||||
await deps.cdp.setUserAgentOverride(target.tabId, { userAgent: "" });
|
||||
} catch (err) {
|
||||
@@ -246,6 +256,7 @@ export async function handleEmulate(
|
||||
// values; the merged state is what gets applied (and echoed back).
|
||||
const merged = mergeEmulateOverrides(tabEmulationStates.get(target.tabId), overrides);
|
||||
try {
|
||||
if (deps.signal?.aborted) return { code: "cancelled", message: "emulate aborted" };
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
if (merged.width !== undefined && merged.height !== undefined) {
|
||||
await deps.cdp.setDeviceMetricsOverride(target.tabId, {
|
||||
@@ -254,6 +265,9 @@ export async function handleEmulate(
|
||||
deviceScaleFactor: merged.device_scale_factor ?? 0,
|
||||
mobile: merged.mobile ?? false,
|
||||
});
|
||||
if (deps.signal?.aborted) {
|
||||
return { code: "cancelled", message: "emulate aborted after applying device metrics" };
|
||||
}
|
||||
}
|
||||
if (merged.user_agent !== undefined) {
|
||||
await deps.cdp.setUserAgentOverride(target.tabId, {
|
||||
@@ -263,6 +277,9 @@ export async function handleEmulate(
|
||||
? { userAgentMetadata: toCdpUserAgentMetadata(merged.user_agent_metadata) }
|
||||
: {}),
|
||||
});
|
||||
if (deps.signal?.aborted) {
|
||||
return { code: "cancelled", message: "emulate aborted after applying user agent" };
|
||||
}
|
||||
}
|
||||
if (merged.touch !== undefined || merged.max_touch_points !== undefined) {
|
||||
await deps.cdp.setTouchEmulationEnabled(
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
import { type CdpFrame, type CdpFrameGraph, type CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { nodeContentRegion, scrollNodeIntoView } from "./element-geometry";
|
||||
import {
|
||||
clipPolygon,
|
||||
type GeometryProjection,
|
||||
type Point,
|
||||
type Polygon,
|
||||
type ProjectiveEdge,
|
||||
parseCdpQuad,
|
||||
polygonArea,
|
||||
polygonCentroid,
|
||||
projectRegionToViewport,
|
||||
type Quad,
|
||||
type Region,
|
||||
rectPolygon,
|
||||
regionBounds,
|
||||
type Size,
|
||||
type ViewportRect,
|
||||
} from "./geometry";
|
||||
import { type CdpRunner, cdpRunnerForTarget, isRpcError, sendToCdpTarget } from "./shared";
|
||||
|
||||
export interface NodeAddress {
|
||||
target: CdpTarget;
|
||||
backendNodeId: number;
|
||||
frameId?: string;
|
||||
}
|
||||
|
||||
export interface ResolvedNodeGeometry {
|
||||
topVisibleRegions: Region;
|
||||
topBounds: ViewportRect;
|
||||
actionPoint: Point;
|
||||
}
|
||||
|
||||
function geometryError(message: string): RpcError {
|
||||
return { code: "cdp_failed", message };
|
||||
}
|
||||
|
||||
function frameMap(graph: CdpFrameGraph): Map<string, CdpFrame> {
|
||||
return new Map(graph.frames.map((frame) => [frame.frameId, frame]));
|
||||
}
|
||||
|
||||
function targetRootFrame(byId: Map<string, CdpFrame>, frame: CdpFrame): CdpFrame | null {
|
||||
const seen = new Set<string>();
|
||||
let current = frame;
|
||||
while (current.parentFrameId) {
|
||||
if (seen.has(current.frameId)) return null;
|
||||
seen.add(current.frameId);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent || parent.target.sessionId !== current.target.sessionId) break;
|
||||
current = parent;
|
||||
}
|
||||
return current;
|
||||
}
|
||||
|
||||
function frameAncestry(graph: CdpFrameGraph, frameId: string): CdpFrame[] | null {
|
||||
const byId = frameMap(graph);
|
||||
const path: CdpFrame[] = [];
|
||||
const seen = new Set<string>();
|
||||
let current = byId.get(frameId);
|
||||
if (!current) return null;
|
||||
while (current.parentFrameId) {
|
||||
if (seen.has(current.frameId)) return null;
|
||||
seen.add(current.frameId);
|
||||
path.push(current);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent) return null;
|
||||
current = parent;
|
||||
}
|
||||
return path;
|
||||
}
|
||||
|
||||
async function targetViewport(cdp: CdpRunner, target: CdpTarget): Promise<Size | null> {
|
||||
const metrics = await sendToCdpTarget<{
|
||||
cssLayoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
layoutViewport?: { clientWidth?: number; clientHeight?: number };
|
||||
}>(cdp, target, "Page.getLayoutMetrics", {});
|
||||
const viewport = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
const width = viewport.clientWidth ?? 0;
|
||||
const height = viewport.clientHeight ?? 0;
|
||||
return width > 0 && height > 0 ? { width, height } : null;
|
||||
}
|
||||
|
||||
async function ownerContentQuad(
|
||||
cdp: CdpRunner,
|
||||
parent: CdpFrame,
|
||||
ownerBackendNodeId: number,
|
||||
): Promise<Quad | null> {
|
||||
const result = await sendToCdpTarget<{ model?: { content?: number[] } }>(
|
||||
cdp,
|
||||
parent.target,
|
||||
"DOM.getBoxModel",
|
||||
{ backendNodeId: ownerBackendNodeId },
|
||||
);
|
||||
return parseCdpQuad(result.model?.content);
|
||||
}
|
||||
|
||||
async function sameTargetFrameClips(
|
||||
cdp: CdpRunner,
|
||||
byId: Map<string, CdpFrame>,
|
||||
frame: CdpFrame,
|
||||
targetRoot: CdpFrame,
|
||||
): Promise<Polygon[] | null> {
|
||||
const clips: Polygon[] = [];
|
||||
let current = frame;
|
||||
const seen = new Set<string>();
|
||||
while (current.frameId !== targetRoot.frameId) {
|
||||
if (seen.has(current.frameId) || !current.parentFrameId) return null;
|
||||
seen.add(current.frameId);
|
||||
const parent = byId.get(current.parentFrameId);
|
||||
if (!parent || current.ownerBackendNodeId === undefined) return null;
|
||||
const clip = await ownerContentQuad(cdp, parent, current.ownerBackendNodeId);
|
||||
if (!clip) return null;
|
||||
clips.push(clip);
|
||||
current = parent;
|
||||
}
|
||||
return clips;
|
||||
}
|
||||
|
||||
export async function resolveFrameProjection(
|
||||
cdp: CdpRunner,
|
||||
graph: CdpFrameGraph,
|
||||
frameId: string,
|
||||
): Promise<GeometryProjection | null> {
|
||||
const byId = frameMap(graph);
|
||||
const frame = byId.get(frameId);
|
||||
if (!frame) return null;
|
||||
let targetRoot = targetRootFrame(byId, frame);
|
||||
if (!targetRoot) return null;
|
||||
const sourceViewport = await targetViewport(cdp, targetRoot.target);
|
||||
if (!sourceViewport) return null;
|
||||
const sourceClips = await sameTargetFrameClips(cdp, byId, frame, targetRoot);
|
||||
if (!sourceClips) return null;
|
||||
|
||||
const edges: ProjectiveEdge[] = [];
|
||||
while (targetRoot.parentFrameId) {
|
||||
const parent = byId.get(targetRoot.parentFrameId);
|
||||
if (!parent || targetRoot.ownerBackendNodeId === undefined) return null;
|
||||
const destinationQuad = await ownerContentQuad(cdp, parent, targetRoot.ownerBackendNodeId);
|
||||
if (!destinationQuad) return null;
|
||||
const source =
|
||||
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
||||
if (!source) return null;
|
||||
const parentTargetRoot = targetRootFrame(byId, parent);
|
||||
if (!parentTargetRoot) return null;
|
||||
const destinationClips = await sameTargetFrameClips(cdp, byId, parent, parentTargetRoot);
|
||||
if (!destinationClips) return null;
|
||||
edges.push({ sourceViewport: source, destinationQuad, destinationClips });
|
||||
targetRoot = parentTargetRoot;
|
||||
}
|
||||
|
||||
const topViewport =
|
||||
edges.length === 0 ? sourceViewport : await targetViewport(cdp, targetRoot.target);
|
||||
return topViewport ? { sourceClips, edges, topViewport } : null;
|
||||
}
|
||||
|
||||
async function loadFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
|
||||
if (!cdp.getFrameGraph) return null;
|
||||
try {
|
||||
return await cdp.getFrameGraph(tabId);
|
||||
} catch (error) {
|
||||
console.debug("[bsk frame-geometry] frame graph resolution failed", error);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function scrollFrameOwners(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
graph: CdpFrameGraph,
|
||||
frameId: string,
|
||||
): Promise<RpcError | null> {
|
||||
const byId = frameMap(graph);
|
||||
const ancestry = frameAncestry(graph, frameId);
|
||||
if (!ancestry) return geometryError(`could not resolve frame ancestry for ${frameId}`);
|
||||
for (const child of [...ancestry].reverse()) {
|
||||
const parent = child.parentFrameId ? byId.get(child.parentFrameId) : undefined;
|
||||
if (!parent || child.ownerBackendNodeId === undefined) {
|
||||
return geometryError(`could not resolve frame owner for ${child.frameId}`);
|
||||
}
|
||||
const error = await scrollNodeIntoView(
|
||||
cdpRunnerForTarget(cdp, parent.target),
|
||||
tabId,
|
||||
child.ownerBackendNodeId,
|
||||
);
|
||||
if (error) return error;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function scrollElementWithFrameGraph(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
target: CdpTarget,
|
||||
backendNodeId: number,
|
||||
frameId: string | undefined,
|
||||
graph: CdpFrameGraph | null,
|
||||
): Promise<RpcError | null> {
|
||||
if (frameId) {
|
||||
if (!graph) return geometryError(`could not resolve frame graph for ${frameId}`);
|
||||
const error = await scrollFrameOwners(cdp, tabId, graph, frameId);
|
||||
if (error) return error;
|
||||
} else if (target.sessionId) {
|
||||
return geometryError("an OOPIF node address requires frameId");
|
||||
}
|
||||
return scrollNodeIntoView(cdpRunnerForTarget(cdp, target), tabId, backendNodeId);
|
||||
}
|
||||
|
||||
export async function scrollElementAndFramesIntoView(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
target: CdpTarget,
|
||||
backendNodeId: number,
|
||||
frameId?: string,
|
||||
): Promise<RpcError | null> {
|
||||
const graph = frameId ? await loadFrameGraph(cdp, tabId) : null;
|
||||
return scrollElementWithFrameGraph(cdp, tabId, target, backendNodeId, frameId, graph);
|
||||
}
|
||||
|
||||
function largestRegion(regions: Region): Polygon | null {
|
||||
let largest: { polygon: Polygon; area: number } | null = null;
|
||||
for (const polygon of regions) {
|
||||
const area = polygonArea(polygon);
|
||||
if (area <= 0) continue;
|
||||
if (!largest || area > largest.area) largest = { polygon, area };
|
||||
}
|
||||
return largest?.polygon ?? null;
|
||||
}
|
||||
|
||||
export async function resolveNodeGeometry(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
address: NodeAddress,
|
||||
options: { scrollIntoView?: boolean } = {},
|
||||
): Promise<ResolvedNodeGeometry | RpcError> {
|
||||
try {
|
||||
if (address.target.sessionId && !address.frameId) {
|
||||
return geometryError("an OOPIF node address requires frameId");
|
||||
}
|
||||
const graph = address.frameId ? await loadFrameGraph(cdp, tabId) : null;
|
||||
if (address.frameId && !graph) {
|
||||
return geometryError(`could not resolve frame graph for ${address.frameId}`);
|
||||
}
|
||||
|
||||
if (options.scrollIntoView) {
|
||||
const scrollError = await scrollElementWithFrameGraph(
|
||||
cdp,
|
||||
tabId,
|
||||
address.target,
|
||||
address.backendNodeId,
|
||||
address.frameId,
|
||||
graph,
|
||||
);
|
||||
if (scrollError) return scrollError;
|
||||
}
|
||||
|
||||
const localRegion = await nodeContentRegion(
|
||||
cdpRunnerForTarget(cdp, address.target),
|
||||
tabId,
|
||||
address.backendNodeId,
|
||||
);
|
||||
if (isRpcError(localRegion)) return localRegion;
|
||||
|
||||
let topVisibleRegions: Region;
|
||||
if (address.frameId && graph) {
|
||||
const projection = await resolveFrameProjection(cdp, graph, address.frameId);
|
||||
if (!projection)
|
||||
return geometryError(`could not resolve frame geometry for ${address.frameId}`);
|
||||
topVisibleRegions = projectRegionToViewport(localRegion, projection);
|
||||
} else {
|
||||
const viewport = await targetViewport(cdp, address.target);
|
||||
if (!viewport) return geometryError("could not resolve top viewport geometry");
|
||||
topVisibleRegions = localRegion
|
||||
.map((polygon) =>
|
||||
clipPolygon(polygon, rectPolygon({ x: 0, y: 0, w: viewport.width, h: viewport.height })),
|
||||
)
|
||||
.filter((polygon) => polygon.length >= 3);
|
||||
}
|
||||
|
||||
const topBounds = regionBounds(topVisibleRegions);
|
||||
const actionRegion = largestRegion(topVisibleRegions);
|
||||
const actionPoint = actionRegion ? polygonCentroid(actionRegion) : null;
|
||||
if (!topBounds || !actionPoint) {
|
||||
return { code: "permission_denied", message: "element not visible" };
|
||||
}
|
||||
return { topVisibleRegions, topBounds, actionPoint };
|
||||
} catch (error) {
|
||||
return geometryError(error instanceof Error ? error.message : String(error));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,266 @@
|
||||
export interface Point {
|
||||
x: number;
|
||||
y: number;
|
||||
}
|
||||
|
||||
export type Polygon = Point[];
|
||||
export type Region = Polygon[];
|
||||
export type Quad = [Point, Point, Point, Point];
|
||||
|
||||
export interface Size {
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface ViewportRect {
|
||||
x: number;
|
||||
y: number;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
export interface ProjectiveEdge {
|
||||
sourceViewport: Size;
|
||||
destinationQuad: Quad;
|
||||
destinationClips?: Polygon[];
|
||||
}
|
||||
|
||||
export interface GeometryProjection {
|
||||
sourceClips: Polygon[];
|
||||
edges: ProjectiveEdge[];
|
||||
topViewport: Size;
|
||||
}
|
||||
|
||||
export function parseCdpQuad(raw: number[] | undefined): Quad | null {
|
||||
if (!raw || raw.length !== 8 || !raw.every(Number.isFinite)) return null;
|
||||
return [
|
||||
{ x: raw[0], y: raw[1] },
|
||||
{ x: raw[2], y: raw[3] },
|
||||
{ x: raw[4], y: raw[5] },
|
||||
{ x: raw[6], y: raw[7] },
|
||||
];
|
||||
}
|
||||
|
||||
export function rectPolygon(rect: { x: number; y: number; w: number; h: number }): Quad {
|
||||
return [
|
||||
{ x: rect.x, y: rect.y },
|
||||
{ x: rect.x + rect.w, y: rect.y },
|
||||
{ x: rect.x + rect.w, y: rect.y + rect.h },
|
||||
{ x: rect.x, y: rect.y + rect.h },
|
||||
];
|
||||
}
|
||||
|
||||
export function viewportPolygon(viewport: Size): Quad {
|
||||
return rectPolygon({ x: 0, y: 0, w: viewport.width, h: viewport.height });
|
||||
}
|
||||
|
||||
/** Project a normalized unit-square point into an arbitrary content quad. */
|
||||
export function projectUnitPoint(point: Point, quad: Quad): Point {
|
||||
const [p0, p1, p2, p3] = quad;
|
||||
const dx1 = p1.x - p2.x;
|
||||
const dx2 = p3.x - p2.x;
|
||||
const dx3 = p0.x - p1.x + p2.x - p3.x;
|
||||
const dy1 = p1.y - p2.y;
|
||||
const dy2 = p3.y - p2.y;
|
||||
const dy3 = p0.y - p1.y + p2.y - p3.y;
|
||||
|
||||
let a: number;
|
||||
let b: number;
|
||||
let c: number;
|
||||
let d: number;
|
||||
let e: number;
|
||||
let f: number;
|
||||
let g = 0;
|
||||
let h = 0;
|
||||
|
||||
if (Math.abs(dx3) < 1e-9 && Math.abs(dy3) < 1e-9) {
|
||||
a = p1.x - p0.x;
|
||||
b = p3.x - p0.x;
|
||||
c = p0.x;
|
||||
d = p1.y - p0.y;
|
||||
e = p3.y - p0.y;
|
||||
f = p0.y;
|
||||
} else {
|
||||
const denominator = dx1 * dy2 - dx2 * dy1;
|
||||
if (Math.abs(denominator) < 1e-9) {
|
||||
return {
|
||||
x: p0.x + (p1.x - p0.x) * point.x + (p3.x - p0.x) * point.y,
|
||||
y: p0.y + (p1.y - p0.y) * point.x + (p3.y - p0.y) * point.y,
|
||||
};
|
||||
}
|
||||
g = (dx3 * dy2 - dx2 * dy3) / denominator;
|
||||
h = (dx1 * dy3 - dx3 * dy1) / denominator;
|
||||
a = p1.x - p0.x + g * p1.x;
|
||||
b = p3.x - p0.x + h * p3.x;
|
||||
c = p0.x;
|
||||
d = p1.y - p0.y + g * p1.y;
|
||||
e = p3.y - p0.y + h * p3.y;
|
||||
f = p0.y;
|
||||
}
|
||||
|
||||
const scale = g * point.x + h * point.y + 1;
|
||||
return {
|
||||
x: (a * point.x + b * point.y + c) / scale,
|
||||
y: (d * point.x + e * point.y + f) / scale,
|
||||
};
|
||||
}
|
||||
|
||||
export function projectPolygon(polygon: Polygon, edge: ProjectiveEdge): Polygon {
|
||||
const { width, height } = edge.sourceViewport;
|
||||
if (width <= 0 || height <= 0) return [];
|
||||
return polygon.map((point) =>
|
||||
projectUnitPoint({ x: point.x / width, y: point.y / height }, edge.destinationQuad),
|
||||
);
|
||||
}
|
||||
|
||||
function polygonSignedArea(points: Polygon): number {
|
||||
let area = 0;
|
||||
for (let index = 0; index < points.length; index += 1) {
|
||||
const current = points[index];
|
||||
const next = points[(index + 1) % points.length];
|
||||
area += current.x * next.y - next.x * current.y;
|
||||
}
|
||||
return area / 2;
|
||||
}
|
||||
|
||||
function edgeIntersection(start: Point, end: Point, clipStart: Point, clipEnd: Point): Point {
|
||||
const subjectX = end.x - start.x;
|
||||
const subjectY = end.y - start.y;
|
||||
const clipX = clipEnd.x - clipStart.x;
|
||||
const clipY = clipEnd.y - clipStart.y;
|
||||
const denominator = subjectX * clipY - subjectY * clipX;
|
||||
if (Math.abs(denominator) < 1e-9) return end;
|
||||
const offsetX = clipStart.x - start.x;
|
||||
const offsetY = clipStart.y - start.y;
|
||||
const scale = (offsetX * clipY - offsetY * clipX) / denominator;
|
||||
return { x: start.x + scale * subjectX, y: start.y + scale * subjectY };
|
||||
}
|
||||
|
||||
/** Clip a polygon to a convex polygon, preserving every resulting vertex. */
|
||||
export function clipPolygon(subject: Polygon, clip: Polygon): Polygon {
|
||||
if (subject.length < 3 || clip.length < 3) return [];
|
||||
const orientation = polygonSignedArea(clip) >= 0 ? 1 : -1;
|
||||
let output = [...subject];
|
||||
for (let edge = 0; edge < clip.length && output.length > 0; edge += 1) {
|
||||
const clipStart = clip[edge];
|
||||
const clipEnd = clip[(edge + 1) % clip.length];
|
||||
const input = output;
|
||||
output = [];
|
||||
const inside = (point: Point) =>
|
||||
orientation *
|
||||
((clipEnd.x - clipStart.x) * (point.y - clipStart.y) -
|
||||
(clipEnd.y - clipStart.y) * (point.x - clipStart.x)) >=
|
||||
-1e-9;
|
||||
let start = input[input.length - 1];
|
||||
for (const end of input) {
|
||||
const startInside = inside(start);
|
||||
const endInside = inside(end);
|
||||
if (endInside) {
|
||||
if (!startInside) output.push(edgeIntersection(start, end, clipStart, clipEnd));
|
||||
output.push(end);
|
||||
} else if (startInside) {
|
||||
output.push(edgeIntersection(start, end, clipStart, clipEnd));
|
||||
}
|
||||
start = end;
|
||||
}
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
export function polygonBounds(points: Polygon): ViewportRect | null {
|
||||
if (points.length < 3) return null;
|
||||
const xs = points.map((point) => point.x);
|
||||
const ys = points.map((point) => point.y);
|
||||
const x = Math.min(...xs);
|
||||
const y = Math.min(...ys);
|
||||
const right = Math.max(...xs);
|
||||
const bottom = Math.max(...ys);
|
||||
if (right <= x || bottom <= y) return null;
|
||||
return { x, y, width: right - x, height: bottom - y };
|
||||
}
|
||||
|
||||
export function regionBounds(region: Region): ViewportRect | null {
|
||||
return polygonBounds(region.flat());
|
||||
}
|
||||
|
||||
export function polygonArea(points: Polygon): number {
|
||||
return Math.abs(polygonSignedArea(points));
|
||||
}
|
||||
|
||||
export function polygonCentroid(points: Polygon): Point | null {
|
||||
if (points.length < 3) return null;
|
||||
let twiceArea = 0;
|
||||
let x = 0;
|
||||
let y = 0;
|
||||
for (let index = 0; index < points.length; index += 1) {
|
||||
const current = points[index];
|
||||
const next = points[(index + 1) % points.length];
|
||||
const cross = current.x * next.y - next.x * current.y;
|
||||
twiceArea += cross;
|
||||
x += (current.x + next.x) * cross;
|
||||
y += (current.y + next.y) * cross;
|
||||
}
|
||||
if (Math.abs(twiceArea) < 1e-9) return null;
|
||||
return { x: x / (3 * twiceArea), y: y / (3 * twiceArea) };
|
||||
}
|
||||
|
||||
export function projectAndClipRegion(
|
||||
region: Region,
|
||||
edges: ProjectiveEdge[],
|
||||
topViewport: Size,
|
||||
): Region {
|
||||
let projected = region;
|
||||
for (const edge of edges) {
|
||||
const sourceClip = viewportPolygon(edge.sourceViewport);
|
||||
projected = projected
|
||||
.map((polygon) => clipPolygon(polygon, sourceClip))
|
||||
.filter((polygon) => polygon.length >= 3)
|
||||
.map((polygon) =>
|
||||
[edge.destinationQuad, ...(edge.destinationClips ?? [])].reduce(
|
||||
(current, clip) => clipPolygon(current, clip),
|
||||
projectPolygon(polygon, edge),
|
||||
),
|
||||
)
|
||||
.filter((polygon) => polygon.length >= 3);
|
||||
}
|
||||
const topClip = viewportPolygon(topViewport);
|
||||
return projected
|
||||
.map((polygon) => clipPolygon(polygon, topClip))
|
||||
.filter((polygon) => polygon.length >= 3);
|
||||
}
|
||||
|
||||
export function projectRegionToViewport(region: Region, projection: GeometryProjection): Region {
|
||||
const clipped = region
|
||||
.map((polygon) =>
|
||||
projection.sourceClips.reduce((current, clip) => clipPolygon(current, clip), polygon),
|
||||
)
|
||||
.filter((polygon) => polygon.length >= 3);
|
||||
return projectAndClipRegion(clipped, projection.edges, projection.topViewport);
|
||||
}
|
||||
|
||||
export function projectRectToViewport(
|
||||
rect: { x: number; y: number; w: number; h: number } | null,
|
||||
projection: GeometryProjection,
|
||||
): ViewportRect | null {
|
||||
if (!rect) return null;
|
||||
return regionBounds(projectRegionToViewport([rectPolygon(rect)], projection));
|
||||
}
|
||||
|
||||
export function childFrameProjection(
|
||||
parent: GeometryProjection,
|
||||
ownerRectInParent: { x: number; y: number; w: number; h: number },
|
||||
): GeometryProjection {
|
||||
const destinationQuad = rectPolygon(ownerRectInParent);
|
||||
return {
|
||||
sourceClips: [],
|
||||
edges: [
|
||||
{
|
||||
sourceViewport: { width: ownerRectInParent.w, height: ownerRectInParent.h },
|
||||
destinationQuad,
|
||||
destinationClips: parent.sourceClips,
|
||||
},
|
||||
...parent.edges,
|
||||
],
|
||||
topViewport: parent.topViewport,
|
||||
};
|
||||
}
|
||||
@@ -8,6 +8,7 @@ import {
|
||||
HELP_REQUEST,
|
||||
type HelpCancelMessage,
|
||||
type HelpFinishMessage,
|
||||
type HelpHighlightRect,
|
||||
type HelpQueryResponse,
|
||||
type HelpRequestMessage,
|
||||
isHelpAckMessage,
|
||||
@@ -25,6 +26,7 @@ import type {
|
||||
ResolvedTarget,
|
||||
RpcError,
|
||||
} from "@/transport/types";
|
||||
import { resolveNodeGeometry } from "./frame-geometry";
|
||||
import {
|
||||
type CdpRunner,
|
||||
type ChromeTabsApi,
|
||||
@@ -37,7 +39,6 @@ import {
|
||||
import { lookupSnapshotRef } from "./snapshot-ref";
|
||||
|
||||
const DEFAULT_HELP_TIMEOUT_MS = 300_000;
|
||||
const HELP_ATTR = "data-bsk-help";
|
||||
const HELP_SEND_RETRIES = 3;
|
||||
const HELP_SEND_RETRY_DELAY_MS = 350;
|
||||
const HELP_REARM_DEBOUNCE_MS = 150;
|
||||
@@ -74,6 +75,7 @@ interface ActiveHelpRequest {
|
||||
title?: string;
|
||||
targets: HelpTarget[];
|
||||
selectors: string[];
|
||||
rects: HelpHighlightRect[];
|
||||
timeoutMs: number;
|
||||
notificationId: string;
|
||||
resolvedTargets?: ResolvedTarget[];
|
||||
@@ -134,6 +136,7 @@ function helpRequestMessage(help: ActiveHelpRequest, tabId: number): HelpRequest
|
||||
...(help.title ? { title: help.title } : {}),
|
||||
displayMode: isPrimaryTab ? "full" : "compact",
|
||||
selectors: isPrimaryTab ? help.selectors : [],
|
||||
rects: isPrimaryTab ? help.rects : [],
|
||||
timeoutMs: help.timeoutMs,
|
||||
};
|
||||
}
|
||||
@@ -165,33 +168,6 @@ async function sendHelpRequestWithAck(
|
||||
throw lastError ?? new Error("failed to show help overlay");
|
||||
}
|
||||
|
||||
async function tagRefTarget(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
backendNodeId: number,
|
||||
index: number,
|
||||
): Promise<string | null> {
|
||||
let objectId: string | undefined;
|
||||
try {
|
||||
const resolved = await cdp.send<{ object?: { objectId?: string } }>(tabId, "DOM.resolveNode", {
|
||||
backendNodeId,
|
||||
});
|
||||
objectId = resolved.object?.objectId;
|
||||
if (!objectId) return null;
|
||||
await cdp.send(tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function(){ this.setAttribute("${HELP_ATTR}", "${index}"); }`,
|
||||
});
|
||||
return `[${HELP_ATTR}="${index}"]`;
|
||||
} catch {
|
||||
return null;
|
||||
} finally {
|
||||
if (objectId) {
|
||||
await cdp.send(tabId, "Runtime.releaseObject", { objectId }).catch(() => {});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function selectorExists(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
@@ -209,36 +185,22 @@ async function selectorExists(
|
||||
}
|
||||
}
|
||||
|
||||
async function clearRefTags(cdp: CdpRunner | undefined, tabId: number): Promise<void> {
|
||||
if (!cdp) return;
|
||||
try {
|
||||
const doc = await cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
||||
depth: 0,
|
||||
});
|
||||
const rootId = doc.root?.nodeId;
|
||||
if (rootId === undefined) return;
|
||||
const { nodeIds } = await cdp.send<{ nodeIds: number[] }>(tabId, "DOM.querySelectorAll", {
|
||||
nodeId: rootId,
|
||||
selector: `[${HELP_ATTR}]`,
|
||||
});
|
||||
for (const nodeId of nodeIds ?? []) {
|
||||
await cdp.send(tabId, "DOM.removeAttribute", { nodeId, name: HELP_ATTR }).catch(() => {});
|
||||
}
|
||||
} catch {
|
||||
// Best-effort cleanup.
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveHelpTargets(
|
||||
ctx: ReturnType<SessionManager["get"]>,
|
||||
tabId: number,
|
||||
targets: HelpTarget[],
|
||||
deps: RequestHelpDeps,
|
||||
): Promise<{ selectors: string[]; resolvedTargets?: ResolvedTarget[] }> {
|
||||
options: { scrollIntoView: boolean },
|
||||
): Promise<{
|
||||
selectors: string[];
|
||||
rects: HelpHighlightRect[];
|
||||
resolvedTargets?: ResolvedTarget[];
|
||||
}> {
|
||||
const selectors: string[] = [];
|
||||
const rects: HelpHighlightRect[] = [];
|
||||
const resolved: ResolvedTarget[] = [];
|
||||
let rootNodeId: number | undefined;
|
||||
if (deps.cdp && targets.length > 0) {
|
||||
if (deps.cdp && targets.some((target) => Boolean(target.selector))) {
|
||||
try {
|
||||
const doc = await deps.cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
||||
depth: 0,
|
||||
@@ -262,28 +224,55 @@ async function resolveHelpTargets(
|
||||
resolved.push({ matched, selector: tgt.selector });
|
||||
} else if (tgt.ref) {
|
||||
const looked = ctx ? lookupSnapshotRef(ctx, tgt.ref, tabId) : null;
|
||||
const backendNodeId = looked?.backendNodeId ?? null;
|
||||
let sel: string | null = null;
|
||||
if (backendNodeId !== null && deps.cdp) {
|
||||
sel = await tagRefTarget(deps.cdp, tabId, backendNodeId, i);
|
||||
let rect: HelpHighlightRect | null = null;
|
||||
if (looked && deps.cdp) {
|
||||
const target = {
|
||||
tabId,
|
||||
...(looked.cdpSessionId ? { sessionId: looked.cdpSessionId } : {}),
|
||||
};
|
||||
const geometry = await resolveNodeGeometry(
|
||||
deps.cdp,
|
||||
tabId,
|
||||
{
|
||||
target,
|
||||
backendNodeId: looked.backendNodeId,
|
||||
...(looked.frameId ? { frameId: looked.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: options.scrollIntoView },
|
||||
);
|
||||
if (!isRpcError(geometry)) {
|
||||
rect = {
|
||||
top: geometry.topBounds.y,
|
||||
left: geometry.topBounds.x,
|
||||
width: geometry.topBounds.width,
|
||||
height: geometry.topBounds.height,
|
||||
};
|
||||
}
|
||||
}
|
||||
if (sel) selectors.push(sel);
|
||||
resolved.push({ matched: sel !== null, ref: tgt.ref });
|
||||
if (rect) rects.push(rect);
|
||||
resolved.push({
|
||||
matched: rect !== null,
|
||||
ref: tgt.ref,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return { selectors, resolvedTargets: resolved.length > 0 ? resolved : undefined };
|
||||
return { selectors, rects, resolvedTargets: resolved.length > 0 ? resolved : undefined };
|
||||
}
|
||||
|
||||
async function refreshHelpTargets(help: ActiveHelpRequest): Promise<void> {
|
||||
await clearRefTags(help.deps.cdp, help.primaryTabId);
|
||||
const { selectors, resolvedTargets } = await resolveHelpTargets(
|
||||
async function refreshHelpTargets(
|
||||
help: ActiveHelpRequest,
|
||||
options: { scrollIntoView: boolean } = { scrollIntoView: true },
|
||||
): Promise<void> {
|
||||
const { selectors, rects, resolvedTargets } = await resolveHelpTargets(
|
||||
help.ctx,
|
||||
help.primaryTabId,
|
||||
help.targets,
|
||||
help.deps,
|
||||
options,
|
||||
);
|
||||
help.selectors = selectors;
|
||||
help.rects = rects;
|
||||
help.resolvedTargets = resolvedTargets;
|
||||
}
|
||||
|
||||
@@ -294,12 +283,7 @@ async function cleanupHelp(help: ActiveHelpRequest): Promise<void> {
|
||||
const tabsToCancel = new Set([help.primaryTabId, ...help.overlayTabIds]);
|
||||
await Promise.all(
|
||||
[...tabsToCancel].map((tabId) =>
|
||||
Promise.all([
|
||||
clearRefTags(help.deps.cdp, tabId),
|
||||
help.deps
|
||||
.sendToTab(tabId, { type: HELP_CANCEL, requestId: help.requestId })
|
||||
.catch(() => {}),
|
||||
]),
|
||||
help.deps.sendToTab(tabId, { type: HELP_CANCEL, requestId: help.requestId }).catch(() => {}),
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -476,7 +460,9 @@ function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void {
|
||||
sendResponse({ active: false });
|
||||
return;
|
||||
}
|
||||
if (tabId === help.primaryTabId) await refreshHelpTargets(help);
|
||||
if (tabId === help.primaryTabId) {
|
||||
await refreshHelpTargets(help, { scrollIntoView: false });
|
||||
}
|
||||
help.overlayTabIds.add(tabId);
|
||||
sendResponse({
|
||||
active: true,
|
||||
@@ -486,6 +472,7 @@ function attachHelpRuntimeListener(deps: RequestHelpDeps): () => void {
|
||||
...(help.title ? { title: help.title } : {}),
|
||||
displayMode: tabId === help.primaryTabId ? "full" : "compact",
|
||||
selectors: tabId === help.primaryTabId ? help.selectors : [],
|
||||
rects: tabId === help.primaryTabId ? help.rects : [],
|
||||
timeoutMs: help.timeoutMs,
|
||||
},
|
||||
});
|
||||
@@ -686,7 +673,13 @@ export async function handleRequestHelp(
|
||||
if (deps.autoAttachLifecycle !== false) ensureHelpLifecycleListeners(deps);
|
||||
const tabId = target.tabId;
|
||||
const initialTargets = params.targets ?? [];
|
||||
const { selectors, resolvedTargets } = await resolveHelpTargets(ctx, tabId, initialTargets, deps);
|
||||
const { selectors, rects, resolvedTargets } = await resolveHelpTargets(
|
||||
ctx,
|
||||
tabId,
|
||||
initialTargets,
|
||||
deps,
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
|
||||
await deps.windows.update(target.windowId, { focused: true }).catch(() => {});
|
||||
await deps.activateTab(tabId).catch(() => {});
|
||||
@@ -729,6 +722,7 @@ export async function handleRequestHelp(
|
||||
...(params.title ? { title: params.title } : {}),
|
||||
targets: initialTargets,
|
||||
selectors,
|
||||
rects,
|
||||
timeoutMs,
|
||||
notificationId,
|
||||
resolvedTargets,
|
||||
|
||||
@@ -3,14 +3,15 @@
|
||||
//
|
||||
// All interaction tools:
|
||||
// 1. Resolve target tab (sandbox: must be inside Agent Window).
|
||||
// 2. Resolve target element by `ref` (RefStore.resolve with tabId
|
||||
// binding) or `selector` (DOM.querySelector + describeNode).
|
||||
// 3. Scroll the node into view, then dispatch the appropriate
|
||||
// `Input.*` CDP events.
|
||||
// 2. Resolve target element by `ref` (compound frame/session identity)
|
||||
// or `selector` (DOM.querySelector + describeNode).
|
||||
// 3. Scroll the node and its frame owners into view, project its live
|
||||
// content quad into the top viewport, then dispatch `Input.*` events.
|
||||
// 4. Honour `AbortSignal` so canceled calls don't issue follow-up CDP
|
||||
// commands.
|
||||
|
||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||
import type {
|
||||
ClickParams,
|
||||
@@ -28,17 +29,13 @@ import type {
|
||||
SelectResult,
|
||||
} from "@/transport/types";
|
||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||
import {
|
||||
backendNodeToObject,
|
||||
boxCentre,
|
||||
nodeCentre,
|
||||
quadCentre,
|
||||
scrollNodeIntoView,
|
||||
} from "./element-geometry";
|
||||
import { backendNodeToObject } from "./element-geometry";
|
||||
import { rpcError } from "./errors";
|
||||
import { resolveNodeGeometry, scrollElementAndFramesIntoView } from "./frame-geometry";
|
||||
import {
|
||||
type CdpRunner,
|
||||
type ChromeTabsApi,
|
||||
cdpRunnerForTarget,
|
||||
chromeTabsApi,
|
||||
enforceAgentWindow,
|
||||
isRpcError,
|
||||
@@ -137,7 +134,16 @@ async function resolveBackendNode(
|
||||
target: { tabId: number },
|
||||
params: { ref?: string; selector?: string },
|
||||
toolName: string,
|
||||
): Promise<{ backendNodeId: number; usedRef?: string; usedSelector?: string } | RpcError> {
|
||||
): Promise<
|
||||
| {
|
||||
backendNodeId: number;
|
||||
cdpTarget: CdpTarget;
|
||||
frameId?: string;
|
||||
usedRef?: string;
|
||||
usedSelector?: string;
|
||||
}
|
||||
| RpcError
|
||||
> {
|
||||
const hasRef = typeof params.ref === "string" && params.ref.length > 0;
|
||||
const hasSelector = typeof params.selector === "string" && params.selector.length > 0;
|
||||
if (hasRef && hasSelector) {
|
||||
@@ -155,7 +161,15 @@ async function resolveBackendNode(
|
||||
if (hasRef) {
|
||||
const resolved = resolveSnapshotRef(ctx, params.ref as string, target.tabId);
|
||||
if (isRpcError(resolved)) return resolved;
|
||||
return { backendNodeId: resolved.backendNodeId, usedRef: resolved.refKey };
|
||||
return {
|
||||
backendNodeId: resolved.backendNodeId,
|
||||
cdpTarget: {
|
||||
tabId: target.tabId,
|
||||
...(resolved.cdpSessionId ? { sessionId: resolved.cdpSessionId } : {}),
|
||||
},
|
||||
...(resolved.frameId ? { frameId: resolved.frameId } : {}),
|
||||
usedRef: resolved.refKey,
|
||||
};
|
||||
}
|
||||
// selector path
|
||||
try {
|
||||
@@ -192,7 +206,7 @@ async function resolveBackendNode(
|
||||
message: "DOM.describeNode returned no backendNodeId",
|
||||
};
|
||||
}
|
||||
return { backendNodeId, usedSelector: params.selector };
|
||||
return { backendNodeId, cdpTarget: { tabId: target.tabId }, usedSelector: params.selector };
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -228,19 +242,19 @@ export async function handleClick(
|
||||
return { code: "cancelled", message: "click aborted" };
|
||||
}
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (scrollErr) return scrollErr;
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
|
||||
const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(centre)) return centre;
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const geometry = await resolveNodeGeometry(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
{
|
||||
target: node.cdpTarget,
|
||||
backendNodeId: node.backendNodeId,
|
||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
const centre = geometry.actionPoint;
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "click aborted" };
|
||||
@@ -353,19 +367,19 @@ export async function handleHover(
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "hover");
|
||||
if (isRpcError(node)) return node;
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (scrollErr) return scrollErr;
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
|
||||
const centre = await nodeCentre(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(centre)) return centre;
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const geometry = await resolveNodeGeometry(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
{
|
||||
target: node.cdpTarget,
|
||||
backendNodeId: node.backendNodeId,
|
||||
...(node.frameId ? { frameId: node.frameId } : {}),
|
||||
},
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
const centre = geometry.actionPoint;
|
||||
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "hover aborted" };
|
||||
@@ -585,10 +599,11 @@ export async function handleFill(
|
||||
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "fill");
|
||||
if (isRpcError(node)) return node;
|
||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const described = await deps.cdp.send<{ node?: DescribedNode }>(
|
||||
const described = await nodeCdp.send<{ node?: DescribedNode }>(
|
||||
target.tabId,
|
||||
"DOM.describeNode",
|
||||
{
|
||||
@@ -602,12 +617,18 @@ export async function handleFill(
|
||||
`element ${described.node?.nodeName ?? "?"} not fillable (need input/textarea/contenteditable)`,
|
||||
);
|
||||
}
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
const scrollErr = await scrollElementAndFramesIntoView(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
node.cdpTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
);
|
||||
if (scrollErr) return scrollErr;
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -619,7 +640,7 @@ export async function handleFill(
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
|
||||
const objectIdOrErr = await backendNodeToObject(deps.cdp, target.tabId, node.backendNodeId);
|
||||
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
||||
const objectId = objectIdOrErr;
|
||||
const clearBefore = params.clear_before ?? true;
|
||||
@@ -628,7 +649,7 @@ export async function handleFill(
|
||||
if (clearBefore) {
|
||||
// Clear input/textarea value or wipe contenteditable innerText,
|
||||
// then fire `input` so frameworks observe the empty state.
|
||||
await deps.cdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
if (this.isContentEditable) { this.textContent = ''; }
|
||||
@@ -655,7 +676,7 @@ export async function handleFill(
|
||||
return { code: "cancelled", message: "fill aborted" };
|
||||
}
|
||||
// Fire `input` + `change` so React / Vue controlled inputs commit.
|
||||
await deps.cdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
await nodeCdp.send(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
functionDeclaration: `function() {
|
||||
this.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
@@ -853,14 +874,21 @@ export async function handlePress(
|
||||
if (params.ref || params.selector) {
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "press");
|
||||
if (isRpcError(node)) return node;
|
||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
const scrollErr = await scrollElementAndFramesIntoView(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
node.cdpTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
);
|
||||
if (scrollErr) return scrollErr;
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "press aborted" };
|
||||
}
|
||||
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -965,10 +993,11 @@ export async function handleSelect(
|
||||
|
||||
const node = await resolveBackendNode(deps.cdp, ctx, target, params, "select");
|
||||
if (isRpcError(node)) return node;
|
||||
const nodeCdp = cdpRunnerForTarget(deps.cdp, node.cdpTarget);
|
||||
|
||||
try {
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
const described = await deps.cdp.send<{ node?: DescribedNode }>(
|
||||
const described = await nodeCdp.send<{ node?: DescribedNode }>(
|
||||
target.tabId,
|
||||
"DOM.describeNode",
|
||||
{ backendNodeId: node.backendNodeId },
|
||||
@@ -991,12 +1020,18 @@ export async function handleSelect(
|
||||
"single-select <select> requires exactly one value",
|
||||
);
|
||||
}
|
||||
const scrollErr = await scrollNodeIntoView(deps.cdp, target.tabId, node.backendNodeId);
|
||||
const scrollErr = await scrollElementAndFramesIntoView(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
node.cdpTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
);
|
||||
if (scrollErr) return scrollErr;
|
||||
if (throwIfAborted(deps.signal)) {
|
||||
return { code: "cancelled", message: "select aborted" };
|
||||
}
|
||||
await deps.cdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
await nodeCdp.send(target.tabId, "DOM.focus", { backendNodeId: node.backendNodeId });
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
@@ -1008,12 +1043,12 @@ export async function handleSelect(
|
||||
return { code: "cancelled", message: "select aborted" };
|
||||
}
|
||||
|
||||
const objectIdOrErr = await backendNodeToObject(deps.cdp, target.tabId, node.backendNodeId);
|
||||
const objectIdOrErr = await backendNodeToObject(nodeCdp, target.tabId, node.backendNodeId);
|
||||
if (isRpcError(objectIdOrErr)) return objectIdOrErr;
|
||||
const objectId = objectIdOrErr;
|
||||
|
||||
try {
|
||||
const evaluated = await deps.cdp.send<{
|
||||
const evaluated = await nodeCdp.send<{
|
||||
result?: { value?: SelectMutationResult | null };
|
||||
}>(target.tabId, "Runtime.callFunctionOn", {
|
||||
objectId,
|
||||
@@ -1072,9 +1107,6 @@ export async function handleSelect(
|
||||
|
||||
export const __testing__ = {
|
||||
DEFAULT_TIMEOUT_MS,
|
||||
quadCentre,
|
||||
boxCentre,
|
||||
resolveBackendNode,
|
||||
nodeCentre,
|
||||
isFillable,
|
||||
};
|
||||
|
||||
@@ -325,14 +325,14 @@ function startLifecycleWait(
|
||||
if (source.tabId !== expectedTabId) return;
|
||||
|
||||
if (targetName === "commit" && method === "Page.frameNavigated") {
|
||||
const p = params as { frame?: { id?: string; parentId?: string } };
|
||||
const p = params as { frame?: { id?: string; parentId?: string; loaderId?: string } };
|
||||
const expectedFrameId = currentFrameId();
|
||||
if (expectedFrameId.length > 0) {
|
||||
if (p.frame?.id !== expectedFrameId) return;
|
||||
} else if (p.frame?.parentId) {
|
||||
return;
|
||||
}
|
||||
if (!acceptLifecycleEvent("commit", p.frame?.id)) return;
|
||||
if (!acceptLifecycleEvent("commit", p.frame?.id, p.frame?.loaderId)) return;
|
||||
finish({ reached: "match", lastLifecycle: "commit" });
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -37,17 +37,22 @@ export async function handleNetwork(
|
||||
manager: SessionManager,
|
||||
params: NetworkParams,
|
||||
deps: NetworkDeps = defaultNetworkDeps(),
|
||||
signal?: AbortSignal,
|
||||
): Promise<NetworkResult | RpcError> {
|
||||
if (signal?.aborted) return { code: "cancelled", message: "network aborted" };
|
||||
const ctxOrErr = lookupSession(manager, params, "network");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const bounds = parseBufferedReadBounds(params);
|
||||
if (isRpcError(bounds)) return bounds;
|
||||
const target = await resolveTargetTab(manager, ctxOrErr, params.tab_id, deps.tabsApi);
|
||||
if (isRpcError(target)) return target;
|
||||
if (signal?.aborted) return { code: "cancelled", message: "network aborted" };
|
||||
|
||||
try {
|
||||
if (signal?.aborted) return { code: "cancelled", message: "network aborted" };
|
||||
deps.cdp.trackSessionTab?.(ctxOrErr.sessionId, target.tabId);
|
||||
await deps.cdp.ensureNetworkCapture(target.tabId);
|
||||
if (signal?.aborted) return { code: "cancelled", message: "network aborted" };
|
||||
return deps.cdp.networkEntriesSince(
|
||||
target.tabId,
|
||||
bounds.since,
|
||||
|
||||
@@ -10,10 +10,16 @@ import {
|
||||
isVomReferenceNode,
|
||||
renderVom,
|
||||
type VomNode,
|
||||
type VomOptions,
|
||||
type VomScene,
|
||||
} from "@browser-skill/vom";
|
||||
import { ChromiumCdp } from "@/browser-driver/chromium-cdp";
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import {
|
||||
type CaptureSuppressSendToTab,
|
||||
withOverlaysHiddenForCapture,
|
||||
} from "@/lib/capture-suppress-bridge";
|
||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||
import type {
|
||||
GetHtmlParams,
|
||||
GetHtmlResult,
|
||||
@@ -26,15 +32,17 @@ import type {
|
||||
SnapshotResult,
|
||||
} from "@/transport/types";
|
||||
import { attachDialogs, markDialogCursor } from "./dialogs";
|
||||
import { nodeBoundingRect, scrollNodeIntoView } from "./element-geometry";
|
||||
import { rpcError } from "./errors";
|
||||
import { resolveNodeGeometry } from "./frame-geometry";
|
||||
import {
|
||||
type ChromeTabsApi,
|
||||
enforceToolTargetScope,
|
||||
isRpcError,
|
||||
lookupSession,
|
||||
type ResolvedTargetTab,
|
||||
resolveCdpAccessibleTargetTab,
|
||||
type CdpRunner as SharedCdpRunner,
|
||||
sendToCdpTarget,
|
||||
normaliseRef as sharedNormaliseRef,
|
||||
type ToolEffect,
|
||||
} from "./shared";
|
||||
@@ -45,6 +53,11 @@ import {
|
||||
captureViewModel,
|
||||
collectOverlayExcludedBackendIds,
|
||||
} from "./vom/capture";
|
||||
import { type CapturedFrameDocument, captureFrameData } from "./vom/frame-capture";
|
||||
import {
|
||||
type CaptureVomObservationResult,
|
||||
projectRecordSafeObservation,
|
||||
} from "./vom/record-safe-observation";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Shared helpers (legacy aliases — observation.ts kept exporting these
|
||||
@@ -116,6 +129,12 @@ export interface ScreenshotDeps {
|
||||
cdp?: SharedCdpRunner;
|
||||
tabsApi: ChromeTabsApi;
|
||||
captureApi: ChromeTabsCaptureApi;
|
||||
/**
|
||||
* Bridge used to hide the in-page overlay while a screenshot is taken,
|
||||
* so captured frames only contain page content. Defaults to
|
||||
* `chrome.tabs.sendMessage`; tests inject a fake.
|
||||
*/
|
||||
sendToTab?: CaptureSuppressSendToTab;
|
||||
}
|
||||
|
||||
function defaultScreenshotDeps(): ScreenshotDeps {
|
||||
@@ -126,35 +145,66 @@ function defaultScreenshotDeps(): ScreenshotDeps {
|
||||
};
|
||||
}
|
||||
|
||||
function cancelled(tool: string): RpcError {
|
||||
return { code: "cancelled", message: `${tool} aborted` };
|
||||
}
|
||||
|
||||
function abortError(tool: string): Error {
|
||||
const error = new Error(`${tool} aborted`);
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
function isAbortError(error: unknown): boolean {
|
||||
return (
|
||||
typeof error === "object" &&
|
||||
error !== null &&
|
||||
(error as { name?: string }).name === "AbortError"
|
||||
);
|
||||
}
|
||||
|
||||
function throwIfAborted(signal: AbortSignal | undefined, tool: string): void {
|
||||
if (signal?.aborted) throw abortError(tool);
|
||||
}
|
||||
|
||||
async function captureElementScreenshot(
|
||||
cdp: SharedCdpRunner,
|
||||
tabId: number,
|
||||
target: CdpTarget,
|
||||
backendNodeId: number,
|
||||
frameId?: string,
|
||||
signal?: AbortSignal,
|
||||
): Promise<{ image_base64: string; width: number; height: number } | RpcError> {
|
||||
const scrollErr = await scrollNodeIntoView(cdp, tabId, backendNodeId);
|
||||
if (scrollErr) return scrollErr;
|
||||
|
||||
const rectOrErr = await nodeBoundingRect(cdp, tabId, backendNodeId);
|
||||
if (isRpcError(rectOrErr)) return rectOrErr;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const geometry = await resolveNodeGeometry(
|
||||
cdp,
|
||||
tabId,
|
||||
{ target, backendNodeId, ...(frameId ? { frameId } : {}) },
|
||||
{ scrollIntoView: true },
|
||||
);
|
||||
if (isRpcError(geometry)) return geometry;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const rect = geometry.topBounds;
|
||||
|
||||
try {
|
||||
const shot = await cdp.send<{ data?: string }>(tabId, "Page.captureScreenshot", {
|
||||
format: "png",
|
||||
clip: {
|
||||
x: rectOrErr.x,
|
||||
y: rectOrErr.y,
|
||||
width: rectOrErr.width,
|
||||
height: rectOrErr.height,
|
||||
x: rect.x,
|
||||
y: rect.y,
|
||||
width: rect.width,
|
||||
height: rect.height,
|
||||
scale: 1,
|
||||
},
|
||||
});
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const image_base64 = shot.data ?? "";
|
||||
if (!image_base64) {
|
||||
return { code: "cdp_failed", message: "Page.captureScreenshot returned no data" };
|
||||
}
|
||||
const dims = parsePngDimensions(image_base64) ?? {
|
||||
width: Math.round(rectOrErr.width),
|
||||
height: Math.round(rectOrErr.height),
|
||||
width: Math.round(rect.width),
|
||||
height: Math.round(rect.height),
|
||||
};
|
||||
return { image_base64, width: dims.width, height: dims.height };
|
||||
} catch (err) {
|
||||
@@ -165,11 +215,71 @@ async function captureElementScreenshot(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Full-tab PNG capture. The primary path is `chrome.tabs.captureVisibleTab`;
|
||||
* when it rejects, fall back to CDP `Page.captureScreenshot` with
|
||||
* `fromSurface: true`. `captureVisibleTab` reads back the window surface,
|
||||
* which fails outright on some Windows/Chrome combinations (Chromium's
|
||||
* FAILURE_REASON_READBACK_FAILED — "Failed to capture tab: image readback
|
||||
* failed"); the CDP path captures through the renderer's BeginFrame
|
||||
* pipeline instead, which does not depend on that readback.
|
||||
*
|
||||
* When both paths fail the returned error carries
|
||||
* `data.reason = "screenshot_capture_failed"` and both underlying messages
|
||||
* so the CLI can point the user at the browser-side cause.
|
||||
*/
|
||||
async function captureFullTabPng(
|
||||
deps: ScreenshotDeps,
|
||||
ctx: SessionContext,
|
||||
target: ResolvedTargetTab,
|
||||
signal?: AbortSignal,
|
||||
): Promise<string | RpcError> {
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
try {
|
||||
const dataUrl = await deps.captureApi.captureVisibleTab(target.windowId, { format: "png" });
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
return stripDataUrlPrefix(dataUrl);
|
||||
} catch (primaryErr) {
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const cdp = deps.cdp;
|
||||
if (!cdp) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: primaryErr instanceof Error ? primaryErr.message : String(primaryErr),
|
||||
};
|
||||
}
|
||||
let fallbackMsg: string;
|
||||
try {
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const shot = await cdp.send<{ data?: string }>(target.tabId, "Page.captureScreenshot", {
|
||||
format: "png",
|
||||
fromSurface: true,
|
||||
});
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
if (shot.data) return shot.data;
|
||||
fallbackMsg = "Page.captureScreenshot returned no data";
|
||||
} catch (fallbackErr) {
|
||||
fallbackMsg = fallbackErr instanceof Error ? fallbackErr.message : String(fallbackErr);
|
||||
}
|
||||
const primaryMsg = primaryErr instanceof Error ? primaryErr.message : String(primaryErr);
|
||||
return rpcError(
|
||||
"cdp_failed",
|
||||
"screenshot_capture_failed",
|
||||
`captureVisibleTab failed: ${primaryMsg}; CDP Page.captureScreenshot fallback failed: ${fallbackMsg}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export async function handleScreenshot(
|
||||
manager: SessionManager,
|
||||
params: ScreenshotParams,
|
||||
deps: ScreenshotDeps = defaultScreenshotDeps(),
|
||||
signal?: AbortSignal,
|
||||
): Promise<ScreenshotResult | RpcError> {
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const ctxOrErr = lookupSession(manager, params, "screenshot");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
@@ -181,6 +291,7 @@ export async function handleScreenshot(
|
||||
"screenshot",
|
||||
);
|
||||
if (isRpcError(target)) return target;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const dialogCursor = deps.cdp ? markDialogCursor(deps.cdp, target.tabId) : 0;
|
||||
const withShotDialogs = <T extends object>(result: T) =>
|
||||
deps.cdp ? attachDialogs(deps.cdp, target.tabId, dialogCursor, result) : result;
|
||||
@@ -192,10 +303,30 @@ export async function handleScreenshot(
|
||||
}
|
||||
const node = resolveSnapshotRef(ctx, ref, target.tabId);
|
||||
if (isRpcError(node)) return node;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||
const captured = await captureElementScreenshot(deps.cdp, target.tabId, node.backendNodeId);
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const cdp = deps.cdp;
|
||||
const nodeTarget = {
|
||||
tabId: target.tabId,
|
||||
...(node.cdpSessionId ? { sessionId: node.cdpSessionId } : {}),
|
||||
};
|
||||
const captured = await withOverlaysHiddenForCapture(
|
||||
target.tabId,
|
||||
() =>
|
||||
captureElementScreenshot(
|
||||
cdp,
|
||||
target.tabId,
|
||||
nodeTarget,
|
||||
node.backendNodeId,
|
||||
node.frameId,
|
||||
signal,
|
||||
),
|
||||
deps.sendToTab,
|
||||
);
|
||||
if (isRpcError(captured)) return captured;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
return withShotDialogs({
|
||||
image_base64: captured.image_base64,
|
||||
width: captured.width,
|
||||
@@ -213,23 +344,22 @@ export async function handleScreenshot(
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const dataUrl = await deps.captureApi.captureVisibleTab(target.windowId, { format: "png" });
|
||||
const image_base64 = stripDataUrlPrefix(dataUrl);
|
||||
const dims = parsePngDimensions(image_base64) ?? { width: 0, height: 0 };
|
||||
return withShotDialogs({
|
||||
image_base64,
|
||||
width: dims.width,
|
||||
height: dims.height,
|
||||
format: "png",
|
||||
tab_id: target.tabId,
|
||||
});
|
||||
} catch (err) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
};
|
||||
}
|
||||
const captured = await withOverlaysHiddenForCapture(
|
||||
target.tabId,
|
||||
() => captureFullTabPng(deps, ctx, target, signal),
|
||||
deps.sendToTab,
|
||||
);
|
||||
if (isRpcError(captured)) return captured;
|
||||
if (signal?.aborted) return cancelled("screenshot");
|
||||
const image_base64 = captured;
|
||||
const dims = parsePngDimensions(image_base64) ?? { width: 0, height: 0 };
|
||||
return withShotDialogs({
|
||||
image_base64,
|
||||
width: dims.width,
|
||||
height: dims.height,
|
||||
format: "png",
|
||||
tab_id: target.tabId,
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -246,6 +376,7 @@ export type CdpRunner = SharedCdpRunner;
|
||||
/** Subset of CDP `AXNode` we care about — see `Accessibility.AXNode`. */
|
||||
export interface CdpAxNode {
|
||||
nodeId: string;
|
||||
frameId?: string;
|
||||
parentId?: string;
|
||||
backendDOMNodeId?: number;
|
||||
ignored?: boolean;
|
||||
@@ -285,7 +416,15 @@ function isModalSignal(
|
||||
|
||||
function isSensitive(capturedNode: CapturedNode | undefined): boolean {
|
||||
const attrs = capturedNode?.attrs ?? {};
|
||||
return (attrs.type ?? "").toLowerCase() === "password";
|
||||
const type = (attrs.type ?? "").toLowerCase();
|
||||
if (type === "password") return true;
|
||||
const autocomplete = (attrs.autocomplete ?? "").toLowerCase();
|
||||
return (
|
||||
autocomplete.startsWith("cc-") ||
|
||||
autocomplete === "one-time-code" ||
|
||||
autocomplete === "current-password" ||
|
||||
autocomplete === "new-password"
|
||||
);
|
||||
}
|
||||
|
||||
interface AxNodeSignals {
|
||||
@@ -335,7 +474,13 @@ const AX_TEXT_STOP_ROLES = new Set([
|
||||
"columnheader",
|
||||
"rowheader",
|
||||
]);
|
||||
const SENSITIVE_AX_INPUT_TYPES = new Set(["password", "credit-card"]);
|
||||
const SENSITIVE_AX_INPUT_TYPES = new Set([
|
||||
"password",
|
||||
"credit-card",
|
||||
"one-time-code",
|
||||
"current-password",
|
||||
"new-password",
|
||||
]);
|
||||
|
||||
function axPropertyString(axNode: CdpAxNode, name: string): string | undefined {
|
||||
const prop = axNode.properties?.find((item) => item.name === name);
|
||||
@@ -478,7 +623,7 @@ function iframeNameFor(node: CapturedNode): string | undefined {
|
||||
function isRenderableIframeControl(node: CapturedNode): boolean {
|
||||
const tag = normalizeTag(node.tag);
|
||||
if (!IFRAME_RENDERABLE_TAGS.has(tag)) return false;
|
||||
if (!node.rect) return false;
|
||||
if (!node.localRect && !node.rect) return false;
|
||||
if (node.pointerEvents === "none") return false;
|
||||
if ((node.attrs.type ?? "").toLowerCase() === "hidden") return false;
|
||||
|
||||
@@ -650,7 +795,12 @@ function applyCapturedSignals(
|
||||
signals: VomNodeDomSignals,
|
||||
): VomNode {
|
||||
if (!capturedNode) return node;
|
||||
const attrs = capturedNode.attrs;
|
||||
const attrs =
|
||||
node.sensitive && Object.prototype.hasOwnProperty.call(capturedNode.attrs, "value")
|
||||
? Object.fromEntries(
|
||||
Object.entries(capturedNode.attrs).filter(([name]) => name.toLowerCase() !== "value"),
|
||||
)
|
||||
: capturedNode.attrs;
|
||||
const text = cleanAttr(capturedNode.textContent);
|
||||
const nearbyText = nearbyTextFor(capturedNode, signals.childrenByParentId);
|
||||
const placeholder = cleanAttr(capturedNode.formPlaceholder) ?? cleanAttr(attrs.placeholder);
|
||||
@@ -717,8 +867,7 @@ function vomNodeFromCaptured(
|
||||
signals: VomNodeDomSignals,
|
||||
): VomNode {
|
||||
const sensitive = isSensitive(capturedNode);
|
||||
const value =
|
||||
capturedNode.formValue ?? (sensitive ? (capturedNode.attrs.value ?? "") : undefined);
|
||||
const value = sensitive ? undefined : capturedNode.formValue;
|
||||
const tag = normalizeTag(capturedNode.tag);
|
||||
const node = applyCapturedSignals(
|
||||
{
|
||||
@@ -754,8 +903,17 @@ function axVomNode(
|
||||
): VomNode {
|
||||
const role = axString(axNode.role);
|
||||
const capturedValue = capturedNode?.formValue;
|
||||
const value = capturedValue ?? axValue(axNode.value);
|
||||
const sourceValue = capturedValue ?? axValue(axNode.value);
|
||||
const signalsForNode = axSignals.get(axNode.nodeId);
|
||||
const sensitive = isSensitive(capturedNode) || signalsForNode?.sensitive === true;
|
||||
const value = sensitive ? undefined : sourceValue;
|
||||
const inputState =
|
||||
inputStateFor(capturedNode, sourceValue) ??
|
||||
(sensitive && ["textbox", "searchbox"].includes(role?.toLowerCase() ?? "")
|
||||
? sourceValue === undefined || sourceValue === ""
|
||||
? "empty"
|
||||
: "filled"
|
||||
: undefined);
|
||||
let name = FORM_CONTROL_TAGS.has(normalizeTag(capturedNode?.tag))
|
||||
? formControlName(capturedNode, axString(axNode.name), axSiblingLabel(axNode, axById), signals)
|
||||
: (axString(axNode.name) ?? signalsForNode?.aggregatedText);
|
||||
@@ -771,8 +929,8 @@ function axVomNode(
|
||||
position: capturedNode?.position || "static",
|
||||
pointerEvents: capturedNode?.pointerEvents || "auto",
|
||||
modal: isModalSignal(axNode, capturedNode),
|
||||
sensitive: isSensitive(capturedNode) || signalsForNode?.sensitive === true,
|
||||
inputState: inputStateFor(capturedNode, value),
|
||||
sensitive,
|
||||
inputState,
|
||||
};
|
||||
if (role) node.role = role;
|
||||
if (name) node.name = name;
|
||||
@@ -1035,6 +1193,8 @@ export interface BuildVomSceneOptions {
|
||||
pageUrl?: string;
|
||||
}
|
||||
|
||||
export type VomFrameDocument = CapturedFrameDocument<CdpAxNode>;
|
||||
|
||||
function externalHrefHost(
|
||||
href: string | undefined,
|
||||
pageUrl: string | undefined,
|
||||
@@ -1128,6 +1288,119 @@ export function buildVomScene(
|
||||
};
|
||||
}
|
||||
|
||||
export function buildFrameVomScene(
|
||||
documents: VomFrameDocument[],
|
||||
captured: CapturedViewModel,
|
||||
options: BuildVomSceneOptions = {},
|
||||
): VomScene {
|
||||
const documentByFrameId = new Map(documents.map((document) => [document.frameId, document]));
|
||||
const rootFrameId =
|
||||
captured.rootFrameId ?? documents.find((document) => !document.parentFrameId)?.frameId;
|
||||
const orderedFrameIds = [
|
||||
...(rootFrameId ? [rootFrameId] : []),
|
||||
...documents.map((document) => document.frameId),
|
||||
].filter((frameId, index, all) => all.indexOf(frameId) === index);
|
||||
|
||||
const localScenes = new Map<string, VomScene>();
|
||||
for (const frameId of orderedFrameIds) {
|
||||
const document = documentByFrameId.get(frameId);
|
||||
if (!document) continue;
|
||||
localScenes.set(
|
||||
frameId,
|
||||
buildVomScene(
|
||||
document.axNodes,
|
||||
{
|
||||
viewport: captured.viewport,
|
||||
nodes: document.domNodes,
|
||||
iframeNodes: new Map(),
|
||||
surfaceProbes: frameId === rootFrameId ? captured.surfaceProbes : [],
|
||||
excludedBackendNodeIds:
|
||||
frameId === rootFrameId ? captured.excludedBackendNodeIds : new Set(),
|
||||
},
|
||||
{ pageUrl: document.url ?? options.pageUrl },
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const sceneIdByFrameAndBackend = new Map<string, number>();
|
||||
let nextSceneId = 1;
|
||||
for (const [frameId, scene] of localScenes) {
|
||||
for (const node of scene.nodes) {
|
||||
sceneIdByFrameAndBackend.set(`${frameId}:${node.id}`, nextSceneId);
|
||||
nextSceneId += 1;
|
||||
}
|
||||
}
|
||||
|
||||
const nodes: VomNode[] = [];
|
||||
const surfaces: CondSurface[] = [];
|
||||
const activeScopeBlocks: ActiveScopeBlock[] = [];
|
||||
|
||||
for (const [frameId, scene] of localScenes) {
|
||||
const document = documentByFrameId.get(frameId);
|
||||
const ownerBackendNodeId = document?.ownerBackendNodeId;
|
||||
const parentFrameId = document?.parentFrameId;
|
||||
const ownerSceneId =
|
||||
ownerBackendNodeId !== undefined && parentFrameId
|
||||
? sceneIdByFrameAndBackend.get(`${parentFrameId}:${ownerBackendNodeId}`)
|
||||
: undefined;
|
||||
if (frameId !== rootFrameId && ownerSceneId === undefined) continue;
|
||||
|
||||
const suppressedRoots = new Set(
|
||||
frameId === rootFrameId
|
||||
? []
|
||||
: scene.nodes
|
||||
.filter(
|
||||
(node) =>
|
||||
node.parentId === null &&
|
||||
["rootwebarea", "webarea"].includes(node.role?.toLowerCase() ?? ""),
|
||||
)
|
||||
.map((node) => node.id),
|
||||
);
|
||||
|
||||
for (const node of scene.nodes) {
|
||||
if (suppressedRoots.has(node.id)) continue;
|
||||
const sceneNodeId = sceneIdByFrameAndBackend.get(`${frameId}:${node.id}`) as number;
|
||||
let parentId =
|
||||
node.parentId !== null
|
||||
? sceneIdByFrameAndBackend.get(`${frameId}:${node.parentId}`)
|
||||
: ownerSceneId;
|
||||
if (node.parentId !== null && suppressedRoots.has(node.parentId)) parentId = ownerSceneId;
|
||||
const remapId = (id: number) => sceneIdByFrameAndBackend.get(`${frameId}:${id}`);
|
||||
nodes.push({
|
||||
...node,
|
||||
id: sceneNodeId,
|
||||
backendNodeId: node.id,
|
||||
frameId,
|
||||
contextScopeId: document?.contextScopeId ?? frameId,
|
||||
parentId: parentId ?? null,
|
||||
...(node.domParentId !== undefined
|
||||
? { domParentId: node.domParentId === null ? null : (remapId(node.domParentId) ?? null) }
|
||||
: {}),
|
||||
...(node.domAncestorIds
|
||||
? { domAncestorIds: node.domAncestorIds.flatMap((id) => remapId(id) ?? []) }
|
||||
: {}),
|
||||
});
|
||||
}
|
||||
|
||||
for (const surface of scene.surfaces ?? []) {
|
||||
const triggerId = sceneIdByFrameAndBackend.get(`${frameId}:${surface.triggerId}`);
|
||||
if (triggerId !== undefined) surfaces.push({ ...surface, triggerId });
|
||||
}
|
||||
for (const block of scene.activeScopeBlocks ?? []) {
|
||||
const triggerId = sceneIdByFrameAndBackend.get(`${frameId}:${block.triggerId}`);
|
||||
if (triggerId !== undefined) activeScopeBlocks.push({ ...block, triggerId });
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
viewport: captured.viewport,
|
||||
nodes,
|
||||
...(rootFrameId ? { rootFrameId } : {}),
|
||||
...(surfaces.length > 0 ? { surfaces } : {}),
|
||||
...(activeScopeBlocks.length > 0 ? { activeScopeBlocks } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export interface SnapshotDeps {
|
||||
cdp: CdpRunner;
|
||||
tabsApi: {
|
||||
@@ -1187,7 +1460,9 @@ export async function handleGetHtml(
|
||||
manager: SessionManager,
|
||||
params: GetHtmlParams,
|
||||
deps: SnapshotDeps = getDefaultDeps(),
|
||||
signal?: AbortSignal,
|
||||
): Promise<GetHtmlResult | RpcError> {
|
||||
if (signal?.aborted) return cancelled("get_html");
|
||||
const ctxOrErr = lookupSession(manager, params, "get_html");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
@@ -1199,21 +1474,31 @@ export async function handleGetHtml(
|
||||
"get_html",
|
||||
);
|
||||
if (isRpcError(target)) return target;
|
||||
if (signal?.aborted) return cancelled("get_html");
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
|
||||
const maxBytes =
|
||||
params.max_bytes && params.max_bytes > 0 ? params.max_bytes : DEFAULT_GET_HTML_MAX_BYTES;
|
||||
|
||||
try {
|
||||
throwIfAborted(signal, "get_html");
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||
throwIfAborted(signal, "get_html");
|
||||
let html: string;
|
||||
if (params.ref) {
|
||||
const resolved = resolveSnapshotRef(ctx, params.ref, target.tabId);
|
||||
if (isRpcError(resolved)) return resolved;
|
||||
const resp = await deps.cdp.send<{ outerHTML?: string }>(target.tabId, "DOM.getOuterHTML", {
|
||||
backendNodeId: resolved.backendNodeId,
|
||||
});
|
||||
const resp = await sendToCdpTarget<{ outerHTML?: string }>(
|
||||
deps.cdp,
|
||||
{
|
||||
tabId: target.tabId,
|
||||
...(resolved.cdpSessionId ? { sessionId: resolved.cdpSessionId } : {}),
|
||||
},
|
||||
"DOM.getOuterHTML",
|
||||
{ backendNodeId: resolved.backendNodeId },
|
||||
);
|
||||
throwIfAborted(signal, "get_html");
|
||||
html = resp.outerHTML ?? "";
|
||||
} else {
|
||||
const doc = await deps.cdp.send<{ root?: { nodeId?: number } }>(
|
||||
@@ -1221,6 +1506,7 @@ export async function handleGetHtml(
|
||||
"DOM.getDocument",
|
||||
{ depth: 0 },
|
||||
);
|
||||
throwIfAborted(signal, "get_html");
|
||||
const nodeId = doc.root?.nodeId;
|
||||
if (typeof nodeId !== "number") {
|
||||
return {
|
||||
@@ -1231,6 +1517,7 @@ export async function handleGetHtml(
|
||||
const resp = await deps.cdp.send<{ outerHTML?: string }>(target.tabId, "DOM.getOuterHTML", {
|
||||
nodeId,
|
||||
});
|
||||
throwIfAborted(signal, "get_html");
|
||||
html = resp.outerHTML ?? "";
|
||||
}
|
||||
const originalBytes = utf8ByteLength(html);
|
||||
@@ -1242,6 +1529,7 @@ export async function handleGetHtml(
|
||||
tab_id: target.tabId,
|
||||
});
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) return cancelled("get_html");
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
@@ -1261,38 +1549,81 @@ interface LayoutMetricsViewportReply {
|
||||
async function fallbackCapturedViewModel(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CapturedViewModel> {
|
||||
throwIfAborted(signal, "observation");
|
||||
let viewport = { width: 0, height: 0 };
|
||||
try {
|
||||
const metrics = await cdp.send<LayoutMetricsViewportReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
throwIfAborted(signal, "observation");
|
||||
const source = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
viewport = {
|
||||
width: vpSrc.clientWidth ?? 0,
|
||||
height: vpSrc.clientHeight ?? 0,
|
||||
width: source.clientWidth ?? 0,
|
||||
height: source.clientHeight ?? 0,
|
||||
};
|
||||
} catch {
|
||||
// viewport stays zero-sized
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
}
|
||||
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId);
|
||||
const excludedBackendNodeIds = await collectOverlayExcludedBackendIds(cdp, tabId, signal);
|
||||
throwIfAborted(signal, "observation");
|
||||
return { ...emptyCapturedViewModel(viewport), excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
async function captureForVom(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
conditionalSurfaceProbe: boolean,
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>,
|
||||
options: CaptureVomObservationOptions,
|
||||
): Promise<CapturedViewModel> {
|
||||
try {
|
||||
return await captureViewModel(cdp, tabId, {
|
||||
conditionalSurfaceProbe,
|
||||
hoverProbeBypassOverlay,
|
||||
conditionalSurfaceProbe: options.conditionalSurfaceProbe ?? false,
|
||||
hoverProbeBypassOverlay: options.hoverProbeBypassOverlay,
|
||||
signal: options.signal,
|
||||
});
|
||||
} catch {
|
||||
return fallbackCapturedViewModel(cdp, tabId);
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
return fallbackCapturedViewModel(cdp, tabId, options.signal);
|
||||
}
|
||||
}
|
||||
|
||||
export interface CaptureVomObservationOptions extends VomOptions {
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export async function captureVomObservation(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
url: string | undefined,
|
||||
options: CaptureVomObservationOptions = {},
|
||||
): Promise<CaptureVomObservationResult> {
|
||||
throwIfAborted(options.signal, "observation");
|
||||
await cdp.ensureAttachedToUrl?.(tabId, url);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const captured = await captureForVom(cdp, tabId, options);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const documents = await captureFrameData<CdpAxNode>(cdp, tabId, captured, options.signal);
|
||||
throwIfAborted(options.signal, "observation");
|
||||
const scene =
|
||||
captured.rootFrameId && captured.frameNodes?.size
|
||||
? buildFrameVomScene(documents, captured, { pageUrl: url })
|
||||
: buildVomScene(documents[0]?.axNodes ?? [], captured, { pageUrl: url });
|
||||
const rendered = renderVom(scene, {
|
||||
maxDepth: options.maxDepth,
|
||||
maxTokens: options.maxTokens,
|
||||
redactValues: options.redactValues,
|
||||
activeRegionPolicy: options.activeRegionPolicy,
|
||||
});
|
||||
throwIfAborted(options.signal, "observation");
|
||||
return projectRecordSafeObservation({
|
||||
rootFrameId: captured.rootFrameId ?? documents[0]?.frameId ?? "root",
|
||||
frameDocuments: documents,
|
||||
rendered,
|
||||
surfaceProbes: captured.surfaceProbes,
|
||||
});
|
||||
}
|
||||
|
||||
async function handleVomObservation(
|
||||
manager: SessionManager,
|
||||
params: SnapshotParams | ObserveParams,
|
||||
@@ -1300,7 +1631,9 @@ async function handleVomObservation(
|
||||
effect: ToolEffect,
|
||||
conditionalSurfaceProbe: boolean,
|
||||
deps: SnapshotDeps = getDefaultDeps(),
|
||||
signal?: AbortSignal,
|
||||
): Promise<SnapshotResult | ObserveResult | RpcError> {
|
||||
if (signal?.aborted) return cancelled(toolName);
|
||||
const ctxOrErr = lookupSession(manager, params, toolName);
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
@@ -1312,48 +1645,51 @@ async function handleVomObservation(
|
||||
toolName,
|
||||
);
|
||||
if (isRpcError(target)) return target;
|
||||
if (signal?.aborted) return cancelled(toolName);
|
||||
const denied = enforceToolTargetScope(ctx, target, effect, toolName);
|
||||
if (denied) return denied;
|
||||
const dialogCursor = markDialogCursor(deps.cdp, target.tabId);
|
||||
|
||||
try {
|
||||
throwIfAborted(signal, toolName);
|
||||
deps.cdp.trackSessionTab?.(ctx.sessionId, target.tabId);
|
||||
await deps.cdp.ensureAttachedToUrl?.(target.tabId, target.url);
|
||||
await deps.cdp.send<unknown>(target.tabId, "Accessibility.enable", {});
|
||||
const result = await deps.cdp.send<{ nodes: CdpAxNode[] }>(
|
||||
target.tabId,
|
||||
"Accessibility.getFullAXTree",
|
||||
{},
|
||||
);
|
||||
const axNodes = result.nodes ?? [];
|
||||
const effectiveConditionalSurfaceProbe =
|
||||
deps.conditionalSurfaceProbe ?? conditionalSurfaceProbe;
|
||||
const captured = await captureForVom(
|
||||
deps.cdp,
|
||||
target.tabId,
|
||||
effectiveConditionalSurfaceProbe,
|
||||
deps.hoverProbeBypassOverlay,
|
||||
);
|
||||
const scene = buildVomScene(axNodes, captured, { pageUrl: target.url });
|
||||
const rendered = renderVom(scene, {
|
||||
const observation = await captureVomObservation(deps.cdp, target.tabId, target.url, {
|
||||
maxDepth: params.max_depth,
|
||||
maxTokens: params.max_tokens,
|
||||
activeRegionPolicy: true,
|
||||
conditionalSurfaceProbe: effectiveConditionalSurfaceProbe,
|
||||
hoverProbeBypassOverlay: deps.hoverProbeBypassOverlay,
|
||||
signal,
|
||||
});
|
||||
throwIfAborted(signal, toolName);
|
||||
const targetByFrameId = new Map(
|
||||
observation.frames.map((frame) => [frame.frameId, frame.target]),
|
||||
);
|
||||
ctx.refStore.replace(
|
||||
rendered.refs.map(
|
||||
(r) => [r.ref, { backendNodeId: r.backendNodeId, tabId: target.tabId }] as const,
|
||||
),
|
||||
observation.refs.map((ref) => {
|
||||
const refTarget = ref.frameId ? targetByFrameId.get(ref.frameId) : undefined;
|
||||
return [
|
||||
ref.ref,
|
||||
{
|
||||
backendNodeId: ref.backendNodeId,
|
||||
tabId: target.tabId,
|
||||
...(ref.frameId ? { frameId: ref.frameId } : {}),
|
||||
...(refTarget?.sessionId ? { cdpSessionId: refTarget.sessionId } : {}),
|
||||
},
|
||||
] as const;
|
||||
}),
|
||||
);
|
||||
return attachDialogs(deps.cdp, target.tabId, dialogCursor, {
|
||||
text: rendered.text,
|
||||
ref_count: rendered.refs.length,
|
||||
text: observation.text,
|
||||
ref_count: observation.refs.length,
|
||||
tab_id: target.tabId,
|
||||
truncated: rendered.truncated,
|
||||
truncated: observation.truncated,
|
||||
...(toolName === "observe" && (params as ObserveParams).debug_surfaces
|
||||
? {
|
||||
debug: {
|
||||
surface_probes: (captured.surfaceProbes ?? []).map((probe) => ({
|
||||
surface_probes: (observation.surfaceProbes ?? []).map((probe) => ({
|
||||
trigger_backend_node_id: probe.triggerBackendNodeId,
|
||||
...(probe.triggerPoint ? { trigger_point: probe.triggerPoint } : {}),
|
||||
trigger_action: probe.triggerAction,
|
||||
@@ -1365,6 +1701,7 @@ async function handleVomObservation(
|
||||
: {}),
|
||||
});
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) return cancelled(toolName);
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: err instanceof Error ? err.message : String(err),
|
||||
@@ -1376,14 +1713,16 @@ export async function handleSnapshot(
|
||||
manager: SessionManager,
|
||||
params: SnapshotParams,
|
||||
deps: SnapshotDeps = getDefaultDeps(),
|
||||
signal?: AbortSignal,
|
||||
): Promise<SnapshotResult | RpcError> {
|
||||
return handleVomObservation(manager, params, "snapshot", "passive_read", false, deps);
|
||||
return handleVomObservation(manager, params, "snapshot", "passive_read", false, deps, signal);
|
||||
}
|
||||
|
||||
export async function handleObserve(
|
||||
manager: SessionManager,
|
||||
params: ObserveParams,
|
||||
deps: SnapshotDeps = getDefaultDeps(),
|
||||
signal?: AbortSignal,
|
||||
): Promise<ObserveResult | RpcError> {
|
||||
return handleVomObservation(manager, params, "observe", "transient_input", true, deps);
|
||||
return handleVomObservation(manager, params, "observe", "transient_input", true, deps, signal);
|
||||
}
|
||||
|
||||
@@ -49,6 +49,7 @@ interface ActiveRecording {
|
||||
purpose?: string;
|
||||
steps: DraftTraceStep[];
|
||||
startedAt: string;
|
||||
startedAtMs: number;
|
||||
finishPromise: Promise<Trace>;
|
||||
resolveFinish: (trace: Trace) => void;
|
||||
rejectFinish: (err: Error) => void;
|
||||
@@ -373,7 +374,11 @@ async function rearmRecording(
|
||||
// stuck with pointer-events:none (page usable, Interrupt dead). RecordOverlay
|
||||
// already hides the control chrome while activeRecord is set.
|
||||
for (let attempt = 0; attempt < RECORD_REARM_MAX_ATTEMPTS; attempt += 1) {
|
||||
const startMsg: RecordStartMessage = { type: RECORD_START, requestId: recording.requestId };
|
||||
const startMsg: RecordStartMessage = {
|
||||
type: RECORD_START,
|
||||
requestId: recording.requestId,
|
||||
startedAtMs: recording.startedAtMs,
|
||||
};
|
||||
try {
|
||||
await sendRecordStartWithAck(targetTabId, startMsg, deps.sendToTab);
|
||||
recording.tabId = targetTabId;
|
||||
@@ -501,7 +506,11 @@ export function attachRecordQueryListener(deps: RecordDeps = getDefaultDeps()):
|
||||
return;
|
||||
}
|
||||
await rearmRecording(recording, tabId, deps);
|
||||
sendResponse({ active: true, requestId: recording.requestId });
|
||||
sendResponse({
|
||||
active: true,
|
||||
requestId: recording.requestId,
|
||||
startedAtMs: recording.startedAtMs,
|
||||
});
|
||||
})();
|
||||
return true;
|
||||
};
|
||||
@@ -572,6 +581,7 @@ export async function handleRecordStart(
|
||||
rejectFinish = reject;
|
||||
});
|
||||
const navigateUrl = params.url ?? RECORD_DEFAULT_START_URL;
|
||||
const startedAtMs = Date.now();
|
||||
recordings.set(params.session_id, {
|
||||
requestId,
|
||||
tabId: target.tabId,
|
||||
@@ -579,7 +589,8 @@ export async function handleRecordStart(
|
||||
startUrl: navigateUrl,
|
||||
...(params.purpose ? { purpose: params.purpose } : {}),
|
||||
steps: [],
|
||||
startedAt: new Date().toISOString(),
|
||||
startedAt: new Date(startedAtMs).toISOString(),
|
||||
startedAtMs,
|
||||
finishPromise,
|
||||
resolveFinish,
|
||||
rejectFinish,
|
||||
@@ -707,7 +718,7 @@ export async function handleRecordStart(
|
||||
if (cancelled) return cancelled;
|
||||
}
|
||||
|
||||
const startMsg: RecordStartMessage = { type: RECORD_START, requestId };
|
||||
const startMsg: RecordStartMessage = { type: RECORD_START, requestId, startedAtMs };
|
||||
|
||||
try {
|
||||
await sendRecordStartWithAck(target.tabId, startMsg, deps.sendToTab);
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import type { SessionManager } from "@/session-manager/manager";
|
||||
import { type SessionManager, SessionStartCleanupError } from "@/session-manager/manager";
|
||||
import type { RpcError } from "@/transport/types";
|
||||
import { rpcError } from "./errors";
|
||||
import { clearRecordingForSession } from "./record";
|
||||
import { isRpcError } from "./shared";
|
||||
import { returnBorrowedTab, type TabManagementDeps } from "./tabs";
|
||||
@@ -50,12 +51,18 @@ export interface SessionStartParams {
|
||||
width?: number;
|
||||
/** Optional Agent Window outer height in CSS pixels (100..=7680). */
|
||||
height?: number;
|
||||
/** Defaults to true so existing clients preserve visible Agent Windows. */
|
||||
focused?: boolean;
|
||||
}
|
||||
|
||||
export interface SessionStartResult {
|
||||
agent_window_id?: number;
|
||||
}
|
||||
|
||||
export interface SessionStartDeps {
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
export interface SessionStopParams {
|
||||
session_id: string;
|
||||
}
|
||||
@@ -89,6 +96,7 @@ export interface SessionStopDeps {
|
||||
export async function handleSessionStart(
|
||||
manager: SessionManager,
|
||||
params: SessionStartParams,
|
||||
deps: SessionStartDeps = {},
|
||||
): Promise<SessionStartResult | RpcError> {
|
||||
if (!params?.session_id) {
|
||||
return {
|
||||
@@ -99,9 +107,26 @@ export async function handleSessionStart(
|
||||
const sizeOrErr = validateWindowSize(params.width, params.height);
|
||||
if (isRpcError(sizeOrErr)) return sizeOrErr;
|
||||
try {
|
||||
const ctx = await manager.start(params.session_id, sizeOrErr);
|
||||
const ctx = await manager.start(params.session_id, {
|
||||
size: sizeOrErr,
|
||||
focused: params.focused,
|
||||
signal: deps.signal,
|
||||
});
|
||||
return { agent_window_id: ctx.agentWindowId };
|
||||
} catch (err) {
|
||||
if (err instanceof SessionStartCleanupError) {
|
||||
return rpcError("protocol_error", "cleanup_failed", err.message, {
|
||||
resource_type: "agent_window",
|
||||
resource_id: err.windowId,
|
||||
});
|
||||
}
|
||||
if (
|
||||
typeof err === "object" &&
|
||||
err !== null &&
|
||||
(err as { name?: string }).name === "AbortError"
|
||||
) {
|
||||
return { code: "cancelled", message: "session_start aborted" };
|
||||
}
|
||||
// chrome.windows.create / SessionManager failures are not CDP
|
||||
// failures (§4.5 reserves cdp_failed for raw CDP errors). Surface
|
||||
// them as protocol_error so the CLI maps to the right exit code
|
||||
|
||||
@@ -6,7 +6,8 @@
|
||||
// exactly the same sandbox + visibility rules as the M6 observation
|
||||
// handlers (review parity).
|
||||
|
||||
import type { DialogCursor } from "@/browser-driver/chromium-cdp";
|
||||
import type { CdpDebuggee, DialogCursor } from "@/browser-driver/chromium-cdp";
|
||||
import type { CdpFrameGraph, CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { SessionContext, SessionManager } from "@/session-manager/manager";
|
||||
import { normaliseRef } from "@/session-manager/ref-store";
|
||||
import type { ConsoleResult, JavaScriptDialogInfo, RpcError } from "@/transport/types";
|
||||
@@ -51,9 +52,11 @@ export type { DialogCursor };
|
||||
*/
|
||||
export interface CdpRunner {
|
||||
send<T = unknown>(tabId: number, method: string, params?: object): Promise<T>;
|
||||
sendToTarget?<T = unknown>(target: CdpTarget, method: string, params?: object): Promise<T>;
|
||||
getFrameGraph?(tabId: number): Promise<CdpFrameGraph>;
|
||||
ensureAttachedToUrl?(tabId: number, expectedUrl: string | undefined): Promise<void>;
|
||||
trackSessionTab?(sessionId: string, tabId: number): void;
|
||||
onEvent?(handler: (source: chrome.debugger.Debuggee, method: string, params: unknown) => void): {
|
||||
onEvent?(handler: (source: CdpDebuggee, method: string, params: unknown) => void): {
|
||||
dispose(): void;
|
||||
};
|
||||
dialogCursor?(tabId: number): DialogCursor;
|
||||
@@ -68,6 +71,25 @@ export interface CdpRunner {
|
||||
): ConsoleResult;
|
||||
}
|
||||
|
||||
export function sendToCdpTarget<T = unknown>(
|
||||
cdp: CdpRunner,
|
||||
target: CdpTarget,
|
||||
method: string,
|
||||
params?: object,
|
||||
): Promise<T> {
|
||||
if (target.sessionId && cdp.sendToTarget) {
|
||||
return cdp.sendToTarget<T>(target, method, params);
|
||||
}
|
||||
return cdp.send<T>(target.tabId, method, params);
|
||||
}
|
||||
|
||||
export function cdpRunnerForTarget(cdp: CdpRunner, target: CdpTarget): CdpRunner {
|
||||
return {
|
||||
send: <T = unknown>(_tabId: number, method: string, params?: object) =>
|
||||
sendToCdpTarget<T>(cdp, target, method, params),
|
||||
};
|
||||
}
|
||||
|
||||
export interface BufferedReadBounds {
|
||||
since: number | undefined;
|
||||
limit: number;
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
// Snapshot ref resolution — normalise `@eN` / `eN`, look up
|
||||
// `backendNodeId` via the session RefStore, and build stable
|
||||
// compound CDP node identity via the session RefStore, and build stable
|
||||
// `ref_not_found` errors for hard-failure tool paths.
|
||||
|
||||
import type { SessionContext } from "@/session-manager/manager";
|
||||
@@ -10,6 +10,13 @@ import { rpcError } from "./errors";
|
||||
export interface SnapshotRefLookup {
|
||||
backendNodeId: number;
|
||||
refKey: string;
|
||||
frameId?: string;
|
||||
cdpSessionId?: string;
|
||||
}
|
||||
|
||||
function refEntryForTab(ctx: SessionContext, refKey: string, tabId: number) {
|
||||
const entry = ctx.refStore.resolveEntry(refKey);
|
||||
return entry?.tabId === tabId ? entry : null;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -23,9 +30,14 @@ export function lookupSnapshotRef(
|
||||
tabId: number,
|
||||
): SnapshotRefLookup | null {
|
||||
const refKey = normaliseRef(ref);
|
||||
const backendNodeId = ctx.refStore.resolve(refKey, { tabId });
|
||||
if (backendNodeId === null) return null;
|
||||
return { backendNodeId, refKey };
|
||||
const entry = refEntryForTab(ctx, refKey, tabId);
|
||||
if (!entry) return null;
|
||||
return {
|
||||
backendNodeId: entry.backendNodeId,
|
||||
refKey,
|
||||
...(entry.frameId ? { frameId: entry.frameId } : {}),
|
||||
...(entry.cdpSessionId ? { cdpSessionId: entry.cdpSessionId } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -37,7 +49,7 @@ export function resolveSnapshotRef(
|
||||
ctx: SessionContext,
|
||||
ref: string,
|
||||
tabId: number,
|
||||
): { backendNodeId: number; refKey: string } | RpcError {
|
||||
): SnapshotRefLookup | RpcError {
|
||||
const looked = lookupSnapshotRef(ctx, ref, tabId);
|
||||
if (looked === null) {
|
||||
return rpcError(
|
||||
|
||||
@@ -157,6 +157,7 @@ export interface ChromeWindowsApi {
|
||||
get(windowId: number): Promise<chrome.windows.Window>;
|
||||
getLastFocused(filters?: chrome.windows.QueryOptions): Promise<chrome.windows.Window>;
|
||||
create(props: chrome.windows.CreateData): Promise<chrome.windows.Window | undefined>;
|
||||
remove(windowId: number): Promise<void>;
|
||||
}
|
||||
|
||||
export const chromeWindowsApi: ChromeWindowsApi = {
|
||||
@@ -164,6 +165,7 @@ export const chromeWindowsApi: ChromeWindowsApi = {
|
||||
getLastFocused: (filters) =>
|
||||
filters ? chrome.windows.getLastFocused(filters) : chrome.windows.getLastFocused(),
|
||||
create: (p) => chrome.windows.create(p),
|
||||
remove: (id) => chrome.windows.remove(id),
|
||||
};
|
||||
|
||||
export interface AgentOverlayResetApi {
|
||||
@@ -215,7 +217,9 @@ export async function handleTabList(
|
||||
manager: SessionManager,
|
||||
params: TabListParams,
|
||||
api: ChromeTabsApi = chromeTabsApi,
|
||||
signal?: AbortSignal,
|
||||
): Promise<TabListResult | RpcError> {
|
||||
if (signal?.aborted) return { code: "cancelled", message: "tab_list aborted" };
|
||||
if (!params || typeof params.session_id !== "string" || params.session_id.length === 0) {
|
||||
return {
|
||||
code: "invalid_params",
|
||||
@@ -249,6 +253,7 @@ export async function handleTabList(
|
||||
const myAgentWindowId = ctx.agentWindowId;
|
||||
|
||||
const allTabs = await api.query({});
|
||||
if (signal?.aborted) return { code: "cancelled", message: "tab_list aborted" };
|
||||
const tabs: TabInfo[] = [];
|
||||
for (const t of allTabs) {
|
||||
if (typeof t.id !== "number") continue;
|
||||
@@ -382,7 +387,12 @@ async function createTabAndCleanup(
|
||||
try {
|
||||
await getTabsApi(deps).remove(tab.id);
|
||||
} catch (cleanupErr) {
|
||||
console.debug("[bsk tab_create] cleanup after abort failed", cleanupErr);
|
||||
return rpcError(
|
||||
"protocol_error",
|
||||
"cleanup_failed",
|
||||
`tab_create aborted but cleanup of tab ${tab.id} failed: ${describeError(cleanupErr)}`,
|
||||
{ resource_type: "tab", resource_id: tab.id },
|
||||
);
|
||||
}
|
||||
}
|
||||
return { code: "cancelled", message: "tab_create aborted" };
|
||||
@@ -693,7 +703,17 @@ async function moveTabForBorrow(
|
||||
try {
|
||||
await tabsApi.move(tabId, { windowId: originalWindowId, index: originalIndex });
|
||||
} catch (rollbackErr) {
|
||||
console.debug("[bsk tab_borrow] rollback move failed", rollbackErr);
|
||||
return rpcError(
|
||||
"protocol_error",
|
||||
"cleanup_failed",
|
||||
`tab_borrow aborted but rollback of tab ${tabId} to window ${originalWindowId} failed: ${describeError(rollbackErr)}`,
|
||||
{
|
||||
resource_type: "tab",
|
||||
resource_id: tabId,
|
||||
original_window_id: originalWindowId,
|
||||
original_index: originalIndex,
|
||||
},
|
||||
);
|
||||
}
|
||||
return { code: "cancelled", message: "tab_borrow aborted" };
|
||||
}
|
||||
@@ -825,6 +845,12 @@ export interface ReturnOutcome {
|
||||
fallback: boolean;
|
||||
}
|
||||
|
||||
interface FallbackWindowTarget {
|
||||
windowId: number;
|
||||
index: number;
|
||||
created: boolean;
|
||||
}
|
||||
|
||||
function describeError(err: unknown): string {
|
||||
return err instanceof Error ? err.message : String(err);
|
||||
}
|
||||
@@ -833,10 +859,13 @@ async function chooseFallbackWindow(
|
||||
ctx: SessionContext,
|
||||
windowsApi: ChromeWindowsApi,
|
||||
isAgentWindowId: (windowId: number) => boolean,
|
||||
): Promise<{ windowId: number; index: number } | RpcError> {
|
||||
signal?: AbortSignal,
|
||||
): Promise<FallbackWindowTarget | RpcError> {
|
||||
let lastFocusedError: unknown;
|
||||
try {
|
||||
const last = await windowsApi.getLastFocused({ windowTypes: ["normal"] });
|
||||
const cancelled = aborted(signal, "tab_return");
|
||||
if (cancelled) return cancelled;
|
||||
const lastId = typeof last?.id === "number" ? last.id : null;
|
||||
// Never relocate a user tab into *any* session's Agent Window: this
|
||||
// session's is excluded explicitly, and other sessions' are excluded
|
||||
@@ -846,12 +875,15 @@ async function chooseFallbackWindow(
|
||||
// session write to it and, worse, see it destroyed when that session
|
||||
// stops and closes its window.
|
||||
if (lastId !== null && lastId !== ctx.agentWindowId && !isAgentWindowId(lastId)) {
|
||||
return { windowId: lastId, index: -1 };
|
||||
return { windowId: lastId, index: -1, created: false };
|
||||
}
|
||||
} catch (err) {
|
||||
lastFocusedError = err;
|
||||
}
|
||||
|
||||
const cancelled = aborted(signal, "tab_return");
|
||||
if (cancelled) return cancelled;
|
||||
|
||||
try {
|
||||
const created = await windowsApi.create({
|
||||
url: NEW_TAB_DEFAULT_URL,
|
||||
@@ -863,7 +895,20 @@ async function chooseFallbackWindow(
|
||||
message: "tab_return: failed to create fallback window",
|
||||
};
|
||||
}
|
||||
return { windowId: created.id, index: 0 };
|
||||
if (aborted(signal, "tab_return")) {
|
||||
try {
|
||||
await windowsApi.remove(created.id);
|
||||
} catch (cleanupErr) {
|
||||
return rpcError(
|
||||
"protocol_error",
|
||||
"cleanup_failed",
|
||||
`tab_return aborted but cleanup of fallback window ${created.id} failed: ${describeError(cleanupErr)}`,
|
||||
{ resource_type: "window", resource_id: created.id },
|
||||
);
|
||||
}
|
||||
return { code: "cancelled", message: "tab_return aborted" };
|
||||
}
|
||||
return { windowId: created.id, index: 0, created: true };
|
||||
} catch (err) {
|
||||
const suffix = lastFocusedError
|
||||
? ` (after getLastFocused failed: ${describeError(lastFocusedError)})`
|
||||
@@ -875,6 +920,25 @@ async function chooseFallbackWindow(
|
||||
}
|
||||
}
|
||||
|
||||
async function cleanupUnusedFallbackWindow(
|
||||
windowsApi: ChromeWindowsApi,
|
||||
target: FallbackWindowTarget,
|
||||
reason: string,
|
||||
): Promise<RpcError | null> {
|
||||
if (!target.created) return null;
|
||||
try {
|
||||
await windowsApi.remove(target.windowId);
|
||||
return null;
|
||||
} catch (cleanupErr) {
|
||||
return rpcError(
|
||||
"protocol_error",
|
||||
"cleanup_failed",
|
||||
`${reason}; cleanup of fallback window ${target.windowId} failed: ${describeError(cleanupErr)}`,
|
||||
{ resource_type: "window", resource_id: target.windowId },
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function resetAgentOverlaysInReturnedTab(
|
||||
ctx: SessionContext,
|
||||
tabId: number,
|
||||
@@ -911,10 +975,13 @@ export async function returnBorrowedTab(
|
||||
const tabsApi = getTabsApi(deps);
|
||||
const windowsApi = getWindowsApi(deps);
|
||||
const isAgentWindowId = getIsAgentWindowId(deps);
|
||||
const alreadyCancelled = aborted(deps.signal, "tab_return");
|
||||
if (alreadyCancelled) return alreadyCancelled;
|
||||
|
||||
let targetWindowId = entry.originalWindowId;
|
||||
let targetIndex = entry.originalIndex;
|
||||
let fallback = false;
|
||||
let fallbackTarget: FallbackWindowTarget | null = null;
|
||||
|
||||
// Check the original window is still around.
|
||||
let originalAlive = true;
|
||||
@@ -924,14 +991,30 @@ export async function returnBorrowedTab(
|
||||
console.debug("[bsk tab_return] original window gone, falling back", err);
|
||||
originalAlive = false;
|
||||
}
|
||||
const cancelledAfterLookup = aborted(deps.signal, "tab_return");
|
||||
if (cancelledAfterLookup) return cancelledAfterLookup;
|
||||
if (!originalAlive) {
|
||||
fallback = true;
|
||||
const target = await chooseFallbackWindow(ctx, windowsApi, isAgentWindowId);
|
||||
const target = await chooseFallbackWindow(ctx, windowsApi, isAgentWindowId, deps.signal);
|
||||
if ("code" in target) return target;
|
||||
fallbackTarget = target;
|
||||
targetWindowId = target.windowId;
|
||||
targetIndex = target.index;
|
||||
}
|
||||
|
||||
const cancelledBeforeMove = aborted(deps.signal, "tab_return");
|
||||
if (cancelledBeforeMove) {
|
||||
if (fallbackTarget) {
|
||||
const cleanupError = await cleanupUnusedFallbackWindow(
|
||||
windowsApi,
|
||||
fallbackTarget,
|
||||
"tab_return aborted before moving the borrowed tab",
|
||||
);
|
||||
if (cleanupError) return cleanupError;
|
||||
}
|
||||
return cancelledBeforeMove;
|
||||
}
|
||||
|
||||
try {
|
||||
const moved = await tabsApi.move(tabId, {
|
||||
windowId: targetWindowId,
|
||||
@@ -947,14 +1030,33 @@ export async function returnBorrowedTab(
|
||||
fallback,
|
||||
};
|
||||
} catch (err) {
|
||||
if (fallbackTarget) {
|
||||
const cleanupError = await cleanupUnusedFallbackWindow(
|
||||
windowsApi,
|
||||
fallbackTarget,
|
||||
`tab_return could not move tab ${tabId}`,
|
||||
);
|
||||
if (cleanupError) return cleanupError;
|
||||
}
|
||||
const cancelledAfterMoveFailure = aborted(deps.signal, "tab_return");
|
||||
if (cancelledAfterMoveFailure) return cancelledAfterMoveFailure;
|
||||
if (!fallback) {
|
||||
const target = await chooseFallbackWindow(ctx, windowsApi, isAgentWindowId);
|
||||
const target = await chooseFallbackWindow(ctx, windowsApi, isAgentWindowId, deps.signal);
|
||||
if ("code" in target) {
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: `tab_return: chrome.tabs.move failed: ${describeError(err)}; fallback failed: ${target.message}`,
|
||||
};
|
||||
}
|
||||
const cancelledBeforeFallbackMove = aborted(deps.signal, "tab_return");
|
||||
if (cancelledBeforeFallbackMove) {
|
||||
const cleanupError = await cleanupUnusedFallbackWindow(
|
||||
windowsApi,
|
||||
target,
|
||||
"tab_return aborted before fallback move",
|
||||
);
|
||||
return cleanupError ?? cancelledBeforeFallbackMove;
|
||||
}
|
||||
try {
|
||||
const moved = await tabsApi.move(tabId, {
|
||||
windowId: target.windowId,
|
||||
@@ -970,6 +1072,12 @@ export async function returnBorrowedTab(
|
||||
fallback: true,
|
||||
};
|
||||
} catch (fallbackErr) {
|
||||
const cleanupError = await cleanupUnusedFallbackWindow(
|
||||
windowsApi,
|
||||
target,
|
||||
`tab_return fallback move for tab ${tabId} failed`,
|
||||
);
|
||||
if (cleanupError) return cleanupError;
|
||||
return {
|
||||
code: "cdp_failed",
|
||||
message: `tab_return: chrome.tabs.move failed: ${describeError(err)}; fallback move failed: ${describeError(fallbackErr)}`,
|
||||
|
||||
@@ -18,6 +18,8 @@ function fakeSnapshotReply() {
|
||||
"auto",
|
||||
"type",
|
||||
"password",
|
||||
"value",
|
||||
"hunter2",
|
||||
];
|
||||
const i = (s: string) => S.indexOf(s);
|
||||
return {
|
||||
@@ -29,7 +31,8 @@ function fakeSnapshotReply() {
|
||||
nodeType: [1, 1, 1, 1],
|
||||
nodeName: [i("html"), i("body"), i("div"), i("input")],
|
||||
backendNodeId: [10, 11, 12, 13],
|
||||
attributes: [[], [], [], [i("type"), i("password")]],
|
||||
attributes: [[], [], [], [i("type"), i("password"), i("value"), i("hunter2")]],
|
||||
inputValue: { index: [3], value: [i("hunter2")] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2, 3],
|
||||
@@ -217,7 +220,8 @@ function makeCdp(snapshot: unknown) {
|
||||
{
|
||||
state: "filled",
|
||||
sensitive: true,
|
||||
defaultValue: "",
|
||||
defaultValue: "hunter2",
|
||||
value: "hunter2",
|
||||
placeholder: "secret",
|
||||
},
|
||||
],
|
||||
@@ -262,10 +266,11 @@ describe("captureViewModel", () => {
|
||||
|
||||
expect(input).toMatchObject({
|
||||
formState: "filled",
|
||||
formDefaultValue: "",
|
||||
formPlaceholder: "secret",
|
||||
attrs: { type: "password" },
|
||||
});
|
||||
expect(input?.formValue).toBeUndefined();
|
||||
expect(input?.formDefaultValue).toBeUndefined();
|
||||
expect(cdp.send).toHaveBeenCalledWith(
|
||||
4,
|
||||
"Runtime.evaluate",
|
||||
@@ -692,9 +697,7 @@ describe("captureViewModel", () => {
|
||||
};
|
||||
const { nodes } = await captureViewModel(cdp, 4);
|
||||
const div = nodes.find((n) => n.backendNodeId === 12);
|
||||
// local rect preserves viewport-relative scroll math.
|
||||
expect(div?.localRect?.y).toBe(-200);
|
||||
// exported rect is clipped to the top-level viewport before VOM consumes it.
|
||||
expect(div?.rect).toMatchObject({ y: 0, h: 600 });
|
||||
});
|
||||
|
||||
@@ -839,6 +842,7 @@ describe("captureViewModel", () => {
|
||||
strings: S,
|
||||
documents: [
|
||||
{
|
||||
frameId: "main-frame",
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
@@ -866,6 +870,7 @@ describe("captureViewModel", () => {
|
||||
},
|
||||
// documents[1]: the iframe's sub-document with a login input
|
||||
{
|
||||
frameId: "child-frame",
|
||||
scrollOffsetX: 0,
|
||||
scrollOffsetY: 0,
|
||||
nodes: {
|
||||
@@ -895,7 +900,14 @@ describe("captureViewModel", () => {
|
||||
throw new Error(method);
|
||||
}) as unknown as <T>(tabId: number, method: string, params?: object) => Promise<T>,
|
||||
};
|
||||
const { nodes, iframeNodes } = await captureViewModel(cdp, 4);
|
||||
const {
|
||||
nodes,
|
||||
iframeNodes,
|
||||
frameNodes,
|
||||
frameOwnerBackendNodeIds,
|
||||
frameParentIds,
|
||||
rootFrameId,
|
||||
} = await captureViewModel(cdp, 4);
|
||||
// Main frame has 4 nodes; iframe is included
|
||||
expect(nodes.find((n) => n.backendNodeId === 13)?.tag).toBe("iframe");
|
||||
// iframeNodes keyed by the <iframe> element's backendNodeId=13
|
||||
@@ -908,9 +920,13 @@ describe("captureViewModel", () => {
|
||||
expect(input?.ownerFrameBackendNodeId).toBe(13);
|
||||
expect(input?.localRect).toEqual({ x: 0, y: 0, w: 200, h: 40 });
|
||||
expect(input?.rect).toEqual({ x: 100, y: 300, w: 200, h: 40 });
|
||||
expect(rootFrameId).toBe("main-frame");
|
||||
expect(frameNodes?.get("child-frame")).toBe(subNodes);
|
||||
expect(frameOwnerBackendNodeIds?.get("child-frame")).toBe(13);
|
||||
expect(frameParentIds?.get("child-frame")).toBe("main-frame");
|
||||
});
|
||||
|
||||
it("clips iframe sub-document rects to the iframe viewport in top coordinates", async () => {
|
||||
it("captures iframe documents without requiring owner-frame geometry", async () => {
|
||||
const S = [
|
||||
"html",
|
||||
"body",
|
||||
@@ -938,16 +954,10 @@ describe("captureViewModel", () => {
|
||||
contentDocumentIndex: { index: [2], value: [1] },
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [1, 2],
|
||||
styles: [
|
||||
[i("static"), i("auto")],
|
||||
[i("static"), i("auto")],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 1000, 800],
|
||||
[950, 760, 200, 100],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
nodeIndex: [1],
|
||||
styles: [[i("static"), i("auto")]],
|
||||
bounds: [[0, 0, 1000, 800]],
|
||||
paintOrders: [0],
|
||||
},
|
||||
},
|
||||
{
|
||||
@@ -985,7 +995,7 @@ describe("captureViewModel", () => {
|
||||
const input = iframeNodes.get(13)?.find((n) => n.backendNodeId === 101);
|
||||
|
||||
expect(input?.localRect).toEqual({ x: 20, y: 20, w: 120, h: 60 });
|
||||
expect(input?.rect).toEqual({ x: 970, y: 780, w: 30, h: 20 });
|
||||
expect(input?.rect).toBeNull();
|
||||
});
|
||||
|
||||
it("recursively normalizes nested iframe sub-documents", async () => {
|
||||
@@ -1080,8 +1090,10 @@ describe("captureViewModel", () => {
|
||||
const nestedIframe = iframeNodes.get(13)?.find((n) => n.backendNodeId === 23);
|
||||
const input = iframeNodes.get(23)?.find((n) => n.backendNodeId === 31);
|
||||
|
||||
expect(nestedIframe?.localRect).toEqual({ x: 5, y: 6, w: 100, h: 80 });
|
||||
expect(nestedIframe?.rect).toEqual({ x: 15, y: 26, w: 100, h: 80 });
|
||||
expect(input?.ownerFrameBackendNodeId).toBe(23);
|
||||
expect(input?.localRect).toEqual({ x: 1, y: 2, w: 40, h: 20 });
|
||||
expect(input?.rect).toEqual({ x: 16, y: 28, w: 40, h: 20 });
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { CdpRunner } from "../../shared";
|
||||
import type { CapturedNode, CapturedViewModel } from "../capture";
|
||||
import { captureFrameData } from "../frame-capture";
|
||||
|
||||
function ownerNode(backendNodeId: number, x: number): CapturedNode {
|
||||
return {
|
||||
backendNodeId,
|
||||
parentBackendNodeId: 1,
|
||||
frameId: "main",
|
||||
ownerFrameBackendNodeId: null,
|
||||
tag: "iframe",
|
||||
attrs: {},
|
||||
rect: { x, y: 100, w: 300, h: 200 },
|
||||
localRect: { x, y: 100, w: 300, h: 200 },
|
||||
paintOrder: 1,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
};
|
||||
}
|
||||
|
||||
function childSnapshot(frameId: string, backendNodeId: number) {
|
||||
const strings = [frameId, "body", "button", "static", "auto", "pointer"];
|
||||
return {
|
||||
strings,
|
||||
documents: [
|
||||
{
|
||||
frameId,
|
||||
nodes: {
|
||||
parentIndex: [-1, 0],
|
||||
nodeName: [1, 2],
|
||||
backendNodeId: [backendNodeId - 1, backendNodeId],
|
||||
attributes: [[], []],
|
||||
},
|
||||
layout: {
|
||||
nodeIndex: [0, 1],
|
||||
styles: [
|
||||
[3, 4, 4],
|
||||
[3, 4, 5],
|
||||
],
|
||||
bounds: [
|
||||
[0, 0, 300, 200],
|
||||
[10, 20, 100, 40],
|
||||
],
|
||||
paintOrders: [0, 1],
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
describe("captureFrameData", () => {
|
||||
it("captures and positions multiple OOPIF documents missing from the root snapshot", async () => {
|
||||
const leftOwner = ownerNode(10, 50);
|
||||
const rightOwner = ownerNode(20, 500);
|
||||
const captured: CapturedViewModel = {
|
||||
nodes: [leftOwner, rightOwner],
|
||||
viewport: { width: 1000, height: 800 },
|
||||
iframeNodes: new Map(),
|
||||
frameNodes: new Map([["main", [leftOwner, rightOwner]]]),
|
||||
frameOwnerBackendNodeIds: new Map(),
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
const sendToTarget = vi.fn(async (target, method) => {
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 300, clientHeight: 200, pageX: 0, pageY: 0 } };
|
||||
}
|
||||
if (method === "DOMSnapshot.enable" || method === "Accessibility.enable") return {};
|
||||
if (method === "DOMSnapshot.captureSnapshot") {
|
||||
return target.sessionId === "left-session"
|
||||
? childSnapshot("left", 101)
|
||||
: childSnapshot("right", 201);
|
||||
}
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
throw new Error(`unexpected ${method}`);
|
||||
});
|
||||
const cdp: CdpRunner = {
|
||||
send: vi.fn(async (_tabId, method, params) => {
|
||||
if (method === "Accessibility.enable") return {};
|
||||
if (method === "Accessibility.getFullAXTree") return { nodes: [] };
|
||||
if (method === "DOM.getBoxModel") {
|
||||
const backendNodeId = (params as { backendNodeId?: number })?.backendNodeId;
|
||||
const x = backendNodeId === 10 ? 50 : 500;
|
||||
return { model: { content: [x, 100, x + 300, 100, x + 300, 300, x, 300] } };
|
||||
}
|
||||
if (method === "Page.getLayoutMetrics") {
|
||||
return { cssLayoutViewport: { clientWidth: 1000, clientHeight: 800 } };
|
||||
}
|
||||
throw new Error(`unexpected root ${method}`);
|
||||
}) as CdpRunner["send"],
|
||||
sendToTarget: sendToTarget as unknown as NonNullable<CdpRunner["sendToTarget"]>,
|
||||
getFrameGraph: vi.fn(async () => ({
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "left",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 10,
|
||||
target: { tabId: 4, sessionId: "left-session" },
|
||||
},
|
||||
{
|
||||
frameId: "right",
|
||||
parentFrameId: "main",
|
||||
ownerBackendNodeId: 20,
|
||||
target: { tabId: 4, sessionId: "right-session" },
|
||||
},
|
||||
],
|
||||
})),
|
||||
};
|
||||
|
||||
const trees = await captureFrameData(cdp, 4, captured);
|
||||
|
||||
expect(trees.map((tree) => tree.frameId)).toEqual(["main", "left", "right"]);
|
||||
expect(
|
||||
captured.frameNodes?.get("left")?.find((node) => node.backendNodeId === 101)?.rect,
|
||||
).toEqual({ x: 60, y: 120, w: 100, h: 40 });
|
||||
expect(
|
||||
captured.frameNodes?.get("right")?.find((node) => node.backendNodeId === 201)?.rect,
|
||||
).toEqual({ x: 510, y: 120, w: 100, h: 40 });
|
||||
expect(captured.frameOwnerBackendNodeIds).toEqual(
|
||||
new Map([
|
||||
["left", 10],
|
||||
["right", 20],
|
||||
]),
|
||||
);
|
||||
expect(captured.frameParentIds).toEqual(
|
||||
new Map([
|
||||
["left", "main"],
|
||||
["right", "main"],
|
||||
]),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,174 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||
import type { CapturedNode, CapturedViewModel } from "../capture";
|
||||
import { buildFrameDocuments, type FrameAxBatch, type FrameOwnedAxNode } from "../frame-document";
|
||||
|
||||
function domNode(frameId: string, backendNodeId: number): CapturedNode {
|
||||
return {
|
||||
backendNodeId,
|
||||
parentBackendNodeId: null,
|
||||
frameId,
|
||||
tag: "div",
|
||||
attrs: {},
|
||||
rect: { x: 0, y: 0, w: 20, h: 20 },
|
||||
paintOrder: 0,
|
||||
position: "static",
|
||||
pointerEvents: "auto",
|
||||
};
|
||||
}
|
||||
|
||||
function captured(frameNodes: Map<string, CapturedNode[]>): CapturedViewModel {
|
||||
return {
|
||||
nodes: frameNodes.get("main") ?? [],
|
||||
viewport: { width: 800, height: 600 },
|
||||
iframeNodes: new Map(),
|
||||
frameNodes,
|
||||
rootFrameId: "main",
|
||||
excludedBackendNodeIds: new Set(),
|
||||
};
|
||||
}
|
||||
|
||||
describe("buildFrameDocuments", () => {
|
||||
it("partitions mixed AX results by node ownership and cuts cross-frame parent edges", () => {
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{ frameId: "child", parentFrameId: "main", target: { tabId: 4 } },
|
||||
],
|
||||
};
|
||||
const batches: FrameAxBatch<FrameOwnedAxNode>[] = [
|
||||
{
|
||||
frame: graph.frames[0],
|
||||
nodes: [
|
||||
{ nodeId: "root", frameId: "main", backendDOMNodeId: 1, childIds: ["h"] },
|
||||
{
|
||||
nodeId: "h",
|
||||
frameId: "child",
|
||||
parentId: "root",
|
||||
backendDOMNodeId: 100,
|
||||
childIds: ["virtual"],
|
||||
},
|
||||
{ nodeId: "virtual", parentId: "h" },
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const documents = buildFrameDocuments(
|
||||
graph,
|
||||
batches,
|
||||
captured(
|
||||
new Map([
|
||||
["main", [domNode("main", 1)]],
|
||||
["child", [domNode("child", 100)]],
|
||||
]),
|
||||
),
|
||||
);
|
||||
|
||||
expect(documents.find((document) => document.frameId === "main")?.axNodes).toHaveLength(1);
|
||||
const childNodes = documents.find((document) => document.frameId === "child")?.axNodes ?? [];
|
||||
expect(childNodes.map((node) => node.nodeId)).toEqual(["h", "virtual"]);
|
||||
expect(childNodes.find((node) => node.nodeId === "h")?.parentId).toBeUndefined();
|
||||
expect(childNodes.find((node) => node.nodeId === "virtual")?.frameId).toBe("child");
|
||||
});
|
||||
|
||||
it("deduplicates overlapping same-target AX results using the strongest ownership", () => {
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{ frameId: "child", parentFrameId: "main", target: { tabId: 4 } },
|
||||
],
|
||||
};
|
||||
const childNode = { nodeId: "shared", backendDOMNodeId: 100 };
|
||||
const batches: FrameAxBatch<FrameOwnedAxNode>[] = [
|
||||
{
|
||||
frame: graph.frames[0],
|
||||
nodes: [{ ...childNode, frameId: "child" }],
|
||||
},
|
||||
{
|
||||
frame: graph.frames[1],
|
||||
nodes: [childNode],
|
||||
},
|
||||
];
|
||||
|
||||
const documents = buildFrameDocuments(
|
||||
graph,
|
||||
batches,
|
||||
captured(
|
||||
new Map([
|
||||
["main", [domNode("main", 1)]],
|
||||
["child", [domNode("child", 100)]],
|
||||
]),
|
||||
),
|
||||
);
|
||||
|
||||
expect(documents.find((document) => document.frameId === "main")?.axNodes).toEqual([]);
|
||||
expect(documents.find((document) => document.frameId === "child")?.axNodes).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("uses target-scoped backend ownership when OOPIF backend ids collide", () => {
|
||||
const graph: CdpFrameGraph = {
|
||||
rootFrameId: "main",
|
||||
frames: [
|
||||
{ frameId: "main", target: { tabId: 4 } },
|
||||
{
|
||||
frameId: "left",
|
||||
parentFrameId: "main",
|
||||
target: { tabId: 4, sessionId: "left-session" },
|
||||
},
|
||||
{
|
||||
frameId: "right",
|
||||
parentFrameId: "main",
|
||||
target: { tabId: 4, sessionId: "right-session" },
|
||||
},
|
||||
],
|
||||
};
|
||||
const batches: FrameAxBatch<FrameOwnedAxNode>[] = graph.frames.slice(1).map((frame) => ({
|
||||
frame,
|
||||
nodes: [{ nodeId: `${frame.frameId}-node`, backendDOMNodeId: 7 }],
|
||||
}));
|
||||
|
||||
const documents = buildFrameDocuments(
|
||||
graph,
|
||||
batches,
|
||||
captured(
|
||||
new Map([
|
||||
["main", []],
|
||||
["left", [domNode("left", 7)]],
|
||||
["right", [domNode("right", 7)]],
|
||||
]),
|
||||
),
|
||||
);
|
||||
|
||||
expect(documents.find((document) => document.frameId === "left")?.axNodes[0]?.nodeId).toBe(
|
||||
"left-node",
|
||||
);
|
||||
expect(documents.find((document) => document.frameId === "right")?.axNodes[0]?.nodeId).toBe(
|
||||
"right-node",
|
||||
);
|
||||
});
|
||||
|
||||
it("uses captured frame parent identity instead of guessing from backend ids", () => {
|
||||
const frameNodes = new Map([
|
||||
["main", [domNode("main", 1)]],
|
||||
["left", [domNode("left", 7)]],
|
||||
["right", [domNode("right", 7)]],
|
||||
["child", [domNode("child", 9)]],
|
||||
]);
|
||||
const capture = captured(frameNodes);
|
||||
capture.frameOwnerBackendNodeIds = new Map([["child", 7]]);
|
||||
capture.frameParentIds = new Map([["child", "left"]]);
|
||||
const frames = ["main", "left", "right", "child"].map((frameId) => ({
|
||||
frame: { frameId, target: { tabId: 4 } },
|
||||
nodes: [],
|
||||
}));
|
||||
|
||||
const documents = buildFrameDocuments(null, frames, capture);
|
||||
|
||||
expect(documents.find((document) => document.frameId === "child")).toMatchObject({
|
||||
parentFrameId: "left",
|
||||
ownerBackendNodeId: 7,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -7,6 +7,7 @@
|
||||
import type { Rect, Viewport } from "@browser-skill/vom";
|
||||
import { evaluateHoverTrigger } from "@/lib/hover-trigger-policy";
|
||||
import { isOverlayHostNode, OVERLAY_HOST_SELECTOR } from "../../lib/overlay-bridge";
|
||||
import { childFrameProjection, type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import type { CdpRunner } from "../shared";
|
||||
|
||||
const REQUESTED_STYLES = ["position", "pointer-events", "cursor"] as const;
|
||||
@@ -17,13 +18,14 @@ const STYLE_COL = Object.fromEntries(
|
||||
export interface CapturedNode {
|
||||
backendNodeId: number;
|
||||
parentBackendNodeId: number | null;
|
||||
frameId?: string;
|
||||
/** Owning iframe backend node id; `null` for the top-level document. */
|
||||
ownerFrameBackendNodeId?: number | null;
|
||||
tag: string;
|
||||
attrs: Record<string, string>;
|
||||
/** Top-level viewport-relative CSS px, clipped to the owning frame viewport. */
|
||||
rect: Rect | null;
|
||||
/** Frame-local viewport-relative CSS px before top-level translation/clipping. */
|
||||
/** Frame-local viewport-relative CSS px before top-level projection. */
|
||||
localRect?: Rect | null;
|
||||
paintOrder: number;
|
||||
position: string;
|
||||
@@ -57,6 +59,11 @@ export interface CapturedViewModel {
|
||||
nodes: CapturedNode[];
|
||||
viewport: Viewport;
|
||||
iframeNodes: CapturedIframeNodes;
|
||||
frameNodes?: Map<string, CapturedNode[]>;
|
||||
frameOwnerBackendNodeIds?: Map<string, number>;
|
||||
/** Explicit DOMSnapshot frame ancestry; never inferred from backend node ids. */
|
||||
frameParentIds?: Map<string, string>;
|
||||
rootFrameId?: string;
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
/** Backend node ids belonging to the agent overlay host + its shadow subtree. */
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
@@ -65,6 +72,25 @@ export interface CapturedViewModel {
|
||||
export interface CaptureViewModelOptions {
|
||||
conditionalSurfaceProbe?: boolean;
|
||||
hoverProbeBypassOverlay?: (tabId: number, enabled: boolean) => Promise<void>;
|
||||
signal?: AbortSignal;
|
||||
}
|
||||
|
||||
function captureAbortError(): Error {
|
||||
const error = new Error("observation aborted");
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
function isAbortError(error: unknown): boolean {
|
||||
return (
|
||||
typeof error === "object" &&
|
||||
error !== null &&
|
||||
(error as { name?: string }).name === "AbortError"
|
||||
);
|
||||
}
|
||||
|
||||
function throwIfAborted(signal: AbortSignal | undefined): void {
|
||||
if (signal?.aborted) throw captureAbortError();
|
||||
}
|
||||
|
||||
/** Sparse array format Chrome uses for infrequently-set per-node fields. */
|
||||
@@ -73,7 +99,12 @@ interface SparseArray {
|
||||
value: number[];
|
||||
}
|
||||
|
||||
interface RareBooleanData {
|
||||
index: number[];
|
||||
}
|
||||
|
||||
interface SnapshotDocument {
|
||||
frameId?: string | number;
|
||||
scrollOffsetX?: number;
|
||||
scrollOffsetY?: number;
|
||||
nodes?: {
|
||||
@@ -88,6 +119,10 @@ interface SnapshotDocument {
|
||||
nodeValue?: number[];
|
||||
/** Maps node array index → index into `documents[]` for frame content. */
|
||||
contentDocumentIndex?: SparseArray;
|
||||
inputValue?: SparseArray;
|
||||
textValue?: SparseArray;
|
||||
inputChecked?: RareBooleanData;
|
||||
optionSelected?: RareBooleanData;
|
||||
};
|
||||
layout?: {
|
||||
nodeIndex?: number[];
|
||||
@@ -97,6 +132,12 @@ interface SnapshotDocument {
|
||||
};
|
||||
}
|
||||
|
||||
function snapshotFrameId(document: SnapshotDocument, strings: string[]): string | undefined {
|
||||
if (typeof document.frameId === "string") return document.frameId || undefined;
|
||||
if (typeof document.frameId === "number") return str(strings, document.frameId) || undefined;
|
||||
return undefined;
|
||||
}
|
||||
|
||||
interface SnapshotReply {
|
||||
strings?: string[];
|
||||
documents?: SnapshotDocument[];
|
||||
@@ -116,6 +157,25 @@ function isFormControlTag(tag: string): boolean {
|
||||
return tag === "input" || tag === "textarea" || tag === "select";
|
||||
}
|
||||
|
||||
function isSensitiveFormControl(node: Pick<CapturedNode, "tag" | "attrs">): boolean {
|
||||
return node.tag === "input" && (node.attrs.type ?? "").toLowerCase() === "password";
|
||||
}
|
||||
|
||||
function snapshotFormState(
|
||||
value: string | undefined,
|
||||
defaultValue: string,
|
||||
hasDefaultValue: boolean,
|
||||
sensitive: boolean,
|
||||
): CapturedNode["formState"] {
|
||||
if (sensitive) {
|
||||
if (value === undefined && !hasDefaultValue) return undefined;
|
||||
return (value ?? defaultValue) === "" ? "empty" : "filled";
|
||||
}
|
||||
if (value === undefined) return undefined;
|
||||
if (value === "") return "empty";
|
||||
return value === defaultValue ? "default" : "filled";
|
||||
}
|
||||
|
||||
const MAX_FORM_ENRICH_CONTROLS = 250;
|
||||
|
||||
interface CapturedFormState {
|
||||
@@ -147,9 +207,8 @@ function formStateBatchExpression(maxControls: number): string {
|
||||
return {
|
||||
state,
|
||||
sensitive,
|
||||
defaultValue,
|
||||
placeholder,
|
||||
...(sensitive ? {} : { value: rawValue }),
|
||||
...(sensitive ? {} : { value: rawValue, defaultValue }),
|
||||
};
|
||||
};
|
||||
const collect = (doc) => {
|
||||
@@ -191,10 +250,14 @@ function applyFormStates(nodes: CapturedNode[], states: CapturedFormState[]): vo
|
||||
index += 1;
|
||||
if (!state) continue;
|
||||
node.formState = state.state;
|
||||
node.formDefaultValue = state.defaultValue ?? "";
|
||||
node.formPlaceholder = state.placeholder ?? "";
|
||||
if (!state.sensitive && state.value !== undefined) {
|
||||
node.formValue = state.value;
|
||||
if (state.sensitive || isSensitiveFormControl(node)) {
|
||||
delete node.formValue;
|
||||
delete node.formDefaultValue;
|
||||
delete node.attrs.value;
|
||||
} else {
|
||||
node.formDefaultValue = state.defaultValue ?? "";
|
||||
if (state.value !== undefined) node.formValue = state.value;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -203,21 +266,25 @@ async function enrichFormControlStates(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
frameNodeGroups: CapturedNode[][],
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
const hasControls = frameNodeGroups.some((nodes) =>
|
||||
nodes.some((node) => isFormControlTag(node.tag)),
|
||||
);
|
||||
if (!hasControls) return;
|
||||
try {
|
||||
throwIfAborted(signal);
|
||||
const result = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: formStateBatchExpression(MAX_FORM_ENRICH_CONTROLS),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(signal);
|
||||
const frameStates = flattenFrameFormStates(runtimeValue<CapturedFrameFormState>(result));
|
||||
for (let i = 0; i < frameNodeGroups.length; i += 1) {
|
||||
applyFormStates(frameNodeGroups[i], frameStates[i] ?? []);
|
||||
}
|
||||
} catch {
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
// Best-effort enrichment. DOMSnapshot/AX data still carries the nodes.
|
||||
}
|
||||
}
|
||||
@@ -249,9 +316,9 @@ interface ParseDocumentResult {
|
||||
}
|
||||
|
||||
interface FrameContext {
|
||||
frameId?: string;
|
||||
ownerFrameBackendNodeId: number | null;
|
||||
originInTop: { x: number; y: number };
|
||||
clipRectInTop: Rect;
|
||||
projection: GeometryProjection | null;
|
||||
scrollX: number;
|
||||
scrollY: number;
|
||||
}
|
||||
@@ -270,38 +337,6 @@ function devicePixelRatio(metrics: LayoutMetricsReply): number {
|
||||
return dpr >= 1 ? dpr : 1;
|
||||
}
|
||||
|
||||
function viewportRect(viewport: Viewport): Rect {
|
||||
return { x: 0, y: 0, w: Math.max(0, viewport.width), h: Math.max(0, viewport.height) };
|
||||
}
|
||||
|
||||
function intersectRects(a: Rect, b: Rect): Rect | null {
|
||||
const x1 = Math.max(a.x, b.x);
|
||||
const y1 = Math.max(a.y, b.y);
|
||||
const x2 = Math.min(a.x + a.w, b.x + b.w);
|
||||
const y2 = Math.min(a.y + a.h, b.y + b.h);
|
||||
if (x2 <= x1 || y2 <= y1) return null;
|
||||
return { x: x1, y: y1, w: x2 - x1, h: y2 - y1 };
|
||||
}
|
||||
|
||||
function rectInTop(localRect: Rect, context: FrameContext): Rect | null {
|
||||
const translated = {
|
||||
x: context.originInTop.x + localRect.x,
|
||||
y: context.originInTop.y + localRect.y,
|
||||
w: localRect.w,
|
||||
h: localRect.h,
|
||||
};
|
||||
return intersectRects(translated, context.clipRectInTop);
|
||||
}
|
||||
|
||||
function unclipRectInTop(localRect: Rect, context: FrameContext): Rect {
|
||||
return {
|
||||
x: context.originInTop.x + localRect.x,
|
||||
y: context.originInTop.y + localRect.y,
|
||||
w: localRect.w,
|
||||
h: localRect.h,
|
||||
};
|
||||
}
|
||||
|
||||
function sparseIndexMap(sparse: SparseArray | undefined): Map<number, number> {
|
||||
const out = new Map<number, number>();
|
||||
if (!sparse?.index || !sparse.value) return out;
|
||||
@@ -425,8 +460,20 @@ function hoverStateExpression(): string {
|
||||
})()`;
|
||||
}
|
||||
|
||||
async function wait(ms: number): Promise<void> {
|
||||
await new Promise((resolve) => setTimeout(resolve, ms));
|
||||
async function wait(ms: number, signal?: AbortSignal): Promise<void> {
|
||||
throwIfAborted(signal);
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
resolve();
|
||||
}, ms);
|
||||
const onAbort = () => {
|
||||
clearTimeout(timer);
|
||||
signal?.removeEventListener("abort", onAbort);
|
||||
reject(captureAbortError());
|
||||
};
|
||||
signal?.addEventListener("abort", onAbort, { once: true });
|
||||
});
|
||||
}
|
||||
|
||||
async function clearHover(cdp: CdpRunner, tabId: number): Promise<void> {
|
||||
@@ -598,41 +645,51 @@ async function probeHoverSurfaces(
|
||||
): Promise<CapturedSurfaceProbe[]> {
|
||||
const started = Date.now();
|
||||
try {
|
||||
throwIfAborted(options.signal);
|
||||
const cssScan = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverCssTriggerScanExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
const cssHoverPoints = runtimeValue<Array<{ x: number; y: number }>>(cssScan) ?? [];
|
||||
const candidates = buildHoverCandidates(nodes, cssHoverPoints);
|
||||
if (candidates.length === 0) return [];
|
||||
|
||||
const results: CapturedSurfaceProbe[] = [];
|
||||
const seen = new Set<number>();
|
||||
throwIfAborted(options.signal);
|
||||
await options.hoverProbeBypassOverlay?.(tabId, true).catch(() => undefined);
|
||||
try {
|
||||
throwIfAborted(options.signal);
|
||||
for (const candidate of candidates.slice(0, MAX_HOVER_TRIGGERS)) {
|
||||
throwIfAborted(options.signal);
|
||||
if (Date.now() - started > MAX_HOVER_PROBE_MS) break;
|
||||
if (results.length >= MAX_HOVER_SURFACES) break;
|
||||
if (seen.has(candidate.backendNodeId)) continue;
|
||||
try {
|
||||
await clearHover(cdp, tabId);
|
||||
await wait(HOVER_SETTLE_MS);
|
||||
throwIfAborted(options.signal);
|
||||
await wait(HOVER_SETTLE_MS, options.signal);
|
||||
const baselineReply = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
const baselineItems = runtimeValue<HoverRuntimeItem[]>(baselineReply) ?? [];
|
||||
|
||||
throwIfAborted(options.signal);
|
||||
await cdp.send(tabId, "Input.dispatchMouseEvent", {
|
||||
type: "mouseMoved",
|
||||
x: candidate.x,
|
||||
y: candidate.y,
|
||||
});
|
||||
await wait(HOVER_SETTLE_MS);
|
||||
throwIfAborted(options.signal);
|
||||
await wait(HOVER_SETTLE_MS, options.signal);
|
||||
const collected = await cdp.send<RuntimeEvaluateReply>(tabId, "Runtime.evaluate", {
|
||||
expression: hoverStateExpression(),
|
||||
returnByValue: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
const subItems = diffHoverItems(
|
||||
baselineItems,
|
||||
runtimeValue<HoverRuntimeItem[]>(collected) ?? [],
|
||||
@@ -646,7 +703,8 @@ async function probeHoverSurfaces(
|
||||
subItems,
|
||||
confidence: confidenceForHover(candidate, subItems),
|
||||
});
|
||||
} catch {
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
continue;
|
||||
} finally {
|
||||
await clearHover(cdp, tabId);
|
||||
@@ -657,6 +715,7 @@ async function probeHoverSurfaces(
|
||||
}
|
||||
return results;
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
console.debug("[bsk capture] hover surface probe failed", err);
|
||||
return [];
|
||||
}
|
||||
@@ -669,13 +728,16 @@ async function probeHoverSurfaces(
|
||||
export async function collectOverlayExcludedBackendIds(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
signal?: AbortSignal,
|
||||
): Promise<Set<number>> {
|
||||
const excluded = new Set<number>();
|
||||
try {
|
||||
throwIfAborted(signal);
|
||||
const doc = await cdp.send<{ root?: { nodeId?: number } }>(tabId, "DOM.getDocument", {
|
||||
depth: 0,
|
||||
pierce: true,
|
||||
});
|
||||
throwIfAborted(signal);
|
||||
const rootNodeId = doc.root?.nodeId;
|
||||
if (typeof rootNodeId !== "number") return excluded;
|
||||
|
||||
@@ -683,6 +745,7 @@ export async function collectOverlayExcludedBackendIds(
|
||||
nodeId: rootNodeId,
|
||||
selector: OVERLAY_HOST_SELECTOR,
|
||||
});
|
||||
throwIfAborted(signal);
|
||||
if (typeof found.nodeId !== "number" || found.nodeId === 0) return excluded;
|
||||
|
||||
const described = await cdp.send<{ node?: CdpDomNode }>(tabId, "DOM.describeNode", {
|
||||
@@ -690,8 +753,10 @@ export async function collectOverlayExcludedBackendIds(
|
||||
depth: -1,
|
||||
pierce: true,
|
||||
});
|
||||
throwIfAborted(signal);
|
||||
collectBackendIdsFromDomNode(described.node, excluded);
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
console.debug("[bsk capture] overlay exclusion fallback failed", err);
|
||||
}
|
||||
return excluded;
|
||||
@@ -752,6 +817,10 @@ function parseDocumentNodes(
|
||||
const posCol = STYLE_COL.position;
|
||||
const peCol = STYLE_COL["pointer-events"];
|
||||
const cursorCol = STYLE_COL.cursor;
|
||||
const inputValues = sparseIndexMap(dn.inputValue);
|
||||
const textValues = sparseIndexMap(dn.textValue);
|
||||
const checkedInputs = new Set(dn.inputChecked?.index ?? []);
|
||||
const selectedOptions = new Set(dn.optionSelected?.index ?? []);
|
||||
|
||||
// Collect visible text from #text child nodes so element CapturedNodes
|
||||
// carry a textContent value usable as a button/link label fallback.
|
||||
@@ -802,7 +871,10 @@ function parseDocumentNodes(
|
||||
w: b[2] / dpr,
|
||||
h: b[3] / dpr,
|
||||
};
|
||||
rect = rectInTop(localRect, context);
|
||||
const bounds = context.projection
|
||||
? projectRectToViewport(localRect, context.projection)
|
||||
: null;
|
||||
rect = bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||
}
|
||||
paintOrder = dl?.paintOrders?.[li] ?? 0;
|
||||
const styleRow = dl?.styles?.[li] ?? [];
|
||||
@@ -817,9 +889,24 @@ function parseDocumentNodes(
|
||||
|
||||
const textContent = nodeTextContent.get(n);
|
||||
|
||||
const rawFormValueIndex = tag === "textarea" ? textValues.get(n) : inputValues.get(n);
|
||||
const rawFormValue =
|
||||
rawFormValueIndex !== undefined ? str(strings, rawFormValueIndex) : undefined;
|
||||
const sensitive = isSensitiveFormControl({ tag, attrs });
|
||||
const formDefaultValue = attrs.value ?? "";
|
||||
const formValue = sensitive ? undefined : rawFormValue;
|
||||
const formState = snapshotFormState(
|
||||
rawFormValue,
|
||||
formDefaultValue,
|
||||
Object.prototype.hasOwnProperty.call(attrs, "value"),
|
||||
sensitive,
|
||||
);
|
||||
if (sensitive) delete attrs.value;
|
||||
|
||||
nodes.push({
|
||||
backendNodeId,
|
||||
parentBackendNodeId,
|
||||
...(context.frameId ? { frameId: context.frameId } : {}),
|
||||
ownerFrameBackendNodeId: context.ownerFrameBackendNodeId,
|
||||
tag,
|
||||
attrs,
|
||||
@@ -830,6 +917,16 @@ function parseDocumentNodes(
|
||||
pointerEvents,
|
||||
cursor,
|
||||
textContent,
|
||||
...(formValue !== undefined ? { formValue } : {}),
|
||||
...(tag === "input" || tag === "textarea"
|
||||
? {
|
||||
formPlaceholder: attrs.placeholder ?? "",
|
||||
...(!sensitive ? { formDefaultValue } : {}),
|
||||
...(formState ? { formState } : {}),
|
||||
}
|
||||
: {}),
|
||||
...(checkedInputs.has(n) ? { formValue: "true", formState: "filled" } : {}),
|
||||
...(selectedOptions.has(n) ? { formValue: attrs.value ?? textContent ?? "" } : {}),
|
||||
});
|
||||
}
|
||||
return { nodes, excludedBackendNodeIds };
|
||||
@@ -837,6 +934,8 @@ function parseDocumentNodes(
|
||||
|
||||
interface ParseFrameDocumentsResult {
|
||||
iframeNodes: CapturedIframeNodes;
|
||||
frameOwnerBackendNodeIds: Map<string, number>;
|
||||
frameParentIds: Map<string, string>;
|
||||
excludedBackendNodeIds: Set<number>;
|
||||
}
|
||||
|
||||
@@ -850,10 +949,14 @@ function parseChildFrameDocuments(
|
||||
visited = new Set<number>(),
|
||||
): ParseFrameDocumentsResult {
|
||||
const iframeNodes: CapturedIframeNodes = new Map();
|
||||
const frameOwnerBackendNodeIds = new Map<string, number>();
|
||||
const frameParentIds = new Map<string, string>();
|
||||
const excludedBackendNodeIds = new Set<number>();
|
||||
const parentDoc = documents[parentDocIndex];
|
||||
const cdi = sparseIndexMap(parentDoc?.nodes?.contentDocumentIndex);
|
||||
if (cdi.size === 0) return { iframeNodes, excludedBackendNodeIds };
|
||||
if (cdi.size === 0) {
|
||||
return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
const parentBackendIds = parentDoc?.nodes?.backendNodeId ?? [];
|
||||
const parentNodeByBackendId = new Map(parentNodes.map((node) => [node.backendNodeId, node]));
|
||||
@@ -865,19 +968,15 @@ function parseChildFrameDocuments(
|
||||
const iframeBackendId = parentBackendIds[nodeArrayIdx];
|
||||
if (iframeBackendId === undefined) continue;
|
||||
const iframeNode = parentNodeByBackendId.get(iframeBackendId);
|
||||
if (!iframeNode?.localRect) continue;
|
||||
|
||||
const iframeRectInTop = unclipRectInTop(iframeNode.localRect, parentContext);
|
||||
const childClip = intersectRects(iframeRectInTop, parentContext.clipRectInTop);
|
||||
if (!childClip) {
|
||||
iframeNodes.set(iframeBackendId, []);
|
||||
continue;
|
||||
}
|
||||
const projection =
|
||||
parentContext.projection && iframeNode?.localRect
|
||||
? childFrameProjection(parentContext.projection, iframeNode.localRect)
|
||||
: null;
|
||||
|
||||
const childContext: FrameContext = {
|
||||
frameId: snapshotFrameId(childDoc, strings),
|
||||
ownerFrameBackendNodeId: iframeBackendId,
|
||||
originInTop: { x: iframeRectInTop.x, y: iframeRectInTop.y },
|
||||
clipRectInTop: childClip,
|
||||
projection,
|
||||
scrollX: childDoc.scrollOffsetX ?? 0,
|
||||
scrollY: childDoc.scrollOffsetY ?? 0,
|
||||
};
|
||||
@@ -885,6 +984,10 @@ function parseChildFrameDocuments(
|
||||
nextVisited.add(childDocIndex);
|
||||
const parsed = parseDocumentNodes(childDoc, strings, dpr, childContext);
|
||||
iframeNodes.set(iframeBackendId, parsed.nodes);
|
||||
if (childContext.frameId) {
|
||||
frameOwnerBackendNodeIds.set(childContext.frameId, iframeBackendId);
|
||||
if (parentContext.frameId) frameParentIds.set(childContext.frameId, parentContext.frameId);
|
||||
}
|
||||
for (const id of parsed.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||
|
||||
const nested = parseChildFrameDocuments(
|
||||
@@ -899,10 +1002,16 @@ function parseChildFrameDocuments(
|
||||
for (const [nestedFrameId, nestedNodes] of nested.iframeNodes) {
|
||||
iframeNodes.set(nestedFrameId, nestedNodes);
|
||||
}
|
||||
for (const [frameId, ownerBackendNodeId] of nested.frameOwnerBackendNodeIds) {
|
||||
frameOwnerBackendNodeIds.set(frameId, ownerBackendNodeId);
|
||||
}
|
||||
for (const [frameId, parentFrameId] of nested.frameParentIds) {
|
||||
frameParentIds.set(frameId, parentFrameId);
|
||||
}
|
||||
for (const id of nested.excludedBackendNodeIds) excludedBackendNodeIds.add(id);
|
||||
}
|
||||
|
||||
return { iframeNodes, excludedBackendNodeIds };
|
||||
return { iframeNodes, frameOwnerBackendNodeIds, frameParentIds, excludedBackendNodeIds };
|
||||
}
|
||||
|
||||
export async function captureViewModel(
|
||||
@@ -910,7 +1019,9 @@ export async function captureViewModel(
|
||||
tabId: number,
|
||||
options: CaptureViewModelOptions = {},
|
||||
): Promise<CapturedViewModel> {
|
||||
throwIfAborted(options.signal);
|
||||
const metrics = await cdp.send<LayoutMetricsReply>(tabId, "Page.getLayoutMetrics", {});
|
||||
throwIfAborted(options.signal);
|
||||
const dpr = devicePixelRatio(metrics);
|
||||
const vpSrc = metrics.cssLayoutViewport ?? metrics.layoutViewport ?? {};
|
||||
const viewport: Viewport = {
|
||||
@@ -921,11 +1032,13 @@ export async function captureViewModel(
|
||||
const scrollY = vpSrc.pageY ?? 0;
|
||||
|
||||
await cdp.send(tabId, "DOMSnapshot.enable", {});
|
||||
throwIfAborted(options.signal);
|
||||
const snap = await cdp.send<SnapshotReply>(tabId, "DOMSnapshot.captureSnapshot", {
|
||||
computedStyles: REQUESTED_STYLES,
|
||||
includePaintOrder: true,
|
||||
includeDOMRects: true,
|
||||
});
|
||||
throwIfAborted(options.signal);
|
||||
|
||||
const strings = snap.strings ?? [];
|
||||
const documents = snap.documents ?? [];
|
||||
@@ -941,9 +1054,13 @@ export async function captureViewModel(
|
||||
}
|
||||
|
||||
const topContext: FrameContext = {
|
||||
frameId: snapshotFrameId(doc0, strings),
|
||||
ownerFrameBackendNodeId: null,
|
||||
originInTop: { x: 0, y: 0 },
|
||||
clipRectInTop: viewportRect(viewport),
|
||||
projection: {
|
||||
sourceClips: [],
|
||||
edges: [],
|
||||
topViewport: viewport,
|
||||
},
|
||||
scrollX,
|
||||
scrollY,
|
||||
};
|
||||
@@ -957,11 +1074,35 @@ export async function captureViewModel(
|
||||
excludedBackendNodeIds.add(id);
|
||||
}
|
||||
|
||||
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()]);
|
||||
await enrichFormControlStates(cdp, tabId, [nodes, ...iframeNodes.values()], options.signal);
|
||||
throwIfAborted(options.signal);
|
||||
|
||||
const surfaceProbes = options.conditionalSurfaceProbe
|
||||
? await probeHoverSurfaces(cdp, tabId, nodes, options)
|
||||
: [];
|
||||
throwIfAborted(options.signal);
|
||||
|
||||
return { nodes, viewport, iframeNodes, surfaceProbes, excludedBackendNodeIds };
|
||||
const frameNodes = new Map<string, CapturedNode[]>();
|
||||
if (topContext.frameId) frameNodes.set(topContext.frameId, nodes);
|
||||
for (const iframeFrameNodes of iframeNodes.values()) {
|
||||
const frameId = iframeFrameNodes.find((node) => node.frameId)?.frameId;
|
||||
if (frameId) frameNodes.set(frameId, iframeFrameNodes);
|
||||
}
|
||||
for (const [frameId, ownerBackendNodeId] of frameParsed.frameOwnerBackendNodeIds) {
|
||||
if (!frameNodes.has(frameId)) {
|
||||
frameNodes.set(frameId, iframeNodes.get(ownerBackendNodeId) ?? []);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
nodes,
|
||||
viewport,
|
||||
iframeNodes,
|
||||
frameNodes,
|
||||
frameOwnerBackendNodeIds: frameParsed.frameOwnerBackendNodeIds,
|
||||
frameParentIds: frameParsed.frameParentIds,
|
||||
...(topContext.frameId ? { rootFrameId: topContext.frameId } : {}),
|
||||
surfaceProbes,
|
||||
excludedBackendNodeIds,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
import type { CdpFrame, CdpFrameGraph } from "@/browser-driver/frame-graph";
|
||||
import { resolveFrameProjection } from "../frame-geometry";
|
||||
import { type GeometryProjection, projectRectToViewport } from "../geometry";
|
||||
import { type CdpRunner, cdpRunnerForTarget, sendToCdpTarget } from "../shared";
|
||||
import { type CapturedNode, type CapturedViewModel, captureViewModel } from "./capture";
|
||||
import {
|
||||
buildFrameDocuments,
|
||||
type FrameAxBatch,
|
||||
type FrameDocument,
|
||||
type FrameOwnedAxNode,
|
||||
} from "./frame-document";
|
||||
|
||||
export type FrameAxNode = FrameOwnedAxNode;
|
||||
export type CapturedFrameDocument<T extends FrameAxNode> = FrameDocument<T>;
|
||||
|
||||
function throwIfAborted(signal?: AbortSignal): void {
|
||||
if (signal?.aborted) throw new DOMException("observation aborted", "AbortError");
|
||||
}
|
||||
|
||||
function isAbortError(err: unknown): boolean {
|
||||
return err instanceof Error && err.name === "AbortError";
|
||||
}
|
||||
|
||||
async function discoverFrameGraph(cdp: CdpRunner, tabId: number): Promise<CdpFrameGraph | null> {
|
||||
if (!cdp.getFrameGraph) return null;
|
||||
try {
|
||||
return await cdp.getFrameGraph(tabId);
|
||||
} catch (err) {
|
||||
console.debug("[bsk observation] frame graph capture failed", err);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function transformFrameNodes(
|
||||
nodes: CapturedNode[],
|
||||
projection: GeometryProjection,
|
||||
): CapturedNode[] {
|
||||
return nodes.map((node) => ({
|
||||
...node,
|
||||
rect: (() => {
|
||||
const bounds = projectRectToViewport(node.rect, projection);
|
||||
return bounds ? { x: bounds.x, y: bounds.y, w: bounds.width, h: bounds.height } : null;
|
||||
})(),
|
||||
}));
|
||||
}
|
||||
|
||||
function frameOwnerNode(captured: CapturedViewModel, frame: CdpFrame): CapturedNode | undefined {
|
||||
if (frame.ownerBackendNodeId === undefined) return undefined;
|
||||
const parentNodes = frame.parentFrameId
|
||||
? captured.frameNodes?.get(frame.parentFrameId)
|
||||
: captured.nodes;
|
||||
return parentNodes?.find((node) => node.backendNodeId === frame.ownerBackendNodeId);
|
||||
}
|
||||
|
||||
async function captureMissingFrameDocuments(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
graph: CdpFrameGraph,
|
||||
captured: CapturedViewModel,
|
||||
signal?: AbortSignal,
|
||||
): Promise<void> {
|
||||
if (!captured.frameNodes) captured.frameNodes = new Map();
|
||||
if (!captured.frameOwnerBackendNodeIds) captured.frameOwnerBackendNodeIds = new Map();
|
||||
if (!captured.frameParentIds) captured.frameParentIds = new Map();
|
||||
|
||||
for (const frame of graph.frames) {
|
||||
if (!frame.target.sessionId || captured.frameNodes.has(frame.frameId)) continue;
|
||||
const owner = frameOwnerNode(captured, frame);
|
||||
if (!owner?.rect || frame.ownerBackendNodeId === undefined) continue;
|
||||
try {
|
||||
const child = await captureViewModel(cdpRunnerForTarget(cdp, frame.target), tabId, {
|
||||
signal,
|
||||
});
|
||||
const projection = await resolveFrameProjection(cdp, graph, frame.frameId);
|
||||
if (!projection) continue;
|
||||
const rootFrameId = child.rootFrameId ?? frame.frameId;
|
||||
const childFrames = child.frameNodes ?? new Map<string, CapturedNode[]>();
|
||||
if (!childFrames.has(rootFrameId)) childFrames.set(rootFrameId, child.nodes);
|
||||
if (rootFrameId !== frame.frameId && !childFrames.has(frame.frameId)) {
|
||||
childFrames.set(
|
||||
frame.frameId,
|
||||
child.nodes.map((node) => ({ ...node, frameId: frame.frameId })),
|
||||
);
|
||||
}
|
||||
for (const [childFrameId, nodes] of childFrames) {
|
||||
captured.frameNodes.set(childFrameId, transformFrameNodes(nodes, projection));
|
||||
}
|
||||
captured.iframeNodes.set(
|
||||
frame.ownerBackendNodeId,
|
||||
captured.frameNodes.get(frame.frameId) ?? [],
|
||||
);
|
||||
captured.frameOwnerBackendNodeIds.set(frame.frameId, frame.ownerBackendNodeId);
|
||||
if (frame.parentFrameId) captured.frameParentIds.set(frame.frameId, frame.parentFrameId);
|
||||
for (const [childFrameId, ownerBackendNodeId] of child.frameOwnerBackendNodeIds ?? []) {
|
||||
captured.frameOwnerBackendNodeIds.set(childFrameId, ownerBackendNodeId);
|
||||
}
|
||||
for (const [childFrameId, parentFrameId] of child.frameParentIds ?? []) {
|
||||
captured.frameParentIds.set(childFrameId, parentFrameId);
|
||||
}
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
console.debug("[bsk observation] child frame DOM capture failed", {
|
||||
frameId: frame.frameId,
|
||||
err,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function captureAxTrees<T extends FrameAxNode>(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
discoveredFrames: CdpFrame[],
|
||||
captured: CapturedViewModel,
|
||||
signal?: AbortSignal,
|
||||
): Promise<FrameAxBatch<T>[]> {
|
||||
let frames = [...discoveredFrames];
|
||||
const capturedFrameIds = [...(captured.frameNodes?.keys() ?? [])];
|
||||
if (frames.length === 0) {
|
||||
const rootFrameId = captured.rootFrameId ?? capturedFrameIds[0] ?? "root";
|
||||
frames = [{ frameId: rootFrameId, target: { tabId } }];
|
||||
}
|
||||
for (const frameId of capturedFrameIds) {
|
||||
if (!frames.some((frame) => frame.frameId === frameId)) {
|
||||
frames.push({ frameId, target: { tabId } });
|
||||
}
|
||||
}
|
||||
|
||||
let firstFailure: unknown;
|
||||
const trees = await Promise.all(
|
||||
frames.map(async (frame): Promise<FrameAxBatch<T>> => {
|
||||
try {
|
||||
throwIfAborted(signal);
|
||||
await sendToCdpTarget(cdp, frame.target, "Accessibility.enable", {});
|
||||
const result = await sendToCdpTarget<{ nodes?: T[] }>(
|
||||
cdp,
|
||||
frame.target,
|
||||
"Accessibility.getFullAXTree",
|
||||
frame.frameId === "root" ? {} : { frameId: frame.frameId },
|
||||
);
|
||||
throwIfAborted(signal);
|
||||
return {
|
||||
frame,
|
||||
nodes: result.nodes ?? [],
|
||||
};
|
||||
} catch (err) {
|
||||
if (isAbortError(err)) throw err;
|
||||
firstFailure ??= err;
|
||||
console.debug("[bsk observation] frame AX capture failed", {
|
||||
frameId: frame.frameId,
|
||||
err,
|
||||
});
|
||||
return { frame, nodes: [] };
|
||||
}
|
||||
}),
|
||||
);
|
||||
const capturedNodeCount = [...captured.iframeNodes.values()].reduce(
|
||||
(count, nodes) => count + nodes.length,
|
||||
captured.nodes.length,
|
||||
);
|
||||
if (firstFailure && capturedNodeCount === 0 && trees.every((tree) => tree.nodes.length === 0)) {
|
||||
throw firstFailure;
|
||||
}
|
||||
return trees;
|
||||
}
|
||||
|
||||
export async function captureFrameData<T extends FrameAxNode>(
|
||||
cdp: CdpRunner,
|
||||
tabId: number,
|
||||
captured: CapturedViewModel,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CapturedFrameDocument<T>[]> {
|
||||
const graph = await discoverFrameGraph(cdp, tabId);
|
||||
const frames = graph?.frames ?? [];
|
||||
if (graph) await captureMissingFrameDocuments(cdp, tabId, graph, captured, signal);
|
||||
throwIfAborted(signal);
|
||||
const batches = await captureAxTrees<T>(cdp, tabId, frames, captured, signal);
|
||||
return buildFrameDocuments(graph, batches, captured);
|
||||
}
|
||||
@@ -0,0 +1,173 @@
|
||||
import {
|
||||
type CdpFrame,
|
||||
type CdpFrameGraph,
|
||||
type CdpTarget,
|
||||
cdpTargetKey,
|
||||
} from "@/browser-driver/frame-graph";
|
||||
import type { CapturedNode, CapturedViewModel } from "./capture";
|
||||
|
||||
export interface FrameOwnedAxNode {
|
||||
nodeId: string;
|
||||
frameId?: string;
|
||||
parentId?: string;
|
||||
childIds?: string[];
|
||||
backendDOMNodeId?: number;
|
||||
}
|
||||
|
||||
export interface FrameAxBatch<T extends FrameOwnedAxNode> {
|
||||
frame: CdpFrame;
|
||||
nodes: T[];
|
||||
}
|
||||
|
||||
export interface FrameDocument<T extends FrameOwnedAxNode> extends CdpFrame {
|
||||
contextScopeId: string;
|
||||
axNodes: T[];
|
||||
domNodes: CapturedNode[];
|
||||
}
|
||||
|
||||
interface Ownership {
|
||||
frameId: string;
|
||||
strength: number;
|
||||
}
|
||||
|
||||
interface OwnedCandidate<T> {
|
||||
node: T;
|
||||
target: CdpTarget;
|
||||
ownership: Ownership;
|
||||
}
|
||||
|
||||
function targetNodeKey(target: CdpTarget, nodeId: string): string {
|
||||
return `${cdpTargetKey(target)}:${nodeId}`;
|
||||
}
|
||||
|
||||
function targetBackendKey(target: CdpTarget, backendNodeId: number): string {
|
||||
return `${cdpTargetKey(target)}:${backendNodeId}`;
|
||||
}
|
||||
|
||||
function frameList<T extends FrameOwnedAxNode>(
|
||||
graph: CdpFrameGraph | null,
|
||||
batches: FrameAxBatch<T>[],
|
||||
captured: CapturedViewModel,
|
||||
): CdpFrame[] {
|
||||
const frames = new Map<string, CdpFrame>();
|
||||
for (const frame of graph?.frames ?? []) frames.set(frame.frameId, frame);
|
||||
for (const batch of batches) {
|
||||
if (!frames.has(batch.frame.frameId)) frames.set(batch.frame.frameId, batch.frame);
|
||||
}
|
||||
for (const frameId of captured.frameNodes?.keys() ?? []) {
|
||||
if (!frames.has(frameId))
|
||||
frames.set(frameId, { frameId, target: { tabId: batches[0]?.frame.target.tabId ?? 0 } });
|
||||
}
|
||||
return [...frames.values()].map((frame) => {
|
||||
const ownerBackendNodeId =
|
||||
frame.ownerBackendNodeId ?? captured.frameOwnerBackendNodeIds?.get(frame.frameId);
|
||||
const parentFrameId = frame.parentFrameId ?? captured.frameParentIds?.get(frame.frameId);
|
||||
return {
|
||||
...frame,
|
||||
...(ownerBackendNodeId !== undefined ? { ownerBackendNodeId } : {}),
|
||||
...(parentFrameId ? { parentFrameId } : {}),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export function buildFrameDocuments<T extends FrameOwnedAxNode>(
|
||||
graph: CdpFrameGraph | null,
|
||||
batches: FrameAxBatch<T>[],
|
||||
captured: CapturedViewModel,
|
||||
): FrameDocument<T>[] {
|
||||
const frames = frameList(graph, batches, captured);
|
||||
const frameById = new Map(frames.map((frame) => [frame.frameId, frame]));
|
||||
const rootFrameId = graph?.rootFrameId ?? captured.rootFrameId ?? frames[0]?.frameId;
|
||||
const domNodesForFrame = (frameId: string): CapturedNode[] =>
|
||||
captured.frameNodes?.get(frameId) ?? (frameId === rootFrameId ? captured.nodes : []);
|
||||
const backendOwner = new Map<string, string>();
|
||||
for (const frame of frames) {
|
||||
for (const node of domNodesForFrame(frame.frameId)) {
|
||||
backendOwner.set(targetBackendKey(frame.target, node.backendNodeId), frame.frameId);
|
||||
}
|
||||
}
|
||||
|
||||
const candidates = new Map<string, OwnedCandidate<T>>();
|
||||
for (const batch of batches) {
|
||||
const nodeById = new Map(batch.nodes.map((node) => [node.nodeId, node]));
|
||||
const ownershipByNodeId = new Map<string, Ownership>();
|
||||
const resolving = new Set<string>();
|
||||
const resolveOwnership = (node: T): Ownership => {
|
||||
const cached = ownershipByNodeId.get(node.nodeId);
|
||||
if (cached) return cached;
|
||||
if (node.frameId && frameById.has(node.frameId)) {
|
||||
const ownership = { frameId: node.frameId, strength: 4 };
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
}
|
||||
if (typeof node.backendDOMNodeId === "number") {
|
||||
const frameId = backendOwner.get(
|
||||
targetBackendKey(batch.frame.target, node.backendDOMNodeId),
|
||||
);
|
||||
if (frameId) {
|
||||
const ownership = { frameId, strength: 3 };
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
}
|
||||
}
|
||||
if (node.parentId && !resolving.has(node.nodeId)) {
|
||||
const parent = nodeById.get(node.parentId);
|
||||
if (parent) {
|
||||
resolving.add(node.nodeId);
|
||||
const inherited = resolveOwnership(parent);
|
||||
resolving.delete(node.nodeId);
|
||||
const ownership = {
|
||||
frameId: inherited.frameId,
|
||||
strength: Math.min(2, inherited.strength),
|
||||
};
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
}
|
||||
}
|
||||
const ownership = { frameId: batch.frame.frameId, strength: 1 };
|
||||
ownershipByNodeId.set(node.nodeId, ownership);
|
||||
return ownership;
|
||||
};
|
||||
|
||||
for (const node of batch.nodes) {
|
||||
const ownership = resolveOwnership(node);
|
||||
const key = targetNodeKey(batch.frame.target, node.nodeId);
|
||||
const existing = candidates.get(key);
|
||||
if (!existing || ownership.strength > existing.ownership.strength) {
|
||||
candidates.set(key, { node, target: batch.frame.target, ownership });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const ownershipByTargetNode = new Map(
|
||||
[...candidates].map(([key, candidate]) => [key, candidate.ownership.frameId]),
|
||||
);
|
||||
const axNodesByFrame = new Map<string, T[]>();
|
||||
for (const candidate of candidates.values()) {
|
||||
const { node, target, ownership } = candidate;
|
||||
const parentFrameId = node.parentId
|
||||
? ownershipByTargetNode.get(targetNodeKey(target, node.parentId))
|
||||
: undefined;
|
||||
const childIds = node.childIds?.filter(
|
||||
(childId) => ownershipByTargetNode.get(targetNodeKey(target, childId)) === ownership.frameId,
|
||||
);
|
||||
const ownedNode = {
|
||||
...node,
|
||||
frameId: ownership.frameId,
|
||||
...(parentFrameId === ownership.frameId
|
||||
? { parentId: node.parentId }
|
||||
: { parentId: undefined }),
|
||||
...(childIds ? { childIds } : {}),
|
||||
};
|
||||
const nodes = axNodesByFrame.get(ownership.frameId) ?? [];
|
||||
nodes.push(ownedNode);
|
||||
axNodesByFrame.set(ownership.frameId, nodes);
|
||||
}
|
||||
|
||||
return frames.map((frame) => ({
|
||||
...frame,
|
||||
contextScopeId: frame.frameId,
|
||||
axNodes: axNodesByFrame.get(frame.frameId) ?? [],
|
||||
domNodes: domNodesForFrame(frame.frameId),
|
||||
}));
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
import type { Rect, RenderedRef, VomResult } from "@browser-skill/vom";
|
||||
import type { CdpTarget } from "@/browser-driver/frame-graph";
|
||||
import type { CapturedSurfaceProbe } from "./capture";
|
||||
import type { CapturedFrameDocument, FrameAxNode } from "./frame-capture";
|
||||
|
||||
/** Frame identity needed to resolve an `@eN` that lives in an iframe. */
|
||||
export interface CaptureVomFrame {
|
||||
frameId: string;
|
||||
target: CdpTarget;
|
||||
parentFrameId?: string;
|
||||
ownerBackendNodeId?: number;
|
||||
}
|
||||
|
||||
/** Allowlisted geometry used to match a recorded action to a rendered ref. */
|
||||
export interface CaptureVomMatchNode {
|
||||
frameId: string;
|
||||
backendNodeId: number;
|
||||
tag: string;
|
||||
/** Top-level viewport-relative geometry. */
|
||||
rect: Rect | null;
|
||||
/** Frame-local viewport-relative geometry, when available. */
|
||||
localRect?: Rect | null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Raw AX/DOM trees are never returned. When `redactValues` is enabled by the
|
||||
* caller, form values in `text` are masked as well.
|
||||
*/
|
||||
export interface CaptureVomObservationResult {
|
||||
text: string;
|
||||
refs: RenderedRef[];
|
||||
truncated: boolean;
|
||||
rootFrameId: string;
|
||||
frames: CaptureVomFrame[];
|
||||
matchNodes: CaptureVomMatchNode[];
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
}
|
||||
|
||||
function projectFrames(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomFrame[] {
|
||||
return documents.map((document) => ({
|
||||
frameId: document.frameId,
|
||||
target: document.target,
|
||||
...(document.parentFrameId ? { parentFrameId: document.parentFrameId } : {}),
|
||||
...(document.ownerBackendNodeId !== undefined
|
||||
? { ownerBackendNodeId: document.ownerBackendNodeId }
|
||||
: {}),
|
||||
}));
|
||||
}
|
||||
|
||||
function projectMatchNodes(documents: CapturedFrameDocument<FrameAxNode>[]): CaptureVomMatchNode[] {
|
||||
return documents.flatMap((document) =>
|
||||
document.domNodes.map((node) => ({
|
||||
frameId: document.frameId,
|
||||
backendNodeId: node.backendNodeId,
|
||||
tag: node.tag,
|
||||
rect: node.rect,
|
||||
...(node.localRect !== undefined ? { localRect: node.localRect } : {}),
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
export function projectRecordSafeObservation(input: {
|
||||
rootFrameId: string;
|
||||
frameDocuments: CapturedFrameDocument<FrameAxNode>[];
|
||||
rendered: VomResult;
|
||||
surfaceProbes?: CapturedSurfaceProbe[];
|
||||
}): CaptureVomObservationResult {
|
||||
return {
|
||||
text: input.rendered.text,
|
||||
refs: input.rendered.refs,
|
||||
truncated: input.rendered.truncated,
|
||||
rootFrameId: input.rootFrameId,
|
||||
frames: projectFrames(input.frameDocuments),
|
||||
matchNodes: projectMatchNodes(input.frameDocuments),
|
||||
surfaceProbes: input.surfaceProbes,
|
||||
};
|
||||
}
|
||||
@@ -47,7 +47,9 @@ export async function handleWindowResize(
|
||||
manager: SessionManager,
|
||||
params: WindowResizeParams,
|
||||
api: WindowResizeApi = chromeWindowResizeApi,
|
||||
signal?: AbortSignal,
|
||||
): Promise<WindowResizeResult | RpcError> {
|
||||
if (signal?.aborted) return { code: "cancelled", message: "window_resize aborted" };
|
||||
const ctxOrErr = lookupSession(manager, params, "window_resize");
|
||||
if (isRpcError(ctxOrErr)) return ctxOrErr;
|
||||
const ctx = ctxOrErr;
|
||||
@@ -64,6 +66,7 @@ export async function handleWindowResize(
|
||||
}
|
||||
|
||||
try {
|
||||
if (signal?.aborted) return { code: "cancelled", message: "window_resize aborted" };
|
||||
await api.update(ctx.agentWindowId, {
|
||||
width: sizeOrErr.width,
|
||||
height: sizeOrErr.height,
|
||||
|
||||
@@ -198,6 +198,25 @@ describe("performHandshake", () => {
|
||||
result: { server: "browser-skill-daemon", protocol_version: "1.0" },
|
||||
});
|
||||
});
|
||||
|
||||
it("stops waiting immediately when the connection generation is aborted", async () => {
|
||||
const { transport } = deferredFakeTransport();
|
||||
const controller = new AbortController();
|
||||
const pending = performHandshake(
|
||||
transport,
|
||||
{
|
||||
instanceId: "x",
|
||||
browser: { name: "chrome", version: "131" },
|
||||
label: "",
|
||||
rpcId: "hs-abort",
|
||||
},
|
||||
{ signal: controller.signal, timeoutMs: 60_000 },
|
||||
);
|
||||
|
||||
controller.abort();
|
||||
|
||||
await expect(pending).rejects.toMatchObject({ name: "AbortError" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("detectBrowserMeta", () => {
|
||||
|
||||
@@ -49,6 +49,10 @@ class FakeSocket {
|
||||
this.emit("close", { code, reason: "server-gone" });
|
||||
}
|
||||
|
||||
emitClose(code = 1006): void {
|
||||
this.emit("close", { code, reason: "delayed-close" });
|
||||
}
|
||||
|
||||
private emit(type: string, ev: unknown): void {
|
||||
for (const l of this.listeners[type] ?? []) {
|
||||
// biome-ignore lint/suspicious/noExplicitAny: minimal fake mirrors WebSocket Event shape
|
||||
@@ -202,6 +206,34 @@ describe("WSTransport", () => {
|
||||
expect(FakeSocket.instances.length).toBe(beforeCount + 1);
|
||||
});
|
||||
|
||||
it("cancels pending backoff and ignores delayed events from a replaced socket", async () => {
|
||||
const t = new WSTransport({
|
||||
url: "ws://127.0.0.1:52800",
|
||||
webSocketFactory: (url) => new FakeSocket(url) as unknown as WebSocket,
|
||||
});
|
||||
const initial = t.connect();
|
||||
const first = lastSocket();
|
||||
first.open();
|
||||
await initial;
|
||||
|
||||
first.serverClose();
|
||||
const reconnect = t.connect();
|
||||
expect(FakeSocket.instances).toHaveLength(2);
|
||||
const second = lastSocket();
|
||||
second.open();
|
||||
await reconnect;
|
||||
|
||||
// A real browser can deliver the old close callback after the replacement
|
||||
// socket is already open. It must not clear or disconnect the new socket.
|
||||
first.emitClose();
|
||||
t.send({ id: "current", method: "system.ping" });
|
||||
expect(second.sent).toEqual([JSON.stringify({ id: "current", method: "system.ping" })]);
|
||||
expect(t.state).toBe("connected");
|
||||
|
||||
await vi.advanceTimersByTimeAsync(5_000);
|
||||
expect(FakeSocket.instances).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("ignores malformed inbound messages (non-JSON) without throwing", async () => {
|
||||
const handler = vi.fn();
|
||||
const t = new WSTransport({
|
||||
|
||||
@@ -57,7 +57,7 @@ function ridToString(): string {
|
||||
export function performHandshake(
|
||||
transport: Transport,
|
||||
input: HandshakeInput,
|
||||
options: { timeoutMs?: number } = {},
|
||||
options: { timeoutMs?: number; signal?: AbortSignal } = {},
|
||||
): Promise<HandshakeOutcome> {
|
||||
const id = input.rpcId ?? ridToString();
|
||||
const params: HandshakeParams = {
|
||||
@@ -75,6 +75,12 @@ export function performHandshake(
|
||||
const timeoutMs = options.timeoutMs ?? 10_000;
|
||||
|
||||
return new Promise<HandshakeOutcome>((resolve, reject) => {
|
||||
const onAbort = () => {
|
||||
cleanup();
|
||||
const err = new Error("[handshake] aborted because the connection changed");
|
||||
err.name = "AbortError";
|
||||
reject(err);
|
||||
};
|
||||
const timer = setTimeout(() => {
|
||||
cleanup();
|
||||
reject(new Error("[handshake] timed out waiting for daemon response"));
|
||||
@@ -97,8 +103,15 @@ export function performHandshake(
|
||||
function cleanup() {
|
||||
clearTimeout(timer);
|
||||
sub.dispose();
|
||||
options.signal?.removeEventListener("abort", onAbort);
|
||||
}
|
||||
|
||||
if (options.signal?.aborted) {
|
||||
onAbort();
|
||||
return;
|
||||
}
|
||||
options.signal?.addEventListener("abort", onAbort, { once: true });
|
||||
|
||||
try {
|
||||
transport.send(req);
|
||||
} catch (err) {
|
||||
|
||||
@@ -31,7 +31,9 @@ export type RpcErrorReason =
|
||||
| "single_select_value_count"
|
||||
| "tab_not_active"
|
||||
| "restricted_tab_url"
|
||||
| "borrow_conflict";
|
||||
| "borrow_conflict"
|
||||
| "screenshot_capture_failed"
|
||||
| "cleanup_failed";
|
||||
|
||||
export interface RpcErrorData {
|
||||
reason?: RpcErrorReason;
|
||||
|
||||
@@ -50,6 +50,7 @@ export class WSTransport implements Transport {
|
||||
private explicitlyClosed = false;
|
||||
private reconnectAttempt = 0;
|
||||
private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private socketGeneration = 0;
|
||||
private connectingPromise: Promise<void> | null = null;
|
||||
private resolveConnect: ((value: void) => void) | null = null;
|
||||
private rejectConnect: ((reason: Error) => void) | null = null;
|
||||
@@ -69,16 +70,26 @@ export class WSTransport implements Transport {
|
||||
}
|
||||
|
||||
connect(): Promise<void> {
|
||||
if (this.connectingPromise) return this.connectingPromise;
|
||||
if (this.currentState === "connected") return Promise.resolve();
|
||||
|
||||
this.explicitlyClosed = false;
|
||||
this.openSocket();
|
||||
if (this.reconnectTimer) {
|
||||
clearTimeout(this.reconnectTimer);
|
||||
this.reconnectTimer = null;
|
||||
}
|
||||
|
||||
if (this.connectingPromise) {
|
||||
// A previous physical attempt closed before reaching OPEN. Keep the
|
||||
// original caller's promise, but start the next socket generation.
|
||||
if (!this.socket) this.openSocket();
|
||||
return this.connectingPromise;
|
||||
}
|
||||
|
||||
this.connectingPromise = new Promise<void>((resolve, reject) => {
|
||||
this.resolveConnect = resolve;
|
||||
this.rejectConnect = reject;
|
||||
});
|
||||
this.openSocket();
|
||||
return this.connectingPromise;
|
||||
}
|
||||
|
||||
@@ -88,9 +99,12 @@ export class WSTransport implements Transport {
|
||||
clearTimeout(this.reconnectTimer);
|
||||
this.reconnectTimer = null;
|
||||
}
|
||||
if (this.socket) {
|
||||
const socket = this.socket;
|
||||
this.socket = null;
|
||||
this.socketGeneration += 1;
|
||||
if (socket) {
|
||||
try {
|
||||
this.socket.close();
|
||||
socket.close();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
@@ -135,10 +149,12 @@ export class WSTransport implements Transport {
|
||||
|
||||
private openSocket(): void {
|
||||
this.setState("connecting");
|
||||
const generation = ++this.socketGeneration;
|
||||
const socket = this.factory(this.url);
|
||||
this.socket = socket;
|
||||
|
||||
socket.addEventListener("open", () => {
|
||||
if (!this.isCurrentSocket(socket, generation)) return;
|
||||
this.reconnectAttempt = 0;
|
||||
this.setState("connected");
|
||||
const resolve = this.resolveConnect;
|
||||
@@ -149,11 +165,12 @@ export class WSTransport implements Transport {
|
||||
});
|
||||
|
||||
socket.addEventListener("message", (ev: MessageEvent) => {
|
||||
if (!this.isCurrentSocket(socket, generation)) return;
|
||||
this.handleInbound((ev as unknown as MessageLikeEvent).data);
|
||||
});
|
||||
|
||||
socket.addEventListener("close", (ev: Event) => {
|
||||
this.handleClose(ev as unknown as CloseLikeEvent);
|
||||
this.handleClose(socket, generation, ev as unknown as CloseLikeEvent);
|
||||
});
|
||||
|
||||
socket.addEventListener("error", () => {
|
||||
@@ -179,7 +196,8 @@ export class WSTransport implements Transport {
|
||||
}
|
||||
}
|
||||
|
||||
private handleClose(_ev: CloseLikeEvent): void {
|
||||
private handleClose(socket: WebSocket, generation: number, _ev: CloseLikeEvent): void {
|
||||
if (!this.isCurrentSocket(socket, generation)) return;
|
||||
this.socket = null;
|
||||
if (this.explicitlyClosed) {
|
||||
this.setState("disconnected");
|
||||
@@ -196,10 +214,16 @@ export class WSTransport implements Transport {
|
||||
this.reconnectTimer = setTimeout(() => {
|
||||
this.reconnectTimer = null;
|
||||
if (this.explicitlyClosed) return;
|
||||
this.openSocket();
|
||||
void this.connect().catch((err) => {
|
||||
console.debug("[WSTransport] reconnect attempt failed", err);
|
||||
});
|
||||
}, delay);
|
||||
}
|
||||
|
||||
private isCurrentSocket(socket: WebSocket, generation: number): boolean {
|
||||
return this.socket === socket && this.socketGeneration === generation;
|
||||
}
|
||||
|
||||
private setState(next: ConnectionState): void {
|
||||
if (this.currentState === next) return;
|
||||
this.currentState = next;
|
||||
|
||||
@@ -25,10 +25,8 @@ export default defineConfig({
|
||||
name: "BrowserSkill",
|
||||
description:
|
||||
"Let AI agents use your logged-in browser in a separate Agent Window—without interrupting your work. Powered by the bsk CLI.",
|
||||
// Chrome 116 is the first version where WebSocket activity resets the
|
||||
// service-worker idle timer, which the 20s heartbeat relies on to
|
||||
// keep the daemon link alive.
|
||||
minimum_chrome_version: "116",
|
||||
// Flat debugger sessions are required to address out-of-process iframes.
|
||||
minimum_chrome_version: "125",
|
||||
permissions: [
|
||||
"alarms",
|
||||
"debugger",
|
||||
|
||||
@@ -39,6 +39,11 @@
|
||||
"**",
|
||||
"!**/node_modules/**",
|
||||
"!**/dist/**",
|
||||
"!packages/dsh-plugin-browserskill/lib/**",
|
||||
"!packages/dsh-plugin-browserskill/src/skill-content.generated.ts",
|
||||
"!packages/dsh-plugin-browserskill/src/client/bsk-ui.nomodule.css",
|
||||
"!packages/dsh-plugin-browserskill/.bsk-ui.raw.css",
|
||||
"!packages/dsh-plugin-browserskill/.bsk-ui.input.css",
|
||||
"!**/target/**",
|
||||
"!**/.wxt/**",
|
||||
"!**/.output/**",
|
||||
|
||||
@@ -42,7 +42,7 @@ Every automation task **must** follow this lifecycle. Do **not** rely on idle ti
|
||||
3. bsk session stop <id> → REQUIRED when done (even on error paths)
|
||||
```
|
||||
|
||||
Optional: `bsk session start --browser <instance-id-or-label>` when multiple browsers are connected (`bsk browsers` / error output lists them).
|
||||
Optional: `bsk session start --browser <instance-id-or-label>` when multiple browsers are connected (`bsk browsers` / error output lists them). Add `--no-focus` to open the Agent Window in the background without stealing focus from the user's current window.
|
||||
|
||||
Emergency cleanup: `bsk session stop --all` or the Agent Window overlay **Stop all**.
|
||||
|
||||
@@ -108,6 +108,8 @@ Do **not** call `bsk get-html` or `bsk screenshot` first just to inspect a page.
|
||||
| `--quiet` | Suppress informational stderr |
|
||||
| `-v` / `-vv` | More verbose logging |
|
||||
|
||||
Auto-update is **on by default** — the background daemon upgrades `bsk` itself when a new release is available (postponed while a session is active). Set `BSK_AUTO_UPDATE=off` to disable it and upgrade manually with `bsk update`.
|
||||
|
||||
Command-specific flags (timeouts, `--tab-id`, `--wait-until`, …): **`bsk <cmd> --help`**
|
||||
|
||||
## CLI command reference (one line each)
|
||||
@@ -127,6 +129,7 @@ Details and flags: **`bsk <cmd> --help`**
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk session start` | Open Agent Window (`--width`/`--height` for initial size); prints **4-letter session id** |
|
||||
| `bsk session start --no-focus` | Open Agent Window in the background without stealing focus |
|
||||
| `bsk session stop <id>` | End session, close Agent Window, auto-return borrowed tabs |
|
||||
| `bsk session stop --all` | Stop every active session |
|
||||
| `bsk session list` | List active sessions |
|
||||
@@ -173,6 +176,15 @@ bsk emulate --session <id> --off
|
||||
| `bsk get-html` | Raw HTML dump after snapshot is insufficient (high token cost) |
|
||||
| `bsk screenshot` | PNG capture after snapshot is insufficient: full visible tab, or `--ref @eN` to crop to one element (`--out` path optional) |
|
||||
|
||||
### Console & network debugging (read-only; require `--session`)
|
||||
|
||||
| Command | Summary |
|
||||
|---------|---------|
|
||||
| `bsk console` | Buffered page console messages, JS exceptions, and browser log entries (`--include-stack` for stack traces) |
|
||||
| `bsk network` | Buffered network responses (status, method, URL, MIME/resource type) and failures (`net::ERR_*` reason) |
|
||||
|
||||
Both capture from the moment the tab is attached and read a bounded per-tab buffer: `--since <seq>` pages from a cursor (`next_since` in the result), `--limit` (default 50, max 200), `--max-text-chars` (default 1000, max 4096), `--tab-id` to target a non-active tab. Both are strictly read-only — they never intercept or modify traffic, and request/response headers, bodies, and timings are not captured.
|
||||
|
||||
### Navigation
|
||||
|
||||
| Command | Summary |
|
||||
|
||||
@@ -18,6 +18,12 @@ use crate::daemon::state::PROTOCOL_VERSION;
|
||||
const EXTENSION_STORE_URL: &str =
|
||||
"https://chromewebstore.google.com/detail/hhcmgoofomhgciiibhipgmgkgnoenaoi";
|
||||
|
||||
/// Edge Add-ons listing for the browser-skill extension.
|
||||
const EXTENSION_STORE_URL_EDGE: &str = "https://microsoftedge.microsoft.com/addons/detail/browserskill/emacgiaaaiojkkpkddmmdfhmokgmnikg";
|
||||
|
||||
/// Store listings highlighted in repair hints, in the order they appear.
|
||||
const EXTENSION_STORE_URLS: [&str; 2] = [EXTENSION_STORE_URL, EXTENSION_STORE_URL_EDGE];
|
||||
|
||||
/// Status of a single doctor check. `Ok` / `Fail` are the legacy two
|
||||
/// states; `NotApplicable` (review M2) is reported as "N/A" in human
|
||||
/// output and as `"status": "na"` in `--json` output, so a check that
|
||||
@@ -420,7 +426,10 @@ fn check_extension_connected(status: Option<&StatusResult>) -> CheckResult {
|
||||
CheckResult::fail(
|
||||
name,
|
||||
"0 browsers connected",
|
||||
format!("install the extension from {EXTENSION_STORE_URL} and load it in Chromium"),
|
||||
format!(
|
||||
"install the extension from {EXTENSION_STORE_URL} (Chrome) \
|
||||
or {EXTENSION_STORE_URL_EDGE} (Edge) and load it in the browser"
|
||||
),
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -428,17 +437,14 @@ fn check_extension_connected(status: Option<&StatusResult>) -> CheckResult {
|
||||
/// Highlight known URLs in repair hints for terminal output. Plain
|
||||
/// text is preserved in `--json` and in stored [`CheckResult::hint`].
|
||||
fn style_hint(hint: &str) -> String {
|
||||
if !hint.contains(EXTENSION_STORE_URL) {
|
||||
return hint.to_string();
|
||||
let mut styled = hint.to_string();
|
||||
for url in EXTENSION_STORE_URLS {
|
||||
if !styled.contains(url) {
|
||||
continue;
|
||||
}
|
||||
styled = styled.replace(url, &style(url).cyan().bold().underlined().to_string());
|
||||
}
|
||||
hint.replace(
|
||||
EXTENSION_STORE_URL,
|
||||
&style(EXTENSION_STORE_URL)
|
||||
.cyan()
|
||||
.bold()
|
||||
.underlined()
|
||||
.to_string(),
|
||||
)
|
||||
styled
|
||||
}
|
||||
|
||||
fn render_human(checks: &[CheckResult]) {
|
||||
@@ -504,6 +510,26 @@ mod m2_tests {
|
||||
hint.contains(EXTENSION_STORE_URL),
|
||||
"hint should include Chrome Web Store URL: {hint}"
|
||||
);
|
||||
assert!(
|
||||
hint.contains(EXTENSION_STORE_URL_EDGE),
|
||||
"hint should include Edge Add-ons URL: {hint}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn style_hint_preserves_every_store_url() {
|
||||
let hint = check_extension_connected(Some(&fake_status(Vec::new(), Vec::new())))
|
||||
.hint
|
||||
.expect("extension disconnected should include a hint");
|
||||
let styled = style_hint(&hint);
|
||||
// Whether or not the terminal accepts colors, styling must never drop or
|
||||
// mangle a URL the user has to click.
|
||||
for url in EXTENSION_STORE_URLS {
|
||||
assert!(
|
||||
styled.contains(url),
|
||||
"styled hint should still contain {url}: {styled}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user